add RTCRtpSender.generateKeyFrame

defined in
  https://w3c.github.io/webrtc-encoded-transform/#rtcrtpsender-extension

Note: this does not implement the "rid(s)" parameter which will be done in a future CL.

VP8 still synchronizes keyframes on all layers even when asked for ones on individual layers while H264 (when implemented as three different encoders in SimulcastEncoderAdapter) can actually utilize this.

This does not change the behavior when receiving a RTCP PLI for a particular layer.

BUG=chromium:1354101

Change-Id: Ic8b14d155242e32c9aeafa55fe6652f346ac76b8
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/274169
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Philipp Hancke <phancke@microsoft.com>
Cr-Commit-Position: refs/heads/main@{#38472}

api/rtp_sender_interface.h

@@ -104,6 +104,9 @@ class RTC_EXPORT RtpSenderInterface : public rtc::RefCountInterface {
       std::unique_ptr<VideoEncoderFactory::EncoderSelectorInterface>
           encoder_selector) = 0;
 
+  // TODO(crbug.com/1354101): make pure virtual again after Chrome roll.
+  virtual RTCError GenerateKeyFrame() { return RTCError::OK(); }
+
  protected:
   ~RtpSenderInterface() override = default;
 };

call/video_send_stream.h

@@ -252,6 +252,8 @@ class VideoSendStream {
 
   virtual Stats GetStats() = 0;
 
+  virtual void GenerateKeyFrame() = 0;
+
  protected:
   virtual ~VideoSendStream() {}
 };

media/base/fake_media_engine.cc

@@ -427,7 +427,8 @@ void FakeVideoMediaChannel::SetRecordableEncodedFrameCallback(
 void FakeVideoMediaChannel::ClearRecordableEncodedFrameCallback(uint32_t ssrc) {
 }
 
-void FakeVideoMediaChannel::GenerateKeyFrame(uint32_t ssrc) {}
+void FakeVideoMediaChannel::RequestRecvKeyFrame(uint32_t ssrc) {}
+void FakeVideoMediaChannel::GenerateSendKeyFrame(uint32_t ssrc) {}
 
 FakeVoiceEngine::FakeVoiceEngine() : fail_create_channel_(false) {
   // Add a fake audio codec. Note that the name must not be "" as there are

media/base/fake_media_engine.h

@@ -462,7 +462,8 @@ class FakeVideoMediaChannel : public RtpHelper<VideoMediaChannel> {
       std::function<void(const webrtc::RecordableEncodedFrame&)> callback)
       override;
   void ClearRecordableEncodedFrameCallback(uint32_t ssrc) override;
-  void GenerateKeyFrame(uint32_t ssrc) override;
+  void RequestRecvKeyFrame(uint32_t ssrc) override;
+  void GenerateSendKeyFrame(uint32_t ssrc) override;
 
  private:
   bool SetRecvCodecs(const std::vector<VideoCodec>& codecs);

media/base/media_channel.h

@@ -921,8 +921,11 @@ class VideoMediaChannel : public MediaChannel, public Delayable {
       std::function<void(const webrtc::RecordableEncodedFrame&)> callback) = 0;
   // Clear recordable encoded frame callback for `ssrc`
   virtual void ClearRecordableEncodedFrameCallback(uint32_t ssrc) = 0;
-  // Cause generation of a keyframe for `ssrc`
+  // Request generation of a keyframe for `ssrc` on a receiving channel via
-  virtual void GenerateKeyFrame(uint32_t ssrc) = 0;
+  // RTCP feedback.
+  virtual void RequestRecvKeyFrame(uint32_t ssrc) = 0;
+  // Cause generation of a keyframe for `ssrc` on a sending channel.
+  virtual void GenerateSendKeyFrame(uint32_t ssrc) = 0;
 
   virtual std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const = 0;
 };

media/engine/fake_webrtc_call.h

@@ -194,6 +194,7 @@ class FakeVideoSendStream final
   rtc::VideoSourceInterface<webrtc::VideoFrame>* source() const {
     return source_;
   }
+  void GenerateKeyFrame() override {}
 
  private:
   // rtc::VideoSinkInterface<VideoFrame> implementation.

media/engine/webrtc_video_engine.cc

@@ -2827,6 +2827,16 @@ void WebRtcVideoChannel::WebRtcVideoSendStream::RecreateWebRtcStream() {
   }
 }
 
+void WebRtcVideoChannel::WebRtcVideoSendStream::GenerateKeyFrame() {
+  RTC_DCHECK_RUN_ON(&thread_checker_);
+  if (stream_ != NULL) {
+    stream_->GenerateKeyFrame();
+  } else {
+    RTC_LOG(LS_WARNING)
+        << "Absent send stream; ignoring request to generate keyframe.";
+  }
+}
+
 WebRtcVideoChannel::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream(
     WebRtcVideoChannel* channel,
     webrtc::Call* call,
@@ -3551,11 +3561,11 @@ void WebRtcVideoChannel::ClearRecordableEncodedFrameCallback(uint32_t ssrc) {
   }
 }
 
-void WebRtcVideoChannel::GenerateKeyFrame(uint32_t ssrc) {
+void WebRtcVideoChannel::RequestRecvKeyFrame(uint32_t ssrc) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc);
   if (stream) {
-    stream->GenerateKeyFrame();
+    return stream->GenerateKeyFrame();
   } else {
     RTC_LOG(LS_ERROR)
         << "Absent receive stream; ignoring key frame generation for ssrc "
@@ -3563,6 +3573,18 @@ void WebRtcVideoChannel::GenerateKeyFrame(uint32_t ssrc) {
   }
 }
 
+void WebRtcVideoChannel::GenerateSendKeyFrame(uint32_t ssrc) {
+  RTC_DCHECK_RUN_ON(&thread_checker_);
+  auto it = send_streams_.find(ssrc);
+  if (it != send_streams_.end()) {
+    it->second->GenerateKeyFrame();
+  } else {
+    RTC_LOG(LS_ERROR)
+        << "Absent send stream; ignoring key frame generation for ssrc "
+        << ssrc;
+  }
+}
+
 void WebRtcVideoChannel::SetEncoderToPacketizerFrameTransformer(
     uint32_t ssrc,
     rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {

media/engine/webrtc_video_engine.h

@@ -248,7 +248,8 @@ class WebRtcVideoChannel : public VideoMediaChannel,
       std::function<void(const webrtc::RecordableEncodedFrame&)> callback)
       override;
   void ClearRecordableEncodedFrameCallback(uint32_t ssrc) override;
-  void GenerateKeyFrame(uint32_t ssrc) override;
+  void RequestRecvKeyFrame(uint32_t ssrc) override;
+  void GenerateSendKeyFrame(uint32_t ssrc) override;
 
   void SetEncoderToPacketizerFrameTransformer(
       uint32_t ssrc,
@@ -390,6 +391,7 @@ class WebRtcVideoChannel : public VideoMediaChannel,
     void SetEncoderToPacketizerFrameTransformer(
         rtc::scoped_refptr<webrtc::FrameTransformerInterface>
             frame_transformer);
+    void GenerateKeyFrame();
 
    private:
     // Parameters needed to reconstruct the underlying stream.

pc/rtp_sender.cc

@@ -596,6 +596,13 @@ rtc::scoped_refptr<DtmfSenderInterface> AudioRtpSender::GetDtmfSender() const {
   return dtmf_sender_proxy_;
 }
 
+RTCError AudioRtpSender::GenerateKeyFrame() {
+  RTC_DCHECK_RUN_ON(signaling_thread_);
+  RTC_DLOG(LS_ERROR) << "Tried to get generate a key frame for audio.";
+  return RTCError(RTCErrorType::UNSUPPORTED_OPERATION,
+                  "Generating key frames for audio is not supported.");
+}
+
 void AudioRtpSender::SetSend() {
   RTC_DCHECK_RUN_ON(signaling_thread_);
   RTC_DCHECK(!stopped_);
@@ -686,6 +693,18 @@ rtc::scoped_refptr<DtmfSenderInterface> VideoRtpSender::GetDtmfSender() const {
   return nullptr;
 }
 
+RTCError VideoRtpSender::GenerateKeyFrame() {
+  RTC_DCHECK_RUN_ON(signaling_thread_);
+  if (video_media_channel() && ssrc_ && !stopped_) {
+    worker_thread_->PostTask(
+        [&] { video_media_channel()->GenerateSendKeyFrame(ssrc_); });
+  } else {
+    RTC_LOG(LS_WARNING) << "Tried to generate key frame for sender that is "
+                           "stopped or has no media channel.";
+  }
+  return RTCError::OK();
+}
+
 void VideoRtpSender::SetSend() {
   RTC_DCHECK_RUN_ON(signaling_thread_);
   RTC_DCHECK(!stopped_);

pc/rtp_sender.h

@@ -351,6 +351,7 @@ class AudioRtpSender : public DtmfProviderInterface, public RtpSenderBase {
   }
 
   rtc::scoped_refptr<DtmfSenderInterface> GetDtmfSender() const override;
+  RTCError GenerateKeyFrame() override;
 
  protected:
   AudioRtpSender(rtc::Thread* worker_thread,
@@ -410,6 +411,7 @@ class VideoRtpSender : public RtpSenderBase {
   }
 
   rtc::scoped_refptr<DtmfSenderInterface> GetDtmfSender() const override;
+  RTCError GenerateKeyFrame() override;
 
   RTCError CheckSVCParameters(const RtpParameters& parameters) override;
 

pc/rtp_sender_proxy.h

@@ -48,6 +48,7 @@ PROXY_METHOD1(void,
 PROXY_METHOD1(void,
               SetEncoderSelector,
               std::unique_ptr<VideoEncoderFactory::EncoderSelectorInterface>)
+PROXY_METHOD0(RTCError, GenerateKeyFrame)
 END_PROXY_MAP(RtpSender)
 
 }  // namespace webrtc

pc/video_rtp_receiver.cc

@@ -279,7 +279,7 @@ void VideoRtpReceiver::SetMediaChannel_w(cricket::MediaChannel* media_channel) {
   if (media_channel_) {
     if (saved_generate_keyframe_) {
       // TODO(bugs.webrtc.org/8694): Stop using 0 to mean unsignalled SSRC
-      media_channel_->GenerateKeyFrame(ssrc_.value_or(0));
+      media_channel_->RequestRecvKeyFrame(ssrc_.value_or(0));
       saved_generate_keyframe_ = false;
     }
     if (encoded_sink_enabled) {
@@ -331,7 +331,7 @@ void VideoRtpReceiver::OnGenerateKeyFrame() {
     return;
   }
   // TODO(bugs.webrtc.org/8694): Stop using 0 to mean unsignalled SSRC
-  media_channel_->GenerateKeyFrame(ssrc_.value_or(0));
+  media_channel_->RequestRecvKeyFrame(ssrc_.value_or(0));
   // We need to remember to request generation of a new key frame if the media
   // channel changes, because there's no feedback whether the keyframe
   // generation has completed on the channel.

pc/video_rtp_receiver_unittest.cc

@@ -49,7 +49,8 @@ class VideoRtpReceiverTest : public testing::Test {
                 ClearRecordableEncodedFrameCallback,
                 (uint32_t),
                 (override));
-    MOCK_METHOD(void, GenerateKeyFrame, (uint32_t), (override));
+    MOCK_METHOD(void, RequestRecvKeyFrame, (uint32_t), (override));
+    MOCK_METHOD(void, GenerateSendKeyFrame, (uint32_t), (override));
   };
 
   class MockVideoSink : public rtc::VideoSinkInterface<RecordableEncodedFrame> {
@@ -96,7 +97,7 @@ TEST_F(VideoRtpReceiverTest, SupportsEncodedOutput) {
 }
 
 TEST_F(VideoRtpReceiverTest, GeneratesKeyFrame) {
-  EXPECT_CALL(channel_, GenerateKeyFrame(0));
+  EXPECT_CALL(channel_, RequestRecvKeyFrame(0));
   Source()->GenerateKeyFrame();
 }
 
@@ -105,17 +106,17 @@ TEST_F(VideoRtpReceiverTest,
   // A channel switch without previous call to GenerateKeyFrame shouldn't
   // cause a call to happen on the new channel.
   MockVideoMediaChannel channel2(nullptr, cricket::VideoOptions());
-  EXPECT_CALL(channel_, GenerateKeyFrame).Times(0);
+  EXPECT_CALL(channel_, RequestRecvKeyFrame).Times(0);
-  EXPECT_CALL(channel2, GenerateKeyFrame).Times(0);
+  EXPECT_CALL(channel2, RequestRecvKeyFrame).Times(0);
   SetMediaChannel(&channel2);
   Mock::VerifyAndClearExpectations(&channel2);
 
   // Generate a key frame. When we switch channel next time, we will have to
   // re-generate it as we don't know if it was eventually received
-  EXPECT_CALL(channel2, GenerateKeyFrame).Times(1);
+  EXPECT_CALL(channel2, RequestRecvKeyFrame).Times(1);
   Source()->GenerateKeyFrame();
   MockVideoMediaChannel channel3(nullptr, cricket::VideoOptions());
-  EXPECT_CALL(channel3, GenerateKeyFrame);
+  EXPECT_CALL(channel3, RequestRecvKeyFrame);
   SetMediaChannel(&channel3);
 
   // Switching to a new channel should now not cause calls to GenerateKeyFrame.

video/video_send_stream.cc

@@ -343,5 +343,11 @@ void VideoSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
   send_stream_.DeliverRtcp(packet, length);
 }
 
+void VideoSendStream::GenerateKeyFrame() {
+  if (video_stream_encoder_) {
+    video_stream_encoder_->SendKeyFrame();
+  }
+}
+
 }  // namespace internal
 }  // namespace webrtc

video/video_send_stream.h

@@ -93,6 +93,7 @@ class VideoSendStream : public webrtc::VideoSendStream {
 
   void StopPermanentlyAndGetRtpStates(RtpStateMap* rtp_state_map,
                                       RtpPayloadStateMap* payload_state_map);
+  void GenerateKeyFrame() override;
 
  private:
   friend class test::VideoSendStreamPeer;