Insert audio frame transformer between encoder and packetizer.

The frame transformer is passed from RTPSenderInterface through the
library to be eventually set in ChannelSend, where the frame
transformation will occur in the follow-up CL.

Insertable Streams Web API explainer:
https://github.com/alvestrand/webrtc-media-streams/blob/master/explainer.md

Design doc for WebRTC library changes:
http://doc/1eiLkjNUkRy2FssCPLUp6eH08BZuXXoHfbbBP1ZN7EVk

Bug: webrtc:11380
Change-Id: I01b2adc3c96b948d182d5401a9a4fe14cf5960a2
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/171870
Commit-Queue: Tommi <tommi@webrtc.org>
Reviewed-by: Per Åhgren <peah@webrtc.org>
Reviewed-by: Tommi <tommi@webrtc.org>
Reviewed-by: Magnus Flodman <mflodman@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30946}

audio/BUILD.gn

@@ -38,6 +38,7 @@ rtc_library("audio") {
   deps = [
     "../api:array_view",
     "../api:call_api",
+    "../api:frame_transformer_interface",
     "../api:function_view",
     "../api:rtp_headers",
     "../api:rtp_parameters",
@@ -84,6 +85,7 @@ rtc_library("audio") {
     "../rtc_base:rtc_task_queue",
     "../rtc_base:safe_minmax",
     "../rtc_base/experiments:field_trial_parser",
+    "../rtc_base/synchronization:sequence_checker",
     "../system_wrappers",
     "../system_wrappers:field_trial",
     "../system_wrappers:metrics",

audio/audio_send_stream.cc

@@ -127,7 +127,8 @@ AudioSendStream::AudioSendStream(
                                              config.crypto_options,
                                              config.rtp.extmap_allow_mixed,
                                              config.rtcp_report_interval_ms,
-                                             config.rtp.ssrc)) {}
+                                             config.rtp.ssrc,
+                                             config.frame_transformer)) {}
 
 AudioSendStream::AudioSendStream(
     Clock* clock,
@@ -249,6 +250,12 @@ void AudioSendStream::ConfigureStream(
     channel_send_->SetFrameEncryptor(new_config.frame_encryptor);
   }
 
+  if (first_time ||
+      new_config.frame_transformer != old_config.frame_transformer) {
+    channel_send_->SetEncoderToPacketizerFrameTransformer(
+        new_config.frame_transformer);
+  }
+
   if (first_time ||
       new_config.rtp.extmap_allow_mixed != old_config.rtp.extmap_allow_mixed) {
     rtp_rtcp_module_->SetExtmapAllowMixed(new_config.rtp.extmap_allow_mixed);

audio/audio_send_stream_unittest.cc

@@ -215,6 +215,8 @@ struct ConfigHelper {
     EXPECT_CALL(rtp_rtcp_, SSRC).WillRepeatedly(Return(kSsrc));
     EXPECT_CALL(*channel_send_, SetRTCP_CNAME(StrEq(kCName))).Times(1);
     EXPECT_CALL(*channel_send_, SetFrameEncryptor(_)).Times(1);
+    EXPECT_CALL(*channel_send_, SetEncoderToPacketizerFrameTransformer(_))
+        .Times(1);
     EXPECT_CALL(rtp_rtcp_, SetExtmapAllowMixed(false)).Times(1);
     EXPECT_CALL(*channel_send_,
                 SetSendAudioLevelIndicationStatus(true, kAudioLevelId))

audio/channel_send.cc

@@ -76,7 +76,8 @@ class ChannelSend : public ChannelSendInterface,
               const webrtc::CryptoOptions& crypto_options,
               bool extmap_allow_mixed,
               int rtcp_report_interval_ms,
-              uint32_t ssrc);
+              uint32_t ssrc,
+              rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
 
   ~ChannelSend() override;
 
@@ -142,6 +143,12 @@ class ChannelSend : public ChannelSendInterface,
   void SetFrameEncryptor(
       rtc::scoped_refptr<FrameEncryptorInterface> frame_encryptor) override;
 
+  // Sets a frame transformer between encoder and packetizer, to transform
+  // encoded frames before sending them out the network.
+  void SetEncoderToPacketizerFrameTransformer(
+      rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)
+      override;
+
  private:
   // From AudioPacketizationCallback in the ACM
   int32_t SendData(AudioFrameType frameType,
@@ -217,6 +224,10 @@ class ChannelSend : public ChannelSendInterface,
   // E2EE Frame Encryption Options
   const webrtc::CryptoOptions crypto_options_;
 
+  // Frame transformer used by insertable streams to transform encoded frames.
+  rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_
+      RTC_GUARDED_BY(encoder_queue_);
+
   rtc::CriticalSection bitrate_crit_section_;
   int configured_bitrate_bps_ RTC_GUARDED_BY(bitrate_crit_section_) = 0;
 
@@ -452,18 +463,20 @@ int32_t ChannelSend::SendRtpAudio(AudioFrameType frameType,
   return 0;
 }
 
-ChannelSend::ChannelSend(Clock* clock,
+ChannelSend::ChannelSend(
-                         TaskQueueFactory* task_queue_factory,
+    Clock* clock,
-                         ProcessThread* module_process_thread,
+    TaskQueueFactory* task_queue_factory,
-                         OverheadObserver* overhead_observer,
+    ProcessThread* module_process_thread,
-                         Transport* rtp_transport,
+    OverheadObserver* overhead_observer,
-                         RtcpRttStats* rtcp_rtt_stats,
+    Transport* rtp_transport,
-                         RtcEventLog* rtc_event_log,
+    RtcpRttStats* rtcp_rtt_stats,
-                         FrameEncryptorInterface* frame_encryptor,
+    RtcEventLog* rtc_event_log,
-                         const webrtc::CryptoOptions& crypto_options,
+    FrameEncryptorInterface* frame_encryptor,
-                         bool extmap_allow_mixed,
+    const webrtc::CryptoOptions& crypto_options,
-                         int rtcp_report_interval_ms,
+    bool extmap_allow_mixed,
-                         uint32_t ssrc)
+    int rtcp_report_interval_ms,
+    uint32_t ssrc,
+    rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)
     : event_log_(rtc_event_log),
       _timeStamp(0),  // This is just an offset, RTP module will add it's own
                       // random offset
@@ -478,6 +491,7 @@ ChannelSend::ChannelSend(Clock* clock,
           new RateLimiter(clock, kMaxRetransmissionWindowMs)),
       frame_encryptor_(frame_encryptor),
       crypto_options_(crypto_options),
+      frame_transformer_(std::move(frame_transformer)),
       encoder_queue_(task_queue_factory->CreateTaskQueue(
           "AudioEncoder",
           TaskQueueFactory::Priority::NORMAL)) {
@@ -898,6 +912,16 @@ void ChannelSend::SetFrameEncryptor(
   });
 }
 
+void ChannelSend::SetEncoderToPacketizerFrameTransformer(
+    rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
+  RTC_DCHECK_RUN_ON(&worker_thread_checker_);
+  encoder_queue_.PostTask(
+      [this, frame_transformer = std::move(frame_transformer)]() mutable {
+        RTC_DCHECK_RUN_ON(&encoder_queue_);
+        frame_transformer_ = std::move(frame_transformer);
+      });
+}
+
 void ChannelSend::OnReceivedRtt(int64_t rtt_ms) {
   // Invoke audio encoders OnReceivedRtt().
   CallEncoder(
@@ -918,11 +942,13 @@ std::unique_ptr<ChannelSendInterface> CreateChannelSend(
     const webrtc::CryptoOptions& crypto_options,
     bool extmap_allow_mixed,
     int rtcp_report_interval_ms,
-    uint32_t ssrc) {
+    uint32_t ssrc,
+    rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) {
   return std::make_unique<ChannelSend>(
       clock, task_queue_factory, module_process_thread, overhead_observer,
       rtp_transport, rtcp_rtt_stats, rtc_event_log, frame_encryptor,
-      crypto_options, extmap_allow_mixed, rtcp_report_interval_ms, ssrc);
+      crypto_options, extmap_allow_mixed, rtcp_report_interval_ms, ssrc,
+      std::move(frame_transformer));
 }
 
 }  // namespace voe

audio/channel_send.h

@@ -18,6 +18,7 @@
 #include "api/audio/audio_frame.h"
 #include "api/audio_codecs/audio_encoder.h"
 #include "api/crypto/crypto_options.h"
+#include "api/frame_transformer_interface.h"
 #include "api/function_view.h"
 #include "api/task_queue/task_queue_factory.h"
 #include "modules/rtp_rtcp/include/report_block_data.h"
@@ -115,6 +116,12 @@ class ChannelSendInterface {
   // E2EE Custom Audio Frame Encryption (Optional)
   virtual void SetFrameEncryptor(
       rtc::scoped_refptr<FrameEncryptorInterface> frame_encryptor) = 0;
+
+  // Sets a frame transformer between encoder and packetizer, to transform
+  // encoded frames before sending them out the network.
+  virtual void SetEncoderToPacketizerFrameTransformer(
+      rtc::scoped_refptr<webrtc::FrameTransformerInterface>
+          frame_transformer) = 0;
 };
 
 std::unique_ptr<ChannelSendInterface> CreateChannelSend(
@@ -129,7 +136,8 @@ std::unique_ptr<ChannelSendInterface> CreateChannelSend(
     const webrtc::CryptoOptions& crypto_options,
     bool extmap_allow_mixed,
     int rtcp_report_interval_ms,
-    uint32_t ssrc);
+    uint32_t ssrc,
+    rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
 
 }  // namespace voe
 }  // namespace webrtc

audio/mock_voe_channel_proxy.h

@@ -118,6 +118,9 @@ class MockChannelSend : public voe::ChannelSendInterface {
   MOCK_METHOD1(
       SetFrameEncryptor,
       void(rtc::scoped_refptr<FrameEncryptorInterface> frame_encryptor));
+  MOCK_METHOD1(SetEncoderToPacketizerFrameTransformer,
+               void(rtc::scoped_refptr<webrtc::FrameTransformerInterface>
+                        frame_transformer));
 };
 }  // namespace test
 }  // namespace webrtc

call/BUILD.gn

@@ -32,6 +32,7 @@ rtc_library("call_interfaces") {
     ":rtp_interfaces",
     ":video_stream_api",
     "../api:fec_controller_api",
+    "../api:frame_transformer_interface",
     "../api:network_state_predictor_api",
     "../api:rtc_error",
     "../api:rtp_headers",

call/audio_send_stream.h

@@ -23,6 +23,7 @@
 #include "api/call/transport.h"
 #include "api/crypto/crypto_options.h"
 #include "api/crypto/frame_encryptor_interface.h"
+#include "api/frame_transformer_interface.h"
 #include "api/rtp_parameters.h"
 #include "api/scoped_refptr.h"
 #include "call/audio_sender.h"
@@ -157,6 +158,10 @@ class AudioSendStream : public AudioSender {
     // encryptor in whatever way the caller choses. This is not required by
     // default.
     rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor;
+
+    // An optional frame transformer used by insertable streams to transform
+    // encoded frames.
+    rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer;
   };
 
   virtual ~AudioSendStream() = default;

media/engine/webrtc_voice_engine.cc

@@ -949,6 +949,13 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream
     return webrtc::RTCError::OK();
   }
 
+  void SetEncoderToPacketizerFrameTransformer(
+      rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
+    RTC_DCHECK(worker_thread_checker_.IsCurrent());
+    config_.frame_transformer = std::move(frame_transformer);
+    ReconfigureAudioSendStream();
+  }
+
  private:
   void UpdateSendState() {
     RTC_DCHECK(worker_thread_checker_.IsCurrent());
@@ -2316,6 +2323,20 @@ std::vector<webrtc::RtpSource> WebRtcVoiceMediaChannel::GetSources(
   return it->second->GetSources();
 }
 
+void WebRtcVoiceMediaChannel::SetEncoderToPacketizerFrameTransformer(
+    uint32_t ssrc,
+    rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
+  RTC_DCHECK(worker_thread_checker_.IsCurrent());
+  auto matching_stream = send_streams_.find(ssrc);
+  if (matching_stream == send_streams_.end()) {
+    RTC_LOG(LS_INFO) << "Attempting to set frame transformer for SSRC:" << ssrc
+                     << " which doesn't exist.";
+    return;
+  }
+  matching_stream->second->SetEncoderToPacketizerFrameTransformer(
+      std::move(frame_transformer));
+}
+
 bool WebRtcVoiceMediaChannel::MaybeDeregisterUnsignaledRecvStream(
     uint32_t ssrc) {
   RTC_DCHECK(worker_thread_checker_.IsCurrent());

media/engine/webrtc_voice_engine.h

@@ -209,6 +209,13 @@ class WebRtcVoiceMediaChannel final : public VoiceMediaChannel,
 
   std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const override;
 
+  // Sets a frame transformer between encoder and packetizer, to transform
+  // encoded frames before sending them out the network.
+  void SetEncoderToPacketizerFrameTransformer(
+      uint32_t ssrc,
+      rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)
+      override;
+
   // implements Transport interface
   bool SendRtp(const uint8_t* data,
                size_t len,