Transform encoded frames in ChannelSend.

This change is part of the implementation of the Insertable Streams Web
API: https://github.com/alvestrand/webrtc-media-streams/blob/master/explainer.md

Design doc for WebRTC library changes:
http://doc/1eiLkjNUkRy2FssCPLUp6eH08BZuXXoHfbbBP1ZN7EVk

Bug: webrtc:11380
Change-Id: I75444283ddb7f8db742687b497bf532c6dda47be
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/171871
Commit-Queue: Marina Ciocea <marinaciocea@webrtc.org>
Reviewed-by: Per Åhgren <peah@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30952}

audio/BUILD.gn

@@ -28,6 +28,8 @@ rtc_library("audio") {
     "channel_receive.h",
     "channel_send.cc",
     "channel_send.h",
+    "channel_send_frame_transformer_delegate.cc",
+    "channel_send_frame_transformer_delegate.h",
     "conversion.h",
     "null_audio_poller.cc",
     "null_audio_poller.h",

audio/channel_send.cc

@@ -21,6 +21,7 @@
 #include "api/call/transport.h"
 #include "api/crypto/frame_encryptor_interface.h"
 #include "api/rtc_event_log/rtc_event_log.h"
+#include "audio/channel_send_frame_transformer_delegate.h"
 #include "audio/utility/audio_frame_operations.h"
 #include "call/rtp_transport_controller_send_interface.h"
 #include "logging/rtc_event_log/events/rtc_event_audio_playout.h"
@@ -170,6 +171,9 @@ class ChannelSend : public ChannelSendInterface,
 
   void OnReceivedRtt(int64_t rtt_ms);
 
+  void InitFrameTransformerDelegate(
+      rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer);
+
   // Thread checkers document and lock usage of some methods on voe::Channel to
   // specific threads we know about. The goal is to eventually split up
   // voe::Channel into parts with single-threaded semantics, and thereby reduce
@@ -224,9 +228,11 @@ class ChannelSend : public ChannelSendInterface,
   // E2EE Frame Encryption Options
   const webrtc::CryptoOptions crypto_options_;
 
-  // Frame transformer used by insertable streams to transform encoded frames.
-  rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_
-      RTC_GUARDED_BY(encoder_queue_);
+  // Delegates calls to a frame transformer to transform audio, and
+  // receives callbacks with the transformed frames; delegates calls to
+  // ChannelSend::SendRtpAudio to send the transformed audio.
+  rtc::scoped_refptr<ChannelSendFrameTransformerDelegate>
+      frame_transformer_delegate_ RTC_GUARDED_BY(encoder_queue_);
 
   rtc::CriticalSection bitrate_crit_section_;
   int configured_bitrate_bps_ RTC_GUARDED_BY(bitrate_crit_section_) = 0;
@@ -379,6 +385,14 @@ int32_t ChannelSend::SendData(AudioFrameType frameType,
                               int64_t absolute_capture_timestamp_ms) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
   rtc::ArrayView<const uint8_t> payload(payloadData, payloadSize);
+  if (frame_transformer_delegate_) {
+    // Asynchronously transform the payload before sending it. After the payload
+    // is transformed, the delegate will call SendRtpAudio to send it.
+    frame_transformer_delegate_->Transform(
+        frameType, payloadType, rtp_timestamp, payloadData, payloadSize,
+        absolute_capture_timestamp_ms, _rtpRtcpModule->SSRC());
+    return 0;
+  }
   return SendRtpAudio(frameType, payloadType, rtp_timestamp, payload,
                       absolute_capture_timestamp_ms);
 }
@@ -491,7 +505,6 @@ ChannelSend::ChannelSend(
           new RateLimiter(clock, kMaxRetransmissionWindowMs)),
       frame_encryptor_(frame_encryptor),
       crypto_options_(crypto_options),
-      frame_transformer_(std::move(frame_transformer)),
       encoder_queue_(task_queue_factory->CreateTaskQueue(
           "AudioEncoder",
           TaskQueueFactory::Priority::NORMAL)) {
@@ -532,11 +545,17 @@ ChannelSend::ChannelSend(
 
   int error = audio_coding_->RegisterTransportCallback(this);
   RTC_DCHECK_EQ(0, error);
+  if (frame_transformer)
+    InitFrameTransformerDelegate(std::move(frame_transformer));
 }
 
 ChannelSend::~ChannelSend() {
   RTC_DCHECK(construction_thread_.IsCurrent());
 
+  // Resets the delegate's callback to ChannelSend::SendRtpAudio.
+  if (frame_transformer_delegate_)
+    frame_transformer_delegate_->Reset();
+
   StopSend();
   int error = audio_coding_->RegisterTransportCallback(NULL);
   RTC_DCHECK_EQ(0, error);
@@ -915,10 +934,13 @@ void ChannelSend::SetFrameEncryptor(
 void ChannelSend::SetEncoderToPacketizerFrameTransformer(
     rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
   RTC_DCHECK_RUN_ON(&worker_thread_checker_);
+  if (!frame_transformer)
+    return;
+
   encoder_queue_.PostTask(
       [this, frame_transformer = std::move(frame_transformer)]() mutable {
         RTC_DCHECK_RUN_ON(&encoder_queue_);
-        frame_transformer_ = std::move(frame_transformer);
+        InitFrameTransformerDelegate(std::move(frame_transformer));
       });
 }
 
@@ -928,6 +950,29 @@ void ChannelSend::OnReceivedRtt(int64_t rtt_ms) {
       [rtt_ms](AudioEncoder* encoder) { encoder->OnReceivedRtt(rtt_ms); });
 }
 
+void ChannelSend::InitFrameTransformerDelegate(
+    rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
+  RTC_DCHECK_RUN_ON(&encoder_queue_);
+  RTC_DCHECK(frame_transformer);
+  RTC_DCHECK(!frame_transformer_delegate_);
+
+  // Pass a callback to ChannelSend::SendRtpAudio, to be called by the delegate
+  // to send the transformed audio.
+  ChannelSendFrameTransformerDelegate::SendFrameCallback send_audio_callback =
+      [this](AudioFrameType frameType, uint8_t payloadType,
+             uint32_t rtp_timestamp, rtc::ArrayView<const uint8_t> payload,
+             int64_t absolute_capture_timestamp_ms) {
+        RTC_DCHECK_RUN_ON(&encoder_queue_);
+        return SendRtpAudio(frameType, payloadType, rtp_timestamp, payload,
+                            absolute_capture_timestamp_ms);
+      };
+  frame_transformer_delegate_ =
+      new rtc::RefCountedObject<ChannelSendFrameTransformerDelegate>(
+          std::move(send_audio_callback), std::move(frame_transformer),
+          &encoder_queue_);
+  frame_transformer_delegate_->Init();
+}
+
 }  // namespace
 
 std::unique_ptr<ChannelSendInterface> CreateChannelSend(

audio/channel_send_frame_transformer_delegate.cc

@@ -0,0 +1,116 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "audio/channel_send_frame_transformer_delegate.h"
+
+#include <utility>
+
+namespace webrtc {
+namespace {
+
+class TransformableAudioFrame : public TransformableFrameInterface {
+ public:
+  TransformableAudioFrame(AudioFrameType frame_type,
+                          uint8_t payload_type,
+                          uint32_t rtp_timestamp,
+                          const uint8_t* payload_data,
+                          size_t payload_size,
+                          int64_t absolute_capture_timestamp_ms,
+                          uint32_t ssrc)
+      : frame_type_(frame_type),
+        payload_type_(payload_type),
+        rtp_timestamp_(rtp_timestamp),
+        payload_(payload_data, payload_size),
+        absolute_capture_timestamp_ms_(absolute_capture_timestamp_ms),
+        ssrc_(ssrc) {}
+  ~TransformableAudioFrame() override = default;
+  rtc::ArrayView<const uint8_t> GetData() const override { return payload_; }
+  void SetData(rtc::ArrayView<const uint8_t> data) override {
+    payload_.SetData(data.data(), data.size());
+  }
+  uint32_t GetTimestamp() const override { return rtp_timestamp_; }
+  uint32_t GetSsrc() const override { return ssrc_; }
+
+  AudioFrameType GetFrameType() const { return frame_type_; }
+  uint8_t GetPayloadType() const { return payload_type_; }
+  int64_t GetAbsoluteCaptureTimestampMs() const {
+    return absolute_capture_timestamp_ms_;
+  }
+
+ private:
+  AudioFrameType frame_type_;
+  uint8_t payload_type_;
+  uint32_t rtp_timestamp_;
+  rtc::Buffer payload_;
+  int64_t absolute_capture_timestamp_ms_;
+  uint32_t ssrc_;
+};
+}  // namespace
+
+ChannelSendFrameTransformerDelegate::ChannelSendFrameTransformerDelegate(
+    SendFrameCallback send_frame_callback,
+    rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
+    rtc::TaskQueue* encoder_queue)
+    : send_frame_callback_(send_frame_callback),
+      frame_transformer_(std::move(frame_transformer)),
+      encoder_queue_(encoder_queue) {}
+
+void ChannelSendFrameTransformerDelegate::Init() {
+  frame_transformer_->RegisterTransformedFrameCallback(
+      rtc::scoped_refptr<TransformedFrameCallback>(this));
+}
+
+void ChannelSendFrameTransformerDelegate::Reset() {
+  frame_transformer_->UnregisterTransformedFrameCallback();
+  frame_transformer_ = nullptr;
+
+  rtc::CritScope lock(&send_lock_);
+  send_frame_callback_ = SendFrameCallback();
+}
+
+void ChannelSendFrameTransformerDelegate::Transform(
+    AudioFrameType frame_type,
+    uint8_t payload_type,
+    uint32_t rtp_timestamp,
+    const uint8_t* payload_data,
+    size_t payload_size,
+    int64_t absolute_capture_timestamp_ms,
+    uint32_t ssrc) {
+  frame_transformer_->Transform(std::make_unique<TransformableAudioFrame>(
+      frame_type, payload_type, rtp_timestamp, payload_data, payload_size,
+      absolute_capture_timestamp_ms, ssrc));
+}
+
+void ChannelSendFrameTransformerDelegate::OnTransformedFrame(
+    std::unique_ptr<TransformableFrameInterface> frame) {
+  rtc::CritScope lock(&send_lock_);
+  if (!send_frame_callback_)
+    return;
+  rtc::scoped_refptr<ChannelSendFrameTransformerDelegate> delegate = this;
+  encoder_queue_->PostTask(
+      [delegate = std::move(delegate), frame = std::move(frame)]() mutable {
+        delegate->SendFrame(std::move(frame));
+      });
+}
+
+void ChannelSendFrameTransformerDelegate::SendFrame(
+    std::unique_ptr<TransformableFrameInterface> frame) const {
+  rtc::CritScope lock(&send_lock_);
+  RTC_DCHECK_RUN_ON(encoder_queue_);
+  if (!send_frame_callback_)
+    return;
+  auto* transformed_frame = static_cast<TransformableAudioFrame*>(frame.get());
+  send_frame_callback_(
+      transformed_frame->GetFrameType(), transformed_frame->GetPayloadType(),
+      transformed_frame->GetTimestamp(), transformed_frame->GetData(),
+      transformed_frame->GetAbsoluteCaptureTimestampMs());
+}
+
+}  // namespace webrtc

audio/channel_send_frame_transformer_delegate.h

@@ -0,0 +1,80 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef AUDIO_CHANNEL_SEND_FRAME_TRANSFORMER_DELEGATE_H_
+#define AUDIO_CHANNEL_SEND_FRAME_TRANSFORMER_DELEGATE_H_
+
+#include <memory>
+
+#include "api/frame_transformer_interface.h"
+#include "modules/audio_coding/include/audio_coding_module_typedefs.h"
+#include "rtc_base/buffer.h"
+#include "rtc_base/critical_section.h"
+#include "rtc_base/synchronization/sequence_checker.h"
+#include "rtc_base/task_queue.h"
+
+namespace webrtc {
+
+// Delegates calls to FrameTransformerInterface to transform frames, and to
+// ChannelSend to send the transformed frames using |send_frame_callback_| on
+// the |encoder_queue_|.
+// OnTransformedFrame() can be called from any thread, the delegate ensures
+// thread-safe access to the ChannelSend callback.
+class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback {
+ public:
+  using SendFrameCallback =
+      std::function<int32_t(AudioFrameType frameType,
+                            uint8_t payloadType,
+                            uint32_t rtp_timestamp,
+                            rtc::ArrayView<const uint8_t> payload,
+                            int64_t absolute_capture_timestamp_ms)>;
+  ChannelSendFrameTransformerDelegate(
+      SendFrameCallback send_frame_callback,
+      rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
+      rtc::TaskQueue* encoder_queue);
+
+  // Registers |this| as callback for |frame_transformer_|, to get the
+  // transformed frames.
+  void Init();
+
+  // Unregisters and releases the |frame_transformer_| reference, and resets
+  // |send_frame_callback_| under lock. Called from ChannelSend destructor to
+  // prevent running the callback on a dangling channel.
+  void Reset();
+
+  // Delegates the call to FrameTransformerInterface::TransformFrame, to
+  // transform the frame asynchronously.
+  void Transform(AudioFrameType frame_type,
+                 uint8_t payload_type,
+                 uint32_t rtp_timestamp,
+                 const uint8_t* payload_data,
+                 size_t payload_size,
+                 int64_t absolute_capture_timestamp_ms,
+                 uint32_t ssrc);
+
+  // Implements TransformedFrameCallback. Can be called on any thread.
+  void OnTransformedFrame(
+      std::unique_ptr<TransformableFrameInterface> frame) override;
+
+  // Delegates the call to ChannelSend::SendRtpAudio on the |encoder_queue_|,
+  // by calling |send_audio_callback_|.
+  void SendFrame(std::unique_ptr<TransformableFrameInterface> frame) const;
+
+ protected:
+  ~ChannelSendFrameTransformerDelegate() override = default;
+
+ private:
+  rtc::CriticalSection send_lock_;
+  SendFrameCallback send_frame_callback_ RTC_GUARDED_BY(send_lock_);
+  rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_;
+  rtc::TaskQueue* encoder_queue_ RTC_GUARDED_BY(send_lock_);
+};
+}  // namespace webrtc
+#endif  // AUDIO_CHANNEL_SEND_FRAME_TRANSFORMER_DELEGATE_H_