Transform received audio frames in ChannelReceive.

This change is part of the implementation of the Insertable Streams Web
API: https://github.com/alvestrand/webrtc-media-streams/blob/master/explainer.md

Design doc for WebRTC library changes:
http://doc/1eiLkjNUkRy2FssCPLUp6eH08BZuXXoHfbbBP1ZN7EVk

Bug: webrtc:11380
No-Try: True
Change-Id: I1a7ef9fd8130936176b5a4f78ad835cba52666d0
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/171873
Commit-Queue: Marina Ciocea <marinaciocea@webrtc.org>
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Reviewed-by: Per Åhgren <peah@webrtc.org>
Reviewed-by: Tommi <tommi@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30961}

api/frame_transformer_interface.h

@@ -50,6 +50,17 @@ class TransformableVideoFrameInterface : public TransformableFrameInterface {
   virtual std::vector<uint8_t> GetAdditionalData() const = 0;
 };
 
+// Extends the TransformableFrameInterface to expose audio-specific information.
+class TransformableAudioFrameInterface : public TransformableFrameInterface {
+ public:
+  virtual ~TransformableAudioFrameInterface() = default;
+
+  // Exposes the frame header, enabling the interface clients to use the
+  // information in the header as needed, for example to compile the list of
+  // csrcs.
+  virtual const RTPHeader& GetHeader() const = 0;
+};
+
 // Objects implement this interface to be notified with the transformed frame.
 class TransformedFrameCallback : public rtc::RefCountInterface {
  public:

audio/BUILD.gn

@@ -26,6 +26,8 @@ rtc_library("audio") {
     "audio_transport_impl.h",
     "channel_receive.cc",
     "channel_receive.h",
+    "channel_receive_frame_transformer_delegate.cc",
+    "channel_receive_frame_transformer_delegate.h",
     "channel_send.cc",
     "channel_send.h",
     "channel_send_frame_transformer_delegate.cc",
@@ -88,6 +90,7 @@ rtc_library("audio") {
     "../rtc_base:safe_minmax",
     "../rtc_base/experiments:field_trial_parser",
     "../rtc_base/synchronization:sequence_checker",
+    "../rtc_base/task_utils:to_queued_task",
     "../system_wrappers",
     "../system_wrappers:field_trial",
     "../system_wrappers:metrics",

audio/channel_receive.cc

@@ -23,6 +23,7 @@
 #include "api/frame_transformer_interface.h"
 #include "api/rtc_event_log/rtc_event_log.h"
 #include "audio/audio_level.h"
+#include "audio/channel_receive_frame_transformer_delegate.h"
 #include "audio/channel_send.h"
 #include "audio/utility/audio_frame_operations.h"
 #include "logging/rtc_event_log/events/rtc_event_audio_playout.h"
@@ -183,6 +184,9 @@ class ChannelReceive : public ChannelReceiveInterface {
   void OnReceivedPayloadData(rtc::ArrayView<const uint8_t> payload,
                              const RTPHeader& rtpHeader);
 
+  void InitFrameTransformerDelegate(
+      rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer);
+
   bool Playing() const {
     rtc::CritScope lock(&playing_lock_);
     return playing_;
@@ -272,7 +276,8 @@ class ChannelReceive : public ChannelReceiveInterface {
 
   webrtc::AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_;
 
-  rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_;
+  rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate>
+      frame_transformer_delegate_;
 };
 
 void ChannelReceive::OnReceivedPayloadData(
@@ -302,6 +307,25 @@ void ChannelReceive::OnReceivedPayloadData(
   }
 }
 
+void ChannelReceive::InitFrameTransformerDelegate(
+    rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
+  RTC_DCHECK(frame_transformer);
+  RTC_DCHECK(!frame_transformer_delegate_);
+
+  // Pass a callback to ChannelReceive::ReceivePacket, to be called by the
+  // delegate to receive transformed audio.
+  ChannelReceiveFrameTransformerDelegate::ReceiveFrameCallback
+      receive_audio_callback = [this](rtc::ArrayView<const uint8_t> packet,
+                                      const RTPHeader& header) {
+        ReceivePacket(packet.data(), packet.size(), header);
+      };
+  frame_transformer_delegate_ =
+      new rtc::RefCountedObject<ChannelReceiveFrameTransformerDelegate>(
+          std::move(receive_audio_callback), std::move(frame_transformer),
+          rtc::Thread::Current());
+  frame_transformer_delegate_->Init();
+}
+
 AudioMixer::Source::AudioFrameInfo ChannelReceive::GetAudioFrameWithInfo(
     int sample_rate_hz,
     AudioFrame* audio_frame) {
@@ -456,8 +480,7 @@ ChannelReceive::ChannelReceive(
       associated_send_channel_(nullptr),
       frame_decryptor_(frame_decryptor),
       crypto_options_(crypto_options),
-      absolute_capture_time_receiver_(clock),
+      absolute_capture_time_receiver_(clock) {
-      frame_transformer_(std::move(frame_transformer)) {
   // TODO(nisse): Use _moduleProcessThreadPtr instead?
   module_process_thread_checker_.Detach();
 
@@ -481,6 +504,9 @@ ChannelReceive::ChannelReceive(
   configuration.event_log = event_log_;
   configuration.local_media_ssrc = local_ssrc;
 
+  if (frame_transformer)
+    InitFrameTransformerDelegate(std::move(frame_transformer));
+
   _rtpRtcpModule = RtpRtcp::Create(configuration);
   _rtpRtcpModule->SetSendingMediaStatus(false);
   _rtpRtcpModule->SetRemoteSSRC(remote_ssrc_);
@@ -569,7 +595,13 @@ void ChannelReceive::OnRtpPacket(const RtpPacketReceived& packet) {
           rtc::saturated_cast<uint32_t>(packet_copy.payload_type_frequency()),
           header.extension.absolute_capture_time);
 
+  if (frame_transformer_delegate_) {
+    // Asynchronously transform the received payload. After the payload is
+    // transformed, the delegate will call ReceivePacket to handle it.
+    frame_transformer_delegate_->Transform(packet_copy, header, remote_ssrc_);
+  } else {
     ReceivePacket(packet_copy.data(), packet_copy.size(), header);
+  }
 }
 
 void ChannelReceive::ReceivePacket(const uint8_t* packet,
@@ -758,7 +790,11 @@ void ChannelReceive::SetAssociatedSendChannel(
 void ChannelReceive::SetDepacketizerToDecoderFrameTransformer(
     rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
   RTC_DCHECK(worker_thread_checker_.IsCurrent());
-  frame_transformer_ = std::move(frame_transformer);
+  // Depending on when the channel is created, the transformer might be set
+  // twice. Don't replace the delegate if it was already initialized.
+  if (!frame_transformer || frame_transformer_delegate_)
+    return;
+  InitFrameTransformerDelegate(std::move(frame_transformer));
 }
 
 NetworkStatistics ChannelReceive::GetNetworkStatistics() const {

audio/channel_receive_frame_transformer_delegate.cc

@@ -0,0 +1,95 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "audio/channel_receive_frame_transformer_delegate.h"
+
+#include <utility>
+
+#include "rtc_base/buffer.h"
+#include "rtc_base/task_utils/to_queued_task.h"
+
+namespace webrtc {
+namespace {
+
+class TransformableAudioFrame : public TransformableAudioFrameInterface {
+ public:
+  TransformableAudioFrame(rtc::ArrayView<const uint8_t> payload,
+                          const RTPHeader& header,
+                          uint32_t ssrc)
+      : payload_(payload.data(), payload.size()),
+        header_(header),
+        ssrc_(ssrc) {}
+  ~TransformableAudioFrame() override = default;
+  rtc::ArrayView<const uint8_t> GetData() const override { return payload_; }
+
+  void SetData(rtc::ArrayView<const uint8_t> data) override {
+    payload_.SetData(data.data(), data.size());
+  }
+
+  uint32_t GetTimestamp() const override { return header_.timestamp; }
+  uint32_t GetSsrc() const override { return ssrc_; }
+  const RTPHeader& GetHeader() const override { return header_; }
+
+ private:
+  rtc::Buffer payload_;
+  RTPHeader header_;
+  uint32_t ssrc_;
+};
+}  // namespace
+
+ChannelReceiveFrameTransformerDelegate::ChannelReceiveFrameTransformerDelegate(
+    ReceiveFrameCallback receive_frame_callback,
+    rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
+    rtc::Thread* channel_receive_thread)
+    : receive_frame_callback_(receive_frame_callback),
+      frame_transformer_(std::move(frame_transformer)),
+      channel_receive_thread_(channel_receive_thread) {}
+
+void ChannelReceiveFrameTransformerDelegate::Init() {
+  RTC_DCHECK_RUN_ON(&sequence_checker_);
+  frame_transformer_->RegisterTransformedFrameCallback(
+      rtc::scoped_refptr<TransformedFrameCallback>(this));
+}
+
+void ChannelReceiveFrameTransformerDelegate::Reset() {
+  RTC_DCHECK_RUN_ON(&sequence_checker_);
+  frame_transformer_->UnregisterTransformedFrameCallback();
+  frame_transformer_ = nullptr;
+  receive_frame_callback_ = ReceiveFrameCallback();
+}
+
+void ChannelReceiveFrameTransformerDelegate::Transform(
+    rtc::ArrayView<const uint8_t> packet,
+    const RTPHeader& header,
+    uint32_t ssrc) {
+  RTC_DCHECK_RUN_ON(&sequence_checker_);
+  frame_transformer_->Transform(
+      std::make_unique<TransformableAudioFrame>(packet, header, ssrc));
+}
+
+void ChannelReceiveFrameTransformerDelegate::OnTransformedFrame(
+    std::unique_ptr<TransformableFrameInterface> frame) {
+  rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate> delegate = this;
+  channel_receive_thread_->PostTask(ToQueuedTask(
+      [delegate = std::move(delegate), frame = std::move(frame)]() mutable {
+        delegate->ReceiveFrame(std::move(frame));
+      }));
+}
+
+void ChannelReceiveFrameTransformerDelegate::ReceiveFrame(
+    std::unique_ptr<TransformableFrameInterface> frame) const {
+  RTC_DCHECK_RUN_ON(&sequence_checker_);
+  if (!receive_frame_callback_)
+    return;
+  auto* transformed_frame = static_cast<TransformableAudioFrame*>(frame.get());
+  receive_frame_callback_(transformed_frame->GetData(),
+                          transformed_frame->GetHeader());
+}
+}  // namespace webrtc

audio/channel_receive_frame_transformer_delegate.h

@@ -0,0 +1,73 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef AUDIO_CHANNEL_RECEIVE_FRAME_TRANSFORMER_DELEGATE_H_
+#define AUDIO_CHANNEL_RECEIVE_FRAME_TRANSFORMER_DELEGATE_H_
+
+#include <memory>
+
+#include "api/frame_transformer_interface.h"
+#include "rtc_base/synchronization/sequence_checker.h"
+#include "rtc_base/task_queue.h"
+#include "rtc_base/thread.h"
+
+namespace webrtc {
+
+// Delegates calls to FrameTransformerInterface to transform frames, and to
+// ChannelReceive to receive the transformed frames using the
+// |receive_frame_callback_| on the |channel_receive_thread_|.
+class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback {
+ public:
+  using ReceiveFrameCallback =
+      std::function<void(rtc::ArrayView<const uint8_t> packet,
+                         const RTPHeader& header)>;
+  ChannelReceiveFrameTransformerDelegate(
+      ReceiveFrameCallback receive_frame_callback,
+      rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
+      rtc::Thread* channel_receive_thread);
+
+  // Registers |this| as callback for |frame_transformer_|, to get the
+  // transformed frames.
+  void Init();
+
+  // Unregisters and releases the |frame_transformer_| reference, and resets
+  // |receive_frame_callback_| on |channel_receive_thread_|. Called from
+  // ChannelReceive destructor to prevent running the callback on a dangling
+  // channel.
+  void Reset();
+
+  // Delegates the call to FrameTransformerInterface::Transform, to transform
+  // the frame asynchronously.
+  void Transform(rtc::ArrayView<const uint8_t> packet,
+                 const RTPHeader& header,
+                 uint32_t ssrc);
+
+  // Implements TransformedFrameCallback. Can be called on any thread.
+  void OnTransformedFrame(
+      std::unique_ptr<TransformableFrameInterface> frame) override;
+
+  // Delegates the call to ChannelReceive::ReceivePacket on the
+  // |channel_receive_thread_|, by calling |receive_frame_callback_|.
+  void ReceiveFrame(std::unique_ptr<TransformableFrameInterface> frame) const;
+
+ protected:
+  ~ChannelReceiveFrameTransformerDelegate() override = default;
+
+ private:
+  SequenceChecker sequence_checker_;
+  ReceiveFrameCallback receive_frame_callback_
+      RTC_GUARDED_BY(sequence_checker_);
+  rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_
+      RTC_GUARDED_BY(sequence_checker_);
+  rtc::Thread* channel_receive_thread_;
+};
+
+}  // namespace webrtc
+#endif  // AUDIO_CHANNEL_RECEIVE_FRAME_TRANSFORMER_DELEGATE_H_