Async audio processing API

API to injecting a heavy audio processing operation into WebRTC audio capture pipeline

Bug: webrtc:12003
Change-Id: I9f6f58f468bd84efd0a9d53d703db6229a03959e
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/165788
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Commit-Queue: Olga Sharonova <olka@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32291}

modules/async_audio_processing/BUILD.gn

@@ -0,0 +1,44 @@
+# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import("../../webrtc.gni")
+
+rtc_library("async_audio_processing") {
+  sources = [
+    "async_audio_processing.cc",
+    "async_audio_processing.h",
+  ]
+
+  public = [ "async_audio_processing.h" ]
+
+  deps = [
+    "../../api:scoped_refptr",
+    "../../api/audio:audio_frame_api",
+    "../../api/audio:audio_frame_processor",
+    "../../api/task_queue:task_queue",
+    "../../rtc_base:checks",
+    "../../rtc_base:rtc_base_approved",
+    "../../rtc_base:rtc_task_queue",
+    "../../rtc_base/synchronization:sequence_checker",
+  ]
+}
+
+if (rtc_include_tests) {
+  rtc_library("async_audio_processing_test") {
+    testonly = true
+
+    sources = []
+
+    deps = [
+      ":async_audio_processing",
+      "../../api/audio:audio_frame_api",
+      "../../rtc_base:checks",
+      "../../rtc_base:rtc_base_approved",
+    ]
+  }
+}

modules/async_audio_processing/async_audio_processing.cc

@@ -0,0 +1,61 @@
+
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/async_audio_processing/async_audio_processing.h"
+
+#include <utility>
+
+#include "api/audio/audio_frame.h"
+#include "api/task_queue/task_queue_factory.h"
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+
+AsyncAudioProcessing::Factory::~Factory() = default;
+AsyncAudioProcessing::Factory::Factory(AudioFrameProcessor& frame_processor,
+                                       TaskQueueFactory& task_queue_factory)
+    : frame_processor_(frame_processor),
+      task_queue_factory_(task_queue_factory) {}
+
+std::unique_ptr<AsyncAudioProcessing>
+AsyncAudioProcessing::Factory::CreateAsyncAudioProcessing(
+    AudioFrameProcessor::OnAudioFrameCallback on_frame_processed_callback) {
+  return std::make_unique<AsyncAudioProcessing>(
+      frame_processor_, task_queue_factory_,
+      std::move(on_frame_processed_callback));
+}
+
+AsyncAudioProcessing::~AsyncAudioProcessing() {
+  frame_processor_.SetSink(nullptr);
+}
+
+AsyncAudioProcessing::AsyncAudioProcessing(
+    AudioFrameProcessor& frame_processor,
+    TaskQueueFactory& task_queue_factory,
+    AudioFrameProcessor::OnAudioFrameCallback on_frame_processed_callback)
+    : on_frame_processed_callback_(std::move(on_frame_processed_callback)),
+      frame_processor_(frame_processor),
+      task_queue_(task_queue_factory.CreateTaskQueue(
+          "AsyncAudioProcessing",
+          TaskQueueFactory::Priority::NORMAL)) {
+  frame_processor_.SetSink([this](std::unique_ptr<AudioFrame> frame) {
+    task_queue_.PostTask([this, frame = std::move(frame)]() mutable {
+      on_frame_processed_callback_(std::move(frame));
+    });
+  });
+}
+
+void AsyncAudioProcessing::Process(std::unique_ptr<AudioFrame> frame) {
+  task_queue_.PostTask([this, frame = std::move(frame)]() mutable {
+    frame_processor_.Process(std::move(frame));
+  });
+}
+
+}  // namespace webrtc

modules/async_audio_processing/async_audio_processing.h

@@ -0,0 +1,76 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_ASYNC_AUDIO_PROCESSING_ASYNC_AUDIO_PROCESSING_H_
+#define MODULES_ASYNC_AUDIO_PROCESSING_ASYNC_AUDIO_PROCESSING_H_
+
+#include <memory>
+
+#include "api/audio/audio_frame_processor.h"
+#include "rtc_base/ref_count.h"
+#include "rtc_base/task_queue.h"
+
+namespace webrtc {
+
+class AudioFrame;
+class TaskQueueFactory;
+
+// Helper class taking care of interactions with AudioFrameProcessor
+// in asynchronous manner. Offloads AudioFrameProcessor::Process calls
+// to a dedicated task queue. Makes sure that it's always safe for
+// AudioFrameProcessor to pass processed frames back to its sink.
+class AsyncAudioProcessing final {
+ public:
+  // Helper class passing AudioFrameProcessor and TaskQueueFactory into
+  // AsyncAudioProcessing constructor.
+  class Factory : public rtc::RefCountInterface {
+   public:
+    Factory(const Factory&) = delete;
+    Factory& operator=(const Factory&) = delete;
+
+    ~Factory();
+    Factory(AudioFrameProcessor& frame_processor,
+            TaskQueueFactory& task_queue_factory);
+
+    std::unique_ptr<AsyncAudioProcessing> CreateAsyncAudioProcessing(
+        AudioFrameProcessor::OnAudioFrameCallback on_frame_processed_callback);
+
+   private:
+    AudioFrameProcessor& frame_processor_;
+    TaskQueueFactory& task_queue_factory_;
+  };
+
+  AsyncAudioProcessing(const AsyncAudioProcessing&) = delete;
+  AsyncAudioProcessing& operator=(const AsyncAudioProcessing&) = delete;
+
+  ~AsyncAudioProcessing();
+
+  // Creates AsyncAudioProcessing which will pass audio frames to
+  // |frame_processor| on |task_queue_| and reply with processed frames passed
+  // into |on_frame_processed_callback|, which is posted back onto
+  // |task_queue_|. |task_queue_| is created using the provided
+  // |task_queue_factory|.
+  AsyncAudioProcessing(
+      AudioFrameProcessor& frame_processor,
+      TaskQueueFactory& task_queue_factory,
+      AudioFrameProcessor::OnAudioFrameCallback on_frame_processed_callback);
+
+  // Accepts |frame| for asynchronous processing. Thread-safe.
+  void Process(std::unique_ptr<AudioFrame> frame);
+
+ private:
+  AudioFrameProcessor::OnAudioFrameCallback on_frame_processed_callback_;
+  AudioFrameProcessor& frame_processor_;
+  rtc::TaskQueue task_queue_;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_ASYNC_AUDIO_PROCESSING_ASYNC_AUDIO_PROCESSING_H_