Store RuntimeSetting in Aec Dumps.

Also read and apply settings when parsing and replaying dumps.

The implementation contains
* an extra field in debug.proto for the runtime settings
* code in AudioProcessingImpl to initiate the logging of the RS to the
  AecDump
* code in aec_dump/ to log the RS in the AecDump
* code in test/ for re-playing the RS. E.g. for APM simulation with
  audioproc_f.

Bug: webrtc:9138
Change-Id: Ia2a00537c2eb19484ff442fbffd0b95f8495516f
Reviewed-on: https://webrtc-review.googlesource.com/70502
Commit-Queue: Alex Loiko <aleloi@webrtc.org>
Reviewed-by: Alessio Bazzica <alessiob@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24647}

modules/audio_processing/BUILD.gn

@@ -419,6 +419,7 @@ if (rtc_include_tests) {
         ":audioproc_protobuf_utils",
         ":audioproc_test_utils",
         ":audioproc_unittest_proto",
+        ":runtime_settings_protobuf_utils",
         "../../api/audio:audio_frame_api",
         "../../rtc_base:rtc_base_tests_utils",
         "../../rtc_base:rtc_task_queue",
@@ -531,6 +532,7 @@ if (rtc_include_tests) {
         ":audioproc_debug_proto",
         ":audioproc_protobuf_utils",
         ":audioproc_test_utils",
+        ":runtime_settings_protobuf_utils",
         "../../api/audio:aec3_factory",
         "../../common_audio:common_audio",
         "../../rtc_base:checks",
@@ -651,10 +653,26 @@ if (rtc_include_tests) {
       deps = [
         ":audioproc_debug_proto",
         "../..:webrtc_common",
+        "../../rtc_base:checks",
         "../../rtc_base:protobuf_utils",
         "../../rtc_base:rtc_base_approved",
         "../../rtc_base/system:arch",
       ]
     }
+
+    rtc_static_library("runtime_settings_protobuf_utils") {
+      testonly = true
+      sources = [
+        "test/runtime_setting_util.cc",
+        "test/runtime_setting_util.h",
+      ]
+
+      deps = [
+        ":audio_processing",
+        ":audioproc_debug_proto",
+        ":audioproc_protobuf_utils",
+        "../../rtc_base:checks",
+      ]
+    }
   }
 }

modules/audio_processing/aec_dump/aec_dump_impl.cc

@@ -167,6 +167,34 @@ void AecDumpImpl::WriteConfig(const InternalAPMConfig& config) {
   worker_queue_->PostTask(std::unique_ptr<rtc::QueuedTask>(std::move(task)));
 }
 
+void AecDumpImpl::WriteRuntimeSetting(
+    const AudioProcessing::RuntimeSetting& runtime_setting) {
+  RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
+  auto task = CreateWriteToFileTask();
+  auto* event = task->GetEvent();
+  event->set_type(audioproc::Event::RUNTIME_SETTING);
+  audioproc::RuntimeSetting* setting = event->mutable_runtime_setting();
+  switch (runtime_setting.type()) {
+    case AudioProcessing::RuntimeSetting::Type::kCapturePreGain: {
+      float x;
+      runtime_setting.GetFloat(&x);
+      setting->set_capture_pre_gain(x);
+      break;
+    }
+    case AudioProcessing::RuntimeSetting::Type::
+        kCustomRenderProcessingRuntimeSetting: {
+      float x;
+      runtime_setting.GetFloat(&x);
+      setting->set_custom_render_processing_setting(x);
+      break;
+    }
+    case AudioProcessing::RuntimeSetting::Type::kNotSpecified:
+      RTC_NOTREACHED();
+      break;
+  }
+  worker_queue_->PostTask(std::unique_ptr<rtc::QueuedTask>(std::move(task)));
+}
+
 std::unique_ptr<WriteToFileTask> AecDumpImpl::CreateWriteToFileTask() {
   return absl::make_unique<WriteToFileTask>(debug_file_.get(),
                                             &num_bytes_left_for_log_);

modules/audio_processing/aec_dump/aec_dump_impl.h

@@ -67,6 +67,9 @@ class AecDumpImpl : public AecDump {
 
   void WriteConfig(const InternalAPMConfig& config) override;
 
+  void WriteRuntimeSetting(
+      const AudioProcessing::RuntimeSetting& runtime_setting) override;
+
  private:
   std::unique_ptr<WriteToFileTask> CreateWriteToFileTask();
 

modules/audio_processing/aec_dump/mock_aec_dump.h

@@ -43,6 +43,9 @@ class MockAecDump : public AecDump {
                void(const AudioFrameView<const float>& src));
 
   MOCK_METHOD1(WriteConfig, void(const InternalAPMConfig& config));
+
+  MOCK_METHOD1(WriteRuntimeSetting,
+               void(const AudioProcessing::RuntimeSetting& config));
 };
 
 }  // namespace test

modules/audio_processing/audio_processing_impl.cc

@@ -876,6 +876,9 @@ int AudioProcessingImpl::ProcessStream(const float* const* src,
 void AudioProcessingImpl::HandleCaptureRuntimeSettings() {
   RuntimeSetting setting;
   while (capture_runtime_settings_.Remove(&setting)) {
+    if (aec_dump_) {
+      aec_dump_->WriteRuntimeSetting(setting);
+    }
     switch (setting.type()) {
       case RuntimeSetting::Type::kCapturePreGain:
         if (config_.pre_amplifier.enabled) {
@@ -898,6 +901,9 @@ void AudioProcessingImpl::HandleCaptureRuntimeSettings() {
 void AudioProcessingImpl::HandleRenderRuntimeSettings() {
   RuntimeSetting setting;
   while (render_runtime_settings_.Remove(&setting)) {
+    if (aec_dump_) {
+      aec_dump_->WriteRuntimeSetting(setting);
+    }
     switch (setting.type()) {
       case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
         if (private_submodules_->render_pre_processor) {

modules/audio_processing/debug.proto

@@ -80,6 +80,11 @@ message Config {
   // Next field number 21.
 }
 
+message RuntimeSetting {
+  optional float capture_pre_gain = 1;
+  optional float custom_render_processing_setting = 2;
+}
+
 message Event {
   enum Type {
     INIT = 0;
@@ -87,6 +92,7 @@ message Event {
     STREAM = 2;
     CONFIG = 3;
     UNKNOWN_EVENT = 4;
+    RUNTIME_SETTING = 5;
   }
 
   required Type type = 1;
@@ -95,4 +101,5 @@ message Event {
   optional ReverseStream reverse_stream = 3;
   optional Stream stream = 4;
   optional Config config = 5;
+  optional RuntimeSetting runtime_setting = 6;
 }

modules/audio_processing/include/aec_dump.h

@@ -98,6 +98,9 @@ class AecDump {
   virtual void WriteRenderStreamMessage(
       const AudioFrameView<const float>& src) = 0;
 
+  virtual void WriteRuntimeSetting(
+      const AudioProcessing::RuntimeSetting& runtime_setting) = 0;
+
   // Logs Event::Type CONFIG message.
   virtual void WriteConfig(const InternalAPMConfig& config) = 0;
 };

modules/audio_processing/test/aec_dump_based_simulator.cc

@@ -11,6 +11,7 @@
 #include <iostream>
 
 #include "modules/audio_processing/test/aec_dump_based_simulator.h"
+#include "modules/audio_processing/test/runtime_setting_util.h"
 
 #include "modules/audio_processing/test/protobuf_utils.h"
 #include "rtc_base/checks.h"
@@ -253,8 +254,12 @@ void AecDumpBasedSimulator::Process() {
         RTC_CHECK(event_msg.has_config());
         HandleMessage(event_msg.config());
         break;
-      default:
+      case webrtc::audioproc::Event::RUNTIME_SETTING:
+        HandleMessage(event_msg.runtime_setting());
+        break;
+      case webrtc::audioproc::Event::UNKNOWN_EVENT:
         RTC_CHECK(false);
+        break;
     }
   }
 
@@ -552,5 +557,11 @@ void AecDumpBasedSimulator::HandleMessage(
   ProcessReverseStream(interface_used_ == InterfaceType::kFixedInterface);
 }
 
+void AecDumpBasedSimulator::HandleMessage(
+    const webrtc::audioproc::RuntimeSetting& msg) {
+  RTC_CHECK(ap_.get());
+  ReplayRuntimeSetting(ap_.get(), msg);
+}
+
 }  // namespace test
 }  // namespace webrtc

modules/audio_processing/test/aec_dump_based_simulator.h

@@ -42,6 +42,7 @@ class AecDumpBasedSimulator final : public AudioProcessingSimulator {
   void HandleMessage(const webrtc::audioproc::Stream& msg);
   void HandleMessage(const webrtc::audioproc::ReverseStream& msg);
   void HandleMessage(const webrtc::audioproc::Config& msg);
+  void HandleMessage(const webrtc::audioproc::RuntimeSetting& msg);
   void PrepareProcessStreamCall(const webrtc::audioproc::Stream& msg);
   void PrepareReverseProcessStreamCall(
       const webrtc::audioproc::ReverseStream& msg);

modules/audio_processing/test/debug_dump_replayer.cc

@@ -11,6 +11,7 @@
 #include "modules/audio_processing/test/debug_dump_replayer.h"
 
 #include "modules/audio_processing/test/protobuf_utils.h"
+#include "modules/audio_processing/test/runtime_setting_util.h"
 #include "rtc_base/checks.h"
 
 namespace webrtc {
@@ -73,8 +74,12 @@ bool DebugDumpReplayer::RunNextEvent() {
     case audioproc::Event::CONFIG:
       OnConfigEvent(next_event_.config());
       break;
+    case audioproc::Event::RUNTIME_SETTING:
+      OnRuntimeSettingEvent(next_event_.runtime_setting());
+      break;
     case audioproc::Event::UNKNOWN_EVENT:
       // We do not expect to receive UNKNOWN event.
+      RTC_CHECK(false);
       return false;
   }
   LoadNextMessage();
@@ -167,6 +172,12 @@ void DebugDumpReplayer::OnConfigEvent(const audioproc::Config& msg) {
   ConfigureApm(msg);
 }
 
+void DebugDumpReplayer::OnRuntimeSettingEvent(
+    const audioproc::RuntimeSetting& msg) {
+  RTC_CHECK(apm_.get());
+  ReplayRuntimeSetting(apm_.get(), msg);
+}
+
 void DebugDumpReplayer::MaybeRecreateApm(const audioproc::Config& msg) {
   // These configurations cannot be changed on the fly.
   Config config;
@@ -252,6 +263,11 @@ void DebugDumpReplayer::ConfigureApm(const audioproc::Config& msg) {
                apm_->noise_suppression()->set_level(
                    static_cast<NoiseSuppression::Level>(msg.ns_level())));
 
+  RTC_CHECK(msg.has_pre_amplifier_enabled());
+  apm_config.pre_amplifier.enabled = msg.pre_amplifier_enabled();
+  apm_config.pre_amplifier.fixed_gain_factor =
+      msg.pre_amplifier_fixed_gain_factor();
+
   apm_->ApplyConfig(apm_config);
 }
 

modules/audio_processing/test/debug_dump_replayer.h

@@ -48,6 +48,7 @@ class DebugDumpReplayer {
   void OnStreamEvent(const audioproc::Stream& msg);
   void OnReverseStreamEvent(const audioproc::ReverseStream& msg);
   void OnConfigEvent(const audioproc::Config& msg);
+  void OnRuntimeSettingEvent(const audioproc::RuntimeSetting& msg);
 
   void MaybeRecreateApm(const audioproc::Config& msg);
   void ConfigureApm(const audioproc::Config& msg);

modules/audio_processing/test/debug_dump_test.cc

@@ -48,7 +48,8 @@ class DebugDumpGenerator {
                      int reverse_channels,
                      const Config& config,
                      const std::string& dump_file_name,
-                     bool enable_aec3);
+                     bool enable_aec3,
+                     bool enable_pre_amplifier);
 
   // Constructor that uses default input files.
   explicit DebugDumpGenerator(const Config& config,
@@ -112,6 +113,8 @@ class DebugDumpGenerator {
   std::unique_ptr<ChannelBuffer<float>> reverse_;
   std::unique_ptr<ChannelBuffer<float>> output_;
 
+  bool enable_pre_amplifier_;
+
   rtc::TaskQueue worker_queue_;
   std::unique_ptr<AudioProcessing> apm_;
 
@@ -126,7 +129,8 @@ DebugDumpGenerator::DebugDumpGenerator(const std::string& input_file_name,
                                        int reverse_channels,
                                        const Config& config,
                                        const std::string& dump_file_name,
-                                       bool enable_aec3)
+                                       bool enable_aec3,
+                                       bool enable_pre_amplifier)
     : input_config_(input_rate_hz, input_channels),
       reverse_config_(reverse_rate_hz, reverse_channels),
       output_config_(input_rate_hz, input_channels),
@@ -140,6 +144,7 @@ DebugDumpGenerator::DebugDumpGenerator(const std::string& input_file_name,
                                         reverse_config_.num_channels())),
       output_(new ChannelBuffer<float>(output_config_.num_frames(),
                                        output_config_.num_channels())),
+      enable_pre_amplifier_(enable_pre_amplifier),
       worker_queue_("debug_dump_generator_worker_queue"),
       dump_file_name_(dump_file_name) {
   AudioProcessingBuilder apm_builder;
@@ -162,7 +167,8 @@ DebugDumpGenerator::DebugDumpGenerator(
                          2,
                          config,
                          TempFilename(OutputPath(), "debug_aec"),
-                         enable_aec3) {
+                         enable_aec3,
+                         apm_config.pre_amplifier.enabled) {
   apm_->ApplyConfig(apm_config);
 }
 
@@ -223,6 +229,10 @@ void DebugDumpGenerator::Process(size_t num_blocks) {
     ReadAndDeinterleave(&input_audio_, input_file_channels_, input_config_,
                         input_->channels());
     RTC_CHECK_EQ(AudioProcessing::kNoError, apm_->set_stream_delay_ms(100));
+    if (enable_pre_amplifier_) {
+      apm_->SetRuntimeSetting(
+          AudioProcessing::RuntimeSetting::CreateCapturePreGain(1 + i % 10));
+    }
     apm_->set_stream_key_pressed(i % 10 == 9);
     RTC_CHECK_EQ(AudioProcessing::kNoError,
                  apm_->ProcessStream(input_->channels(), input_config_,
@@ -596,5 +606,16 @@ TEST_F(DebugDumpTest, TransientSuppressionOn) {
   VerifyDebugDump(generator.dump_file_name());
 }
 
+TEST_F(DebugDumpTest, PreAmplifierIsOn) {
+  Config config;
+  AudioProcessing::Config apm_config;
+  apm_config.pre_amplifier.enabled = true;
+  DebugDumpGenerator generator(config, apm_config);
+  generator.StartRecording();
+  generator.Process(100);
+  generator.StopRecording();
+  VerifyDebugDump(generator.dump_file_name());
+}
+
 }  // namespace test
 }  // namespace webrtc

modules/audio_processing/test/runtime_setting_util.cc

@@ -0,0 +1,25 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/test/runtime_setting_util.h"
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+
+void ReplayRuntimeSetting(AudioProcessing* apm,
+                          const webrtc::audioproc::RuntimeSetting& setting) {
+  RTC_CHECK(apm);
+  // TODO(bugs.webrtc.org/9138): Add ability to handle different types
+  // of settings. Currently only CapturePreGain is supported.
+  RTC_CHECK(setting.has_capture_pre_gain());
+  apm->SetRuntimeSetting(AudioProcessing::RuntimeSetting::CreateCapturePreGain(
+      setting.capture_pre_gain()));
+}
+}  // namespace webrtc

modules/audio_processing/test/runtime_setting_util.h

@@ -0,0 +1,23 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_TEST_RUNTIME_SETTING_UTIL_H_
+#define MODULES_AUDIO_PROCESSING_TEST_RUNTIME_SETTING_UTIL_H_
+
+#include "modules/audio_processing/include/audio_processing.h"
+#include "modules/audio_processing/test/protobuf_utils.h"
+
+namespace webrtc {
+
+void ReplayRuntimeSetting(AudioProcessing* apm,
+                          const webrtc::audioproc::RuntimeSetting& setting);
+}
+
+#endif  // MODULES_AUDIO_PROCESSING_TEST_RUNTIME_SETTING_UTIL_H_