diff --git a/modules/audio_processing/agc/BUILD.gn b/modules/audio_processing/agc/BUILD.gn index f622a59925..3b2b205385 100644 --- a/modules/audio_processing/agc/BUILD.gn +++ b/modules/audio_processing/agc/BUILD.gn @@ -19,11 +19,13 @@ rtc_library("agc") { ] configs += [ "..:apm_debug_dump" ] deps = [ + ":clipping_predictor", ":gain_control_interface", ":gain_map", ":level_estimation", "..:apm_logging", "..:audio_buffer", + "..:audio_frame_view", "../../../common_audio", "../../../common_audio:common_audio_c", "../../../rtc_base:checks", @@ -38,18 +40,6 @@ rtc_library("agc") { absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } -rtc_library("clipping_predictor_level_buffer") { - sources = [ - "clipping_predictor_level_buffer.cc", - "clipping_predictor_level_buffer.h", - ] - deps = [ - "../../../rtc_base:checks", - "../../../rtc_base:logging", - ] - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] -} - rtc_library("clipping_predictor") { sources = [ "clipping_predictor.cc", @@ -68,6 +58,18 @@ rtc_library("clipping_predictor") { absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } +rtc_library("clipping_predictor_level_buffer") { + sources = [ + "clipping_predictor_level_buffer.cc", + "clipping_predictor_level_buffer.h", + ] + deps = [ + "../../../rtc_base:checks", + "../../../rtc_base:logging", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + rtc_library("level_estimation") { sources = [ "agc.cc", diff --git a/modules/audio_processing/agc/agc_manager_direct.cc b/modules/audio_processing/agc/agc_manager_direct.cc index ebd978b0d0..46304d2819 100644 --- a/modules/audio_processing/agc/agc_manager_direct.cc +++ b/modules/audio_processing/agc/agc_manager_direct.cc @@ -16,6 +16,7 @@ #include "common_audio/include/audio_util.h" #include "modules/audio_processing/agc/gain_control.h" #include "modules/audio_processing/agc/gain_map_internal.h" +#include "modules/audio_processing/include/audio_frame_view.h" #include "rtc_base/atomic_ops.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" @@ -48,6 +49,9 @@ constexpr int kMaxResidualGainChange = 15; // restrictions from clipping events. constexpr int kSurplusCompressionGain = 6; +using ClippingPredictorConfig = AudioProcessing::Config::GainController1:: + AnalogGainController::ClippingPredictor; + // Returns whether a fall-back solution to choose the maximum level should be // chosen. bool UseMaxAnalogChannelLevel() { @@ -125,6 +129,26 @@ float ComputeClippedRatio(const float* const* audio, return static_cast(num_clipped) / (samples_per_channel); } +std::unique_ptr CreateClippingPredictor( + int num_capture_channels, + const ClippingPredictorConfig& config) { + if (config.enabled) { + RTC_LOG(LS_INFO) << "[agc] Clipping prediction enabled."; + switch (config.mode) { + case ClippingPredictorConfig::kClippingEventPrediction: + return CreateClippingEventPredictor(num_capture_channels, config); + case ClippingPredictorConfig::kAdaptiveStepClippingPeakPrediction: + return CreateAdaptiveStepClippingPeakPredictor(num_capture_channels, + config); + case ClippingPredictorConfig::kFixedStepClippingPeakPrediction: + return CreateFixedStepClippingPeakPredictor(num_capture_channels, + config); + } + } else { + return nullptr; + } +} + } // namespace MonoAgc::MonoAgc(ApmDataDumper* data_dumper, @@ -400,7 +424,8 @@ AgcManagerDirect::AgcManagerDirect(Agc* agc, int sample_rate_hz, int clipped_level_step, float clipped_ratio_threshold, - int clipped_wait_frames) + int clipped_wait_frames, + const ClippingPredictorConfig& clipping_cfg) : AgcManagerDirect(/*num_capture_channels*/ 1, startup_min_level, clipped_level_min, @@ -408,7 +433,8 @@ AgcManagerDirect::AgcManagerDirect(Agc* agc, sample_rate_hz, clipped_level_step, clipped_ratio_threshold, - clipped_wait_frames) { + clipped_wait_frames, + clipping_cfg) { RTC_DCHECK(channel_agcs_[0]); RTC_DCHECK(agc); channel_agcs_[0]->set_agc(agc); @@ -421,7 +447,8 @@ AgcManagerDirect::AgcManagerDirect(int num_capture_channels, int sample_rate_hz, int clipped_level_step, float clipped_ratio_threshold, - int clipped_wait_frames) + int clipped_wait_frames, + const ClippingPredictorConfig& clipping_cfg) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_counter_))), use_min_channel_level_(!UseMaxAnalogChannelLevel()), @@ -434,7 +461,9 @@ AgcManagerDirect::AgcManagerDirect(int num_capture_channels, clipped_ratio_threshold_(clipped_ratio_threshold), clipped_wait_frames_(clipped_wait_frames), channel_agcs_(num_capture_channels), - new_compressions_to_set_(num_capture_channels) { + new_compressions_to_set_(num_capture_channels), + clipping_predictor_( + CreateClippingPredictor(num_capture_channels, clipping_cfg)) { const int min_mic_level = GetMinMicLevel(); for (size_t ch = 0; ch < channel_agcs_.size(); ++ch) { ApmDataDumper* data_dumper_ch = ch == 0 ? data_dumper_.get() : nullptr; @@ -449,7 +478,6 @@ AgcManagerDirect::AgcManagerDirect(int num_capture_channels, RTC_DCHECK_GT(clipped_ratio_threshold, 0.f); RTC_DCHECK_LT(clipped_ratio_threshold, 1.f); RTC_DCHECK_GT(clipped_wait_frames, 0); - channel_agcs_[0]->ActivateLogging(); } @@ -500,6 +528,12 @@ void AgcManagerDirect::AnalyzePreProcess(const float* const* audio, return; } + if (!!clipping_predictor_) { + AudioFrameView frame = AudioFrameView( + audio, num_capture_channels_, static_cast(samples_per_channel)); + clipping_predictor_->Process(frame); + } + if (frames_since_clipped_ < clipped_wait_frames_) { ++frames_since_clipped_; return; @@ -516,14 +550,37 @@ void AgcManagerDirect::AnalyzePreProcess(const float* const* audio, // gain is increased, through SetMaxLevel(). float clipped_ratio = ComputeClippedRatio(audio, num_capture_channels_, samples_per_channel); - - if (clipped_ratio > clipped_ratio_threshold_) { - RTC_DLOG(LS_INFO) << "[agc] Clipping detected. clipped_ratio=" - << clipped_ratio; + const bool clipping_detected = clipped_ratio > clipped_ratio_threshold_; + bool clipping_predicted = false; + int predicted_step = 0; + if (!!clipping_predictor_) { + for (int channel = 0; channel < num_capture_channels_; ++channel) { + const auto step = clipping_predictor_->EstimateClippedLevelStep( + channel, stream_analog_level_, clipped_level_step_, + channel_agcs_[channel]->min_mic_level(), kMaxMicLevel); + if (step.has_value()) { + predicted_step = std::max(predicted_step, step.value()); + clipping_predicted = true; + } + } + } + if (clipping_detected || clipping_predicted) { + int step = clipped_level_step_; + if (clipping_detected) { + RTC_DLOG(LS_INFO) << "[agc] Clipping detected. clipped_ratio=" + << clipped_ratio; + } + if (clipping_predicted) { + step = std::max(predicted_step, clipped_level_step_); + RTC_DLOG(LS_INFO) << "[agc] Clipping predicted. step=" << step; + } for (auto& state_ch : channel_agcs_) { - state_ch->HandleClipping(clipped_level_step_); + state_ch->HandleClipping(step); } frames_since_clipped_ = 0; + if (!!clipping_predictor_) { + clipping_predictor_->Reset(); + } } AggregateChannelLevels(); } @@ -606,4 +663,8 @@ void AgcManagerDirect::AggregateChannelLevels() { } } +bool AgcManagerDirect::clipping_predictor_enabled() const { + return !!clipping_predictor_; +} + } // namespace webrtc diff --git a/modules/audio_processing/agc/agc_manager_direct.h b/modules/audio_processing/agc/agc_manager_direct.h index e0be1a0dd2..55a7ffa2eb 100644 --- a/modules/audio_processing/agc/agc_manager_direct.h +++ b/modules/audio_processing/agc/agc_manager_direct.h @@ -15,6 +15,7 @@ #include "absl/types/optional.h" #include "modules/audio_processing/agc/agc.h" +#include "modules/audio_processing/agc/clipping_predictor.h" #include "modules/audio_processing/audio_buffer.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/gtest_prod_util.h" @@ -47,7 +48,9 @@ class AgcManagerDirect final { int sample_rate_hz, int clipped_level_step, float clipped_ratio_threshold, - int clipped_wait_frames); + int clipped_wait_frames, + const AudioProcessing::Config::GainController1:: + AnalogGainController::ClippingPredictor& clipping_cfg); ~AgcManagerDirect(); AgcManagerDirect(const AgcManagerDirect&) = delete; @@ -69,6 +72,9 @@ class AgcManagerDirect final { int num_channels() const { return num_capture_channels_; } int sample_rate_hz() const { return sample_rate_hz_; } + // Returns true if clipping prediction was set to be used in ctor. + bool clipping_predictor_enabled() const; + // If available, returns a new compression gain for the digital gain control. absl::optional GetDigitalComressionGain(); @@ -91,6 +97,10 @@ class AgcManagerDirect final { AgcMinMicLevelExperimentEnabledAboveStartupLevel); FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectStandaloneTest, ClippingParametersVerified); + FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectStandaloneTest, + DisableClippingPredictorDoesNotLowerVolume); + FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectStandaloneTest, + EnableClippingPredictorLowersVolume); // Dependency injection for testing. Don't delete |agc| as the memory is owned // by the manager. @@ -100,7 +110,9 @@ class AgcManagerDirect final { int sample_rate_hz, int clipped_level_step, float clipped_ratio_threshold, - int clipped_wait_frames); + int clipped_wait_frames, + const AudioProcessing::Config::GainController1:: + AnalogGainController::ClippingPredictor& clipping_cfg); void AnalyzePreProcess(const float* const* audio, size_t samples_per_channel); @@ -124,6 +136,8 @@ class AgcManagerDirect final { std::vector> channel_agcs_; std::vector> new_compressions_to_set_; + + const std::unique_ptr clipping_predictor_; }; class MonoAgc { diff --git a/modules/audio_processing/agc/agc_manager_direct_unittest.cc b/modules/audio_processing/agc/agc_manager_direct_unittest.cc index 6fdfa6d805..07bb04022b 100644 --- a/modules/audio_processing/agc/agc_manager_direct_unittest.cc +++ b/modules/audio_processing/agc/agc_manager_direct_unittest.cc @@ -37,6 +37,9 @@ constexpr int kClippedLevelStep = 15; constexpr float kClippedRatioThreshold = 0.1f; constexpr int kClippedWaitFrames = 300; +using ClippingPredictorConfig = AudioProcessing::Config::GainController1:: + AnalogGainController::ClippingPredictor; + class MockGainControl : public GainControl { public: virtual ~MockGainControl() {} @@ -67,7 +70,46 @@ std::unique_ptr CreateAgcManagerDirect( return std::make_unique( /*num_capture_channels=*/1, startup_min_level, kClippedMin, /*disable_digital_adaptive=*/true, kSampleRateHz, clipped_level_step, - clipped_ratio_threshold, clipped_wait_frames); + clipped_ratio_threshold, clipped_wait_frames, ClippingPredictorConfig()); +} + +std::unique_ptr CreateAgcManagerDirect( + int startup_min_level, + int clipped_level_step, + float clipped_ratio_threshold, + int clipped_wait_frames, + const ClippingPredictorConfig& clipping_cfg) { + return std::make_unique( + /*num_capture_channels=*/1, startup_min_level, kClippedMin, + /*disable_digital_adaptive=*/true, kSampleRateHz, clipped_level_step, + clipped_ratio_threshold, clipped_wait_frames, clipping_cfg); +} + +void CallPreProcessAudioBuffer(int num_calls, + float peak_ratio, + AgcManagerDirect& manager) { + RTC_DCHECK_GE(1.f, peak_ratio); + AudioBuffer audio_buffer(kSampleRateHz, 1, kSampleRateHz, 1, kSampleRateHz, + 1); + const int num_channels = audio_buffer.num_channels(); + const int num_frames = audio_buffer.num_frames(); + for (int ch = 0; ch < num_channels; ++ch) { + for (int i = 0; i < num_frames; i += 2) { + audio_buffer.channels()[ch][i] = peak_ratio * 32767.f; + audio_buffer.channels()[ch][i + 1] = 0.0f; + } + } + for (int n = 0; n < num_calls / 2; ++n) { + manager.AnalyzePreProcess(&audio_buffer); + } + for (int ch = 0; ch < num_channels; ++ch) { + for (int i = 0; i < num_frames; ++i) { + audio_buffer.channels()[ch][i] = peak_ratio * 32767.f; + } + } + for (int n = 0; n < num_calls - num_calls / 2; ++n) { + manager.AnalyzePreProcess(&audio_buffer); + } } } // namespace @@ -82,7 +124,8 @@ class AgcManagerDirectTest : public ::testing::Test { kSampleRateHz, kClippedLevelStep, kClippedRatioThreshold, - kClippedWaitFrames), + kClippedWaitFrames, + ClippingPredictorConfig()), audio(kNumChannels), audio_data(kNumChannels * kSamplesPerChannel, 0.f) { ExpectInitialize(); @@ -137,12 +180,32 @@ class AgcManagerDirectTest : public ::testing::Test { audio[ch][k] = 32767.f; } } - for (int i = 0; i < num_calls; ++i) { manager_.AnalyzePreProcess(audio.data(), kSamplesPerChannel); } } + void CallPreProcForChangingAudio(int num_calls, float peak_ratio) { + RTC_DCHECK_GE(1.f, peak_ratio); + std::fill(audio_data.begin(), audio_data.end(), 0.f); + for (size_t ch = 0; ch < kNumChannels; ++ch) { + for (size_t k = 0; k < kSamplesPerChannel; k += 2) { + audio[ch][k] = peak_ratio * 32767.f; + } + } + for (int i = 0; i < num_calls / 2; ++i) { + manager_.AnalyzePreProcess(audio.data(), kSamplesPerChannel); + } + for (size_t ch = 0; ch < kNumChannels; ++ch) { + for (size_t k = 0; k < kSamplesPerChannel; ++k) { + audio[ch][k] = peak_ratio * 32767.f; + } + } + for (int i = 0; i < num_calls - num_calls / 2; ++i) { + manager_.AnalyzePreProcess(audio.data(), kSamplesPerChannel); + } + } + MockAgc* agc_; MockGainControl gctrl_; AgcManagerDirect manager_; @@ -709,6 +772,25 @@ TEST_F(AgcManagerDirectTest, TakesNoActionOnZeroMicVolume) { EXPECT_EQ(0, manager_.stream_analog_level()); } +TEST_F(AgcManagerDirectTest, ClippingDetectionLowersVolume) { + SetVolumeAndProcess(255); + EXPECT_EQ(255, manager_.stream_analog_level()); + CallPreProcForChangingAudio(/*num_calls=*/100, /*peak_ratio=*/0.99f); + EXPECT_EQ(255, manager_.stream_analog_level()); + CallPreProcForChangingAudio(/*num_calls=*/100, /*peak_ratio=*/1.0f); + EXPECT_EQ(240, manager_.stream_analog_level()); +} + +TEST_F(AgcManagerDirectTest, DisabledClippingPredictorDoesNotLowerVolume) { + SetVolumeAndProcess(255); + EXPECT_FALSE(manager_.clipping_predictor_enabled()); + EXPECT_EQ(255, manager_.stream_analog_level()); + CallPreProcForChangingAudio(/*num_calls=*/100, /*peak_ratio=*/0.99f); + EXPECT_EQ(255, manager_.stream_analog_level()); + CallPreProcForChangingAudio(/*num_calls=*/100, /*peak_ratio=*/0.99f); + EXPECT_EQ(255, manager_.stream_analog_level()); +} + TEST(AgcManagerDirectStandaloneTest, DisableDigitalDisablesDigital) { auto agc = std::unique_ptr(new ::testing::NiceMock()); MockGainControl gctrl; @@ -807,13 +889,81 @@ TEST(AgcManagerDirectStandaloneTest, ClippingParametersVerified) { EXPECT_EQ(manager->clipped_wait_frames_, kClippedWaitFrames); std::unique_ptr manager_custom = CreateAgcManagerDirect(kInitialVolume, - /*clipped_level_step*/ 10, - /*clipped_ratio_threshold*/ 0.2f, - /*clipped_wait_frames*/ 50); + /*clipped_level_step=*/10, + /*clipped_ratio_threshold=*/0.2f, + /*clipped_wait_frames=*/50); manager_custom->Initialize(); EXPECT_EQ(manager_custom->clipped_level_step_, 10); EXPECT_EQ(manager_custom->clipped_ratio_threshold_, 0.2f); EXPECT_EQ(manager_custom->clipped_wait_frames_, 50); } +TEST(AgcManagerDirectStandaloneTest, + DisableClippingPredictorDisablesClippingPredictor) { + ClippingPredictorConfig default_config; + EXPECT_FALSE(default_config.enabled); + std::unique_ptr manager = CreateAgcManagerDirect( + kInitialVolume, kClippedLevelStep, kClippedRatioThreshold, + kClippedWaitFrames, default_config); + manager->Initialize(); + EXPECT_FALSE(manager->clipping_predictor_enabled()); +} + +TEST(AgcManagerDirectStandaloneTest, + EnableClippingPredictorEnablesClippingPredictor) { + const ClippingPredictorConfig config( + {/*enabled=*/true, ClippingPredictorConfig::kClippingEventPrediction, + /*window_length=*/5, /*reference_window_length=*/5, + /*reference_window_delay=*/5, /*clipping_threshold=*/-1.0f, + /*crest_factor_margin=*/3.0f}); + std::unique_ptr manager = CreateAgcManagerDirect( + kInitialVolume, kClippedLevelStep, kClippedRatioThreshold, + kClippedWaitFrames, config); + manager->Initialize(); + EXPECT_TRUE(manager->clipping_predictor_enabled()); +} + +TEST(AgcManagerDirectStandaloneTest, + DisableClippingPredictorDoesNotLowerVolume) { + const ClippingPredictorConfig default_config; + EXPECT_FALSE(default_config.enabled); + AgcManagerDirect manager(new ::testing::NiceMock(), kInitialVolume, + kClippedMin, kSampleRateHz, kClippedLevelStep, + kClippedRatioThreshold, kClippedWaitFrames, + default_config); + manager.Initialize(); + manager.set_stream_analog_level(/*level=*/255); + EXPECT_FALSE(manager.clipping_predictor_enabled()); + EXPECT_EQ(manager.stream_analog_level(), 255); + manager.Process(nullptr); + CallPreProcessAudioBuffer(/*num_calls=*/10, /*peak_ratio=*/0.99f, manager); + EXPECT_EQ(manager.stream_analog_level(), 255); + CallPreProcessAudioBuffer(/*num_calls=*/300, /*peak_ratio=*/0.99f, manager); + EXPECT_EQ(manager.stream_analog_level(), 255); + CallPreProcessAudioBuffer(/*num_calls=*/10, /*peak_ratio=*/0.99f, manager); + EXPECT_EQ(manager.stream_analog_level(), 255); +} + +TEST(AgcManagerDirectStandaloneTest, EnableClippingPredictorLowersVolume) { + const ClippingPredictorConfig config( + {/*enabled=*/true, ClippingPredictorConfig::kClippingEventPrediction, + /*window_length=*/5, /*reference_window_length=*/5, + /*reference_window_delay=*/5, /*clipping_threshold=*/-1.0f, + /*crest_factor_margin=*/3.0f}); + AgcManagerDirect manager(new ::testing::NiceMock(), kInitialVolume, + kClippedMin, kSampleRateHz, kClippedLevelStep, + kClippedRatioThreshold, kClippedWaitFrames, config); + manager.Initialize(); + manager.set_stream_analog_level(/*level=*/255); + EXPECT_TRUE(manager.clipping_predictor_enabled()); + EXPECT_EQ(manager.stream_analog_level(), 255); + manager.Process(nullptr); + CallPreProcessAudioBuffer(/*num_calls=*/10, /*peak_ratio=*/0.99f, manager); + EXPECT_EQ(manager.stream_analog_level(), 240); + CallPreProcessAudioBuffer(/*num_calls=*/300, /*peak_ratio=*/0.99f, manager); + EXPECT_EQ(manager.stream_analog_level(), 240); + CallPreProcessAudioBuffer(/*num_calls=*/10, /*peak_ratio=*/0.99f, manager); + EXPECT_EQ(manager.stream_analog_level(), 225); +} + } // namespace webrtc diff --git a/modules/audio_processing/agc/clipping_predictor.h b/modules/audio_processing/agc/clipping_predictor.h index 301e47ed47..0fe98273fb 100644 --- a/modules/audio_processing/agc/clipping_predictor.h +++ b/modules/audio_processing/agc/clipping_predictor.h @@ -48,21 +48,21 @@ class ClippingPredictor { // prediction. std::unique_ptr CreateClippingEventPredictor( int num_channels, - const AudioProcessing::Config::GainController1 ::AnalogGainController:: + const AudioProcessing::Config::GainController1::AnalogGainController:: ClippingPredictor& config); // Creates a ClippingPredictor based on crest factor-based peak estimation and // fixed-step clipped level step estimation. std::unique_ptr CreateFixedStepClippingPeakPredictor( int num_channels, - const AudioProcessing::Config::GainController1 ::AnalogGainController:: + const AudioProcessing::Config::GainController1::AnalogGainController:: ClippingPredictor& config); // Creates a ClippingPredictor based on crest factor-based peak estimation and // adaptive-step clipped level step estimation. std::unique_ptr CreateAdaptiveStepClippingPeakPredictor( int num_channels, - const AudioProcessing::Config::GainController1 ::AnalogGainController:: + const AudioProcessing::Config::GainController1::AnalogGainController:: ClippingPredictor& config); } // namespace webrtc diff --git a/modules/audio_processing/agc/clipping_predictor_unittest.cc b/modules/audio_processing/agc/clipping_predictor_unittest.cc index e27ae287dd..ab76abab68 100644 --- a/modules/audio_processing/agc/clipping_predictor_unittest.cc +++ b/modules/audio_processing/agc/clipping_predictor_unittest.cc @@ -32,7 +32,7 @@ constexpr int kMaxMicLevel = 255; constexpr int kMinMicLevel = 12; constexpr int kDefaultClippedLevelStep = 15; -using ClippingPredictorConfig = AudioProcessing::Config::GainController1 :: +using ClippingPredictorConfig = AudioProcessing::Config::GainController1:: AnalogGainController::ClippingPredictor; void CallProcess(int num_calls, diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc index 3c5d9fb718..4a1985545f 100644 --- a/modules/audio_processing/audio_processing_impl.cc +++ b/modules/audio_processing/audio_processing_impl.cc @@ -1921,7 +1921,8 @@ void AudioProcessingImpl::InitializeGainController1() { capture_nonlocked_.split_rate, config_.gain_controller1.analog_gain_controller.clipped_level_step, config_.gain_controller1.analog_gain_controller.clipped_ratio_threshold, - config_.gain_controller1.analog_gain_controller.clipped_wait_frames)); + config_.gain_controller1.analog_gain_controller.clipped_wait_frames, + config_.gain_controller1.analog_gain_controller.clipping_predictor)); if (re_creation) { submodules_.agc_manager->set_stream_analog_level(stream_analog_level); }