diff --git a/modules/audio_mixer/BUILD.gn b/modules/audio_mixer/BUILD.gn index 739d62d705..5f227c737f 100644 --- a/modules/audio_mixer/BUILD.gn +++ b/modules/audio_mixer/BUILD.gn @@ -46,6 +46,7 @@ rtc_library("audio_mixer_impl") { "../../common_audio", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", + "../../rtc_base:safe_conversions", "../../rtc_base/synchronization:mutex", "../../system_wrappers", "../../system_wrappers:metrics", diff --git a/modules/audio_mixer/audio_mixer_impl.cc b/modules/audio_mixer/audio_mixer_impl.cc index 925cee5092..8cebc38779 100644 --- a/modules/audio_mixer/audio_mixer_impl.cc +++ b/modules/audio_mixer/audio_mixer_impl.cc @@ -126,29 +126,33 @@ struct AudioMixerImpl::HelperContainers { AudioMixerImpl::AudioMixerImpl( std::unique_ptr output_rate_calculator, - bool use_limiter) - : output_rate_calculator_(std::move(output_rate_calculator)), + bool use_limiter, + int max_sources_to_mix) + : max_sources_to_mix_(max_sources_to_mix), + output_rate_calculator_(std::move(output_rate_calculator)), audio_source_list_(), helper_containers_(std::make_unique()), frame_combiner_(use_limiter) { - const int kTypicalMaxNumberOfMixedStreams = 3; - audio_source_list_.reserve(kTypicalMaxNumberOfMixedStreams); - helper_containers_->resize(kTypicalMaxNumberOfMixedStreams); + RTC_CHECK_GE(max_sources_to_mix, 1) << "At least one source must be mixed"; + audio_source_list_.reserve(max_sources_to_mix); + helper_containers_->resize(max_sources_to_mix); } AudioMixerImpl::~AudioMixerImpl() {} -rtc::scoped_refptr AudioMixerImpl::Create() { +rtc::scoped_refptr AudioMixerImpl::Create( + int max_sources_to_mix) { return Create(std::unique_ptr( new DefaultOutputRateCalculator()), - true); + /*use_limiter=*/true, max_sources_to_mix); } rtc::scoped_refptr AudioMixerImpl::Create( std::unique_ptr output_rate_calculator, - bool use_limiter) { + bool use_limiter, + int max_sources_to_mix) { return rtc::make_ref_counted( - std::move(output_rate_calculator), use_limiter); + std::move(output_rate_calculator), use_limiter, max_sources_to_mix); } void AudioMixerImpl::Mix(size_t number_of_channels, @@ -218,7 +222,7 @@ rtc::ArrayView AudioMixerImpl::GetAudioFromSources( std::sort(audio_source_mixing_data_view.begin(), audio_source_mixing_data_view.end(), ShouldMixBefore); - int max_audio_frame_counter = kMaximumAmountOfMixedAudioSources; + int max_audio_frame_counter = max_sources_to_mix_; int ramp_list_lengh = 0; int audio_to_mix_count = 0; // Go through list in order and put unmuted frames in result list. diff --git a/modules/audio_mixer/audio_mixer_impl.h b/modules/audio_mixer/audio_mixer_impl.h index 0a13082725..737fcbdc43 100644 --- a/modules/audio_mixer/audio_mixer_impl.h +++ b/modules/audio_mixer/audio_mixer_impl.h @@ -35,13 +35,16 @@ class AudioMixerImpl : public AudioMixer { // AudioProcessing only accepts 10 ms frames. static const int kFrameDurationInMs = 10; - enum : int { kMaximumAmountOfMixedAudioSources = 3 }; - static rtc::scoped_refptr Create(); + static const int kDefaultNumberOfMixedAudioSources = 3; + + static rtc::scoped_refptr Create( + int max_sources_to_mix = kDefaultNumberOfMixedAudioSources); static rtc::scoped_refptr Create( std::unique_ptr output_rate_calculator, - bool use_limiter); + bool use_limiter, + int max_sources_to_mix = kDefaultNumberOfMixedAudioSources); ~AudioMixerImpl() override; @@ -60,7 +63,8 @@ class AudioMixerImpl : public AudioMixer { protected: AudioMixerImpl(std::unique_ptr output_rate_calculator, - bool use_limiter); + bool use_limiter, + int max_sources_to_mix); private: struct HelperContainers; @@ -76,6 +80,8 @@ class AudioMixerImpl : public AudioMixer { // checks that mixing is done sequentially. mutable Mutex mutex_; + const int max_sources_to_mix_; + std::unique_ptr output_rate_calculator_; // List of all audio sources. diff --git a/modules/audio_mixer/audio_mixer_impl_unittest.cc b/modules/audio_mixer/audio_mixer_impl_unittest.cc index c2f02fbfbd..e4ba6ce4c2 100644 --- a/modules/audio_mixer/audio_mixer_impl_unittest.cc +++ b/modules/audio_mixer/audio_mixer_impl_unittest.cc @@ -12,10 +12,12 @@ #include +#include #include #include #include #include +#include #include "api/audio/audio_mixer.h" #include "modules/audio_mixer/default_output_rate_calculator.h" @@ -160,7 +162,7 @@ void MixMonoAtGivenNativeRate(int native_sample_rate, TEST(AudioMixer, LargestEnergyVadActiveMixed) { constexpr int kAudioSources = - AudioMixerImpl::kMaximumAmountOfMixedAudioSources + 3; + AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 3; const auto mixer = AudioMixerImpl::Create(); @@ -191,7 +193,7 @@ TEST(AudioMixer, LargestEnergyVadActiveMixed) { mixer->GetAudioSourceMixabilityStatusForTest(&participants[i]); if (i == kAudioSources - 1 || i < kAudioSources - 1 - - AudioMixerImpl::kMaximumAmountOfMixedAudioSources) { + AudioMixerImpl::kDefaultNumberOfMixedAudioSources) { EXPECT_FALSE(is_mixed) << "Mixing status of AudioSource #" << i << " wrong."; } else { @@ -322,7 +324,7 @@ TEST(AudioMixer, ParticipantNumberOfChannels) { // another participant with higher energy is added. TEST(AudioMixer, RampedOutSourcesShouldNotBeMarkedMixed) { constexpr int kAudioSources = - AudioMixerImpl::kMaximumAmountOfMixedAudioSources + 1; + AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1; const auto mixer = AudioMixerImpl::Create(); MockMixerAudioSource participants[kAudioSources]; @@ -399,7 +401,7 @@ TEST(AudioMixer, ConstructFromOtherThread) { TEST(AudioMixer, MutedShouldMixAfterUnmuted) { constexpr int kAudioSources = - AudioMixerImpl::kMaximumAmountOfMixedAudioSources + 1; + AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1; std::vector frames(kAudioSources); for (auto& frame : frames) { @@ -417,7 +419,7 @@ TEST(AudioMixer, MutedShouldMixAfterUnmuted) { TEST(AudioMixer, PassiveShouldMixAfterNormal) { constexpr int kAudioSources = - AudioMixerImpl::kMaximumAmountOfMixedAudioSources + 1; + AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1; std::vector frames(kAudioSources); for (auto& frame : frames) { @@ -435,7 +437,7 @@ TEST(AudioMixer, PassiveShouldMixAfterNormal) { TEST(AudioMixer, ActiveShouldMixBeforeLoud) { constexpr int kAudioSources = - AudioMixerImpl::kMaximumAmountOfMixedAudioSources + 1; + AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1; std::vector frames(kAudioSources); for (auto& frame : frames) { @@ -454,9 +456,52 @@ TEST(AudioMixer, ActiveShouldMixBeforeLoud) { MixAndCompare(frames, frame_info, expected_status); } +TEST(AudioMixer, ShouldMixUpToSpecifiedNumberOfSourcesToMix) { + constexpr int kAudioSources = 5; + constexpr int kSourcesToMix = 2; + + std::vector frames(kAudioSources); + for (auto& frame : frames) { + ResetFrame(&frame); + } + + std::vector frame_info( + kAudioSources, AudioMixer::Source::AudioFrameInfo::kNormal); + // Set up to kSourceToMix sources with kVadActive so that they're mixed. + const std::vector kVadActivities = { + AudioFrame::kVadUnknown, AudioFrame::kVadPassive, AudioFrame::kVadPassive, + AudioFrame::kVadActive, AudioFrame::kVadActive}; + // Populate VAD and frame for all sources. + for (int i = 0; i < kAudioSources; i++) { + frames[i].vad_activity_ = kVadActivities[i]; + } + + std::vector participants(kAudioSources); + for (int i = 0; i < kAudioSources; ++i) { + participants[i].fake_frame()->CopyFrom(frames[i]); + participants[i].set_fake_info(frame_info[i]); + } + + const auto mixer = AudioMixerImpl::Create(kSourcesToMix); + for (int i = 0; i < kAudioSources; ++i) { + EXPECT_TRUE(mixer->AddSource(&participants[i])); + EXPECT_CALL(participants[i], GetAudioFrameWithInfo(kDefaultSampleRateHz, _)) + .Times(Exactly(1)); + } + + mixer->Mix(1, &frame_for_mixing); + + std::vector expected_status = {false, false, false, true, true}; + for (int i = 0; i < kAudioSources; ++i) { + EXPECT_EQ(expected_status[i], + mixer->GetAudioSourceMixabilityStatusForTest(&participants[i])) + << "Wrong mix status for source #" << i << " is wrong"; + } +} + TEST(AudioMixer, UnmutedShouldMixBeforeLoud) { constexpr int kAudioSources = - AudioMixerImpl::kMaximumAmountOfMixedAudioSources + 1; + AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1; std::vector frames(kAudioSources); for (auto& frame : frames) { diff --git a/modules/audio_mixer/frame_combiner.cc b/modules/audio_mixer/frame_combiner.cc index fb6f72af75..db301aac72 100644 --- a/modules/audio_mixer/frame_combiner.cc +++ b/modules/audio_mixer/frame_combiner.cc @@ -26,6 +26,7 @@ #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/arraysize.h" #include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" #include "system_wrappers/include/metrics.h" namespace webrtc { @@ -207,10 +208,10 @@ void FrameCombiner::LogMixingStats( uma_logging_counter_ = 0; RTC_HISTOGRAM_COUNTS_100("WebRTC.Audio.AudioMixer.NumIncomingStreams", static_cast(number_of_streams)); - RTC_HISTOGRAM_ENUMERATION( - "WebRTC.Audio.AudioMixer.NumIncomingActiveStreams", - static_cast(mix_list.size()), - AudioMixerImpl::kMaximumAmountOfMixedAudioSources); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.AudioMixer.NumIncomingActiveStreams2", + rtc::dchecked_cast(mix_list.size()), /*min=*/1, /*max=*/16, + /*bucket_count=*/16); using NativeRate = AudioProcessing::NativeRate; static constexpr NativeRate native_rates[] = {