zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AudioMixer: make the number of sources to mix configurable.

Browse Source 
This allows mixing different number of streams depending on the
client's capabilities.
This CL adds `WebRTC.Audio.AudioMixer.NumIncomingActiveStreams2`,
which is defined in [1], since the histogram is not logged anymore
as enum.

[1] https://chromium-review.googlesource.com/c/chromium/src/+/2883627

Bug: webrtc:12746
Change-Id: I0d9b3888f0f95269806539e33b56619b757a5c68
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/218160
Reviewed-by: Alessio Bazzica <alessiob@webrtc.org>
Reviewed-by: Minyue Li <minyue@webrtc.org>
Reviewed-by: Johannes Kron <kron@webrtc.org>
Commit-Queue: Doudou Kisabaka <doudouk@google.com>
Cr-Commit-Position: refs/heads/master@{#34024}
This commit is contained in:
Doudou Kisabaka
2021-05-17 14:15:58 +02:00
committed by WebRTC LUCI CQ
parent 726b0e824b
commit dbf13e32ec
5 changed files with 82 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
modules/audio_mixer/BUILD.gn
View File

@ -46,6 +46,7 @@ rtc_library("audio_mixer_impl") {
"../../common_audio",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"../../rtc_base:safe_conversions",
"../../rtc_base/synchronization:mutex",
"../../system_wrappers",
"../../system_wrappers:metrics",

24
modules/audio_mixer/audio_mixer_impl.cc
View File

@ -126,29 +126,33 @@ struct AudioMixerImpl::HelperContainers {
AudioMixerImpl::AudioMixerImpl(
std::unique_ptr<OutputRateCalculator> output_rate_calculator,
bool use_limiter)
: output_rate_calculator_(std::move(output_rate_calculator)),
bool use_limiter,
int max_sources_to_mix)
: max_sources_to_mix_(max_sources_to_mix),
output_rate_calculator_(std::move(output_rate_calculator)),
audio_source_list_(),
helper_containers_(std::make_unique<HelperContainers>()),
frame_combiner_(use_limiter) {
const int kTypicalMaxNumberOfMixedStreams = 3;
audio_source_list_.reserve(kTypicalMaxNumberOfMixedStreams);
helper_containers_->resize(kTypicalMaxNumberOfMixedStreams);
RTC_CHECK_GE(max_sources_to_mix, 1) << "At least one source must be mixed";
audio_source_list_.reserve(max_sources_to_mix);
helper_containers_->resize(max_sources_to_mix);
}
AudioMixerImpl::~AudioMixerImpl() {}
rtc::scoped_refptr<AudioMixerImpl> AudioMixerImpl::Create() {
rtc::scoped_refptr<AudioMixerImpl> AudioMixerImpl::Create(
int max_sources_to_mix) {
return Create(std::unique_ptr<DefaultOutputRateCalculator>(
new DefaultOutputRateCalculator()),
true);
/*use_limiter=*/true, max_sources_to_mix);
}
rtc::scoped_refptr<AudioMixerImpl> AudioMixerImpl::Create(
std::unique_ptr<OutputRateCalculator> output_rate_calculator,
bool use_limiter) {
bool use_limiter,
int max_sources_to_mix) {
return rtc::make_ref_counted<AudioMixerImpl>(
std::move(output_rate_calculator), use_limiter);
std::move(output_rate_calculator), use_limiter, max_sources_to_mix);
}
void AudioMixerImpl::Mix(size_t number_of_channels,
@ -218,7 +222,7 @@ rtc::ArrayView<AudioFrame* const> AudioMixerImpl::GetAudioFromSources(
std::sort(audio_source_mixing_data_view.begin(),
audio_source_mixing_data_view.end(), ShouldMixBefore);
int max_audio_frame_counter = kMaximumAmountOfMixedAudioSources;
int max_audio_frame_counter = max_sources_to_mix_;
int ramp_list_lengh = 0;
int audio_to_mix_count = 0;
// Go through list in order and put unmuted frames in result list.

14
modules/audio_mixer/audio_mixer_impl.h
View File

@ -35,13 +35,16 @@ class AudioMixerImpl : public AudioMixer {
// AudioProcessing only accepts 10 ms frames.
static const int kFrameDurationInMs = 10;
enum : int { kMaximumAmountOfMixedAudioSources = 3 };
static rtc::scoped_refptr<AudioMixerImpl> Create();
static const int kDefaultNumberOfMixedAudioSources = 3;
static rtc::scoped_refptr<AudioMixerImpl> Create(
int max_sources_to_mix = kDefaultNumberOfMixedAudioSources);
static rtc::scoped_refptr<AudioMixerImpl> Create(
std::unique_ptr<OutputRateCalculator> output_rate_calculator,
bool use_limiter);
bool use_limiter,
int max_sources_to_mix = kDefaultNumberOfMixedAudioSources);
~AudioMixerImpl() override;
@ -60,7 +63,8 @@ class AudioMixerImpl : public AudioMixer {
protected:
AudioMixerImpl(std::unique_ptr<OutputRateCalculator> output_rate_calculator,
bool use_limiter);
bool use_limiter,
int max_sources_to_mix);
private:
struct HelperContainers;
@ -76,6 +80,8 @@ class AudioMixerImpl : public AudioMixer {
// checks that mixing is done sequentially.
mutable Mutex mutex_;
const int max_sources_to_mix_;
std::unique_ptr<OutputRateCalculator> output_rate_calculator_;
// List of all audio sources.

59
modules/audio_mixer/audio_mixer_impl_unittest.cc
View File

@ -12,10 +12,12 @@
#include <string.h>
#include <cstdint>
#include <limits>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "api/audio/audio_mixer.h"
#include "modules/audio_mixer/default_output_rate_calculator.h"
@ -160,7 +162,7 @@ void MixMonoAtGivenNativeRate(int native_sample_rate,
TEST(AudioMixer, LargestEnergyVadActiveMixed) {
constexpr int kAudioSources =
AudioMixerImpl::kMaximumAmountOfMixedAudioSources + 3;
AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 3;
const auto mixer = AudioMixerImpl::Create();
@ -191,7 +193,7 @@ TEST(AudioMixer, LargestEnergyVadActiveMixed) {
mixer->GetAudioSourceMixabilityStatusForTest(&participants[i]);
if (i == kAudioSources - 1 ||
i < kAudioSources - 1 -
AudioMixerImpl::kMaximumAmountOfMixedAudioSources) {
AudioMixerImpl::kDefaultNumberOfMixedAudioSources) {
EXPECT_FALSE(is_mixed)
<< "Mixing status of AudioSource #" << i << " wrong.";
} else {
@ -322,7 +324,7 @@ TEST(AudioMixer, ParticipantNumberOfChannels) {
// another participant with higher energy is added.
TEST(AudioMixer, RampedOutSourcesShouldNotBeMarkedMixed) {
constexpr int kAudioSources =
AudioMixerImpl::kMaximumAmountOfMixedAudioSources + 1;
AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1;
const auto mixer = AudioMixerImpl::Create();
MockMixerAudioSource participants[kAudioSources];
@ -399,7 +401,7 @@ TEST(AudioMixer, ConstructFromOtherThread) {
TEST(AudioMixer, MutedShouldMixAfterUnmuted) {
constexpr int kAudioSources =
AudioMixerImpl::kMaximumAmountOfMixedAudioSources + 1;
AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1;
std::vector<AudioFrame> frames(kAudioSources);
for (auto& frame : frames) {
@ -417,7 +419,7 @@ TEST(AudioMixer, MutedShouldMixAfterUnmuted) {
TEST(AudioMixer, PassiveShouldMixAfterNormal) {
constexpr int kAudioSources =
AudioMixerImpl::kMaximumAmountOfMixedAudioSources + 1;
AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1;
std::vector<AudioFrame> frames(kAudioSources);
for (auto& frame : frames) {
@ -435,7 +437,7 @@ TEST(AudioMixer, PassiveShouldMixAfterNormal) {
TEST(AudioMixer, ActiveShouldMixBeforeLoud) {
constexpr int kAudioSources =
AudioMixerImpl::kMaximumAmountOfMixedAudioSources + 1;
AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1;
std::vector<AudioFrame> frames(kAudioSources);
for (auto& frame : frames) {
@ -454,9 +456,52 @@ TEST(AudioMixer, ActiveShouldMixBeforeLoud) {
MixAndCompare(frames, frame_info, expected_status);
}
TEST(AudioMixer, ShouldMixUpToSpecifiedNumberOfSourcesToMix) {
constexpr int kAudioSources = 5;
constexpr int kSourcesToMix = 2;
std::vector<AudioFrame> frames(kAudioSources);
for (auto& frame : frames) {
ResetFrame(&frame);
}
std::vector<AudioMixer::Source::AudioFrameInfo> frame_info(
kAudioSources, AudioMixer::Source::AudioFrameInfo::kNormal);
// Set up to kSourceToMix sources with kVadActive so that they're mixed.
const std::vector<AudioFrame::VADActivity> kVadActivities = {
AudioFrame::kVadUnknown, AudioFrame::kVadPassive, AudioFrame::kVadPassive,
AudioFrame::kVadActive, AudioFrame::kVadActive};
// Populate VAD and frame for all sources.
for (int i = 0; i < kAudioSources; i++) {
frames[i].vad_activity_ = kVadActivities[i];
}
std::vector<MockMixerAudioSource> participants(kAudioSources);
for (int i = 0; i < kAudioSources; ++i) {
participants[i].fake_frame()->CopyFrom(frames[i]);
participants[i].set_fake_info(frame_info[i]);
}
const auto mixer = AudioMixerImpl::Create(kSourcesToMix);
for (int i = 0; i < kAudioSources; ++i) {
EXPECT_TRUE(mixer->AddSource(&participants[i]));
EXPECT_CALL(participants[i], GetAudioFrameWithInfo(kDefaultSampleRateHz, _))
.Times(Exactly(1));
}
mixer->Mix(1, &frame_for_mixing);
std::vector<bool> expected_status = {false, false, false, true, true};
for (int i = 0; i < kAudioSources; ++i) {
EXPECT_EQ(expected_status[i],
mixer->GetAudioSourceMixabilityStatusForTest(&participants[i]))
<< "Wrong mix status for source #" << i << " is wrong";
}
}
TEST(AudioMixer, UnmutedShouldMixBeforeLoud) {
constexpr int kAudioSources =
AudioMixerImpl::kMaximumAmountOfMixedAudioSources + 1;
AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1;
std::vector<AudioFrame> frames(kAudioSources);
for (auto& frame : frames) {

9
modules/audio_mixer/frame_combiner.cc
View File

@ -26,6 +26,7 @@
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "rtc_base/arraysize.h"
#include "rtc_base/checks.h"
#include "rtc_base/numerics/safe_conversions.h"
#include "system_wrappers/include/metrics.h"
namespace webrtc {
@ -207,10 +208,10 @@ void FrameCombiner::LogMixingStats(
uma_logging_counter_ = 0;
RTC_HISTOGRAM_COUNTS_100("WebRTC.Audio.AudioMixer.NumIncomingStreams",
static_cast<int>(number_of_streams));
RTC_HISTOGRAM_ENUMERATION(
"WebRTC.Audio.AudioMixer.NumIncomingActiveStreams",
static_cast<int>(mix_list.size()),
AudioMixerImpl::kMaximumAmountOfMixedAudioSources);
RTC_HISTOGRAM_COUNTS_LINEAR(
"WebRTC.Audio.AudioMixer.NumIncomingActiveStreams2",
rtc::dchecked_cast<int>(mix_list.size()), /*min=*/1, /*max=*/16,
/*bucket_count=*/16);
using NativeRate = AudioProcessing::NativeRate;
static constexpr NativeRate native_rates[] = {