zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove additional channel constraints when Beamforming is enabled in AudioProcessing

Browse Source 
The general constraints on number of channels for AudioProcessing is:
num_in_channels == num_out_channels || num_out_channels == 1

When Beamforming is enabled and additional constraint was added forcing:
num_out_channels == 1

This artificial constraint was removed by adding upmixing support in CopyTo, since it was already supported for the AudioFrame interface using InterleaveTo.

Review URL: https://codereview.webrtc.org/1571013002

Cr-Commit-Position: refs/heads/master@{#11215}
This commit is contained in:
aluebs
2016-01-11 20:32:29 -08:00
committed by Commit bot
parent e93ad1b129
commit b2328d11dc
7 changed files with 35 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
webrtc/modules/audio_processing/audio_buffer.cc
View File

@ -150,7 +150,7 @@ void AudioBuffer::CopyFrom(const float* const* data,
void AudioBuffer::CopyTo(const StreamConfig& stream_config,
float* const* data) {
assert(stream_config.num_frames() == output_num_frames_);
assert(stream_config.num_channels() == num_channels_);
assert(stream_config.num_channels() == num_channels_ || num_channels_ == 1);
// Convert to the float range.
float* const* data_ptr = data;
@ -173,6 +173,11 @@ void AudioBuffer::CopyTo(const StreamConfig& stream_config,
output_num_frames_);
}
}
// Upmix.
for (int i = num_channels_; i < stream_config.num_channels(); ++i) {
memcpy(data[i], data[0], output_num_frames_ * sizeof(**data));
}
}
void AudioBuffer::InitForNewData() {

35
webrtc/modules/audio_processing/audio_processing_impl.cc
View File

@ -226,9 +226,9 @@ AudioProcessingImpl::AudioProcessingImpl(const Config& config,
#else
capture_(config.Get<ExperimentalNs>().enabled,
#endif
config.Get<Beamforming>().enabled,
config.Get<Beamforming>().array_geometry,
config.Get<Beamforming>().target_direction)
config.Get<Beamforming>().target_direction),
capture_nonlocked_(config.Get<Beamforming>().enabled)
{
{
rtc::CritScope cs_render(&crit_render_);
@ -345,7 +345,7 @@ int AudioProcessingImpl::MaybeInitialize(
int AudioProcessingImpl::InitializeLocked() {
const int fwd_audio_buffer_channels =
capture_.beamformer_enabled
capture_nonlocked_.beamformer_enabled
? formats_.api_format.input_stream().num_channels()
: formats_.api_format.output_stream().num_channels();
const int rev_audio_buffer_out_num_frames =
@ -428,9 +428,8 @@ int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) {
return kBadNumberChannelsError;
}
if (capture_.beamformer_enabled &&
(static_cast<size_t>(num_in_channels) != capture_.array_geometry.size() ||
num_out_channels > 1)) {
if (capture_nonlocked_.beamformer_enabled &&
static_cast<size_t>(num_in_channels) != capture_.array_geometry.size()) {
return kBadNumberChannelsError;
}
@ -500,8 +499,9 @@ void AudioProcessingImpl::SetExtraOptions(const Config& config) {
}
#ifdef WEBRTC_ANDROID_PLATFORM_BUILD
if (capture_.beamformer_enabled != config.Get<Beamforming>().enabled) {
capture_.beamformer_enabled = config.Get<Beamforming>().enabled;
if (capture_nonlocked_.beamformer_enabled !=
config.Get<Beamforming>().enabled) {
capture_nonlocked_.beamformer_enabled = config.Get<Beamforming>().enabled;
if (config.Get<Beamforming>().array_geometry.size() > 1) {
capture_.array_geometry = config.Get<Beamforming>().array_geometry;
}
@ -537,6 +537,11 @@ int AudioProcessingImpl::num_input_channels() const {
return formats_.api_format.input_stream().num_channels();
}
int AudioProcessingImpl::num_proc_channels() const {
// Used as callback from submodules, hence locking is not allowed.
return capture_nonlocked_.beamformer_enabled ? 1 : num_output_channels();
}
int AudioProcessingImpl::num_output_channels() const {
// Used as callback from submodules, hence locking is not allowed.
return formats_.api_format.output_stream().num_channels();
@ -771,7 +776,7 @@ int AudioProcessingImpl::ProcessStreamLocked() {
ca->num_channels());
}
if (capture_.beamformer_enabled) {
if (capture_nonlocked_.beamformer_enabled) {
private_submodules_->beamformer->ProcessChunk(*ca->split_data_f(),
ca->split_data_f());
ca->set_num_channels(1);
@ -793,7 +798,7 @@ int AudioProcessingImpl::ProcessStreamLocked() {
if (constants_.use_new_agc &&
public_submodules_->gain_control->is_enabled() &&
(!capture_.beamformer_enabled ||
(!capture_nonlocked_.beamformer_enabled ||
private_submodules_->beamformer->is_target_present())) {
private_submodules_->agc_manager->Process(
ca->split_bands_const(0)[kBand0To8kHz], ca->num_frames_per_band(),
@ -1183,7 +1188,7 @@ VoiceDetection* AudioProcessingImpl::voice_detection() const {
}
bool AudioProcessingImpl::is_data_processed() const {
if (capture_.beamformer_enabled) {
if (capture_nonlocked_.beamformer_enabled) {
return true;
}
@ -1293,12 +1298,12 @@ void AudioProcessingImpl::InitializeTransient() {
public_submodules_->transient_suppressor->Initialize(
capture_nonlocked_.fwd_proc_format.sample_rate_hz(),
capture_nonlocked_.split_rate,
formats_.api_format.output_stream().num_channels());
num_proc_channels());
}
}
void AudioProcessingImpl::InitializeBeamformer() {
if (capture_.beamformer_enabled) {
if (capture_nonlocked_.beamformer_enabled) {
if (!private_submodules_->beamformer) {
private_submodules_->beamformer.reset(new NonlinearBeamformer(
capture_.array_geometry, capture_.target_direction));
@ -1320,12 +1325,12 @@ void AudioProcessingImpl::InitializeIntelligibility() {
}
void AudioProcessingImpl::InitializeHighPassFilter() {
public_submodules_->high_pass_filter->Initialize(num_output_channels(),
public_submodules_->high_pass_filter->Initialize(num_proc_channels(),
proc_sample_rate_hz());
}
void AudioProcessingImpl::InitializeNoiseSuppression() {
public_submodules_->noise_suppression->Initialize(num_output_channels(),
public_submodules_->noise_suppression->Initialize(num_proc_channels(),
proc_sample_rate_hz());
}

10
webrtc/modules/audio_processing/audio_processing_impl.h
View File

@ -102,6 +102,7 @@ class AudioProcessingImpl : public AudioProcessing {
int proc_sample_rate_hz() const override;
int proc_split_sample_rate_hz() const override;
int num_input_channels() const override;
int num_proc_channels() const override;
int num_output_channels() const override;
int num_reverse_channels() const override;
int stream_delay_ms() const override;
@ -280,7 +281,6 @@ class AudioProcessingImpl : public AudioProcessing {
struct ApmCaptureState {
ApmCaptureState(bool transient_suppressor_enabled,
bool beamformer_enabled,
const std::vector<Point>& array_geometry,
SphericalPointf target_direction)
: aec_system_delay_jumps(-1),
@ -292,7 +292,6 @@ class AudioProcessingImpl : public AudioProcessing {
output_will_be_muted(false),
key_pressed(false),
transient_suppressor_enabled(transient_suppressor_enabled),
beamformer_enabled(beamformer_enabled),
array_geometry(array_geometry),
target_direction(target_direction),
fwd_proc_format(kSampleRate16kHz),
@ -306,7 +305,6 @@ class AudioProcessingImpl : public AudioProcessing {
bool output_will_be_muted;
bool key_pressed;
bool transient_suppressor_enabled;
bool beamformer_enabled;
std::vector<Point> array_geometry;
SphericalPointf target_direction;
rtc::scoped_ptr<AudioBuffer> capture_audio;
@ -318,16 +316,18 @@ class AudioProcessingImpl : public AudioProcessing {
} capture_ GUARDED_BY(crit_capture_);
struct ApmCaptureNonLockedState {
ApmCaptureNonLockedState()
ApmCaptureNonLockedState(bool beamformer_enabled)
: fwd_proc_format(kSampleRate16kHz),
split_rate(kSampleRate16kHz),
stream_delay_ms(0) {}
stream_delay_ms(0),
beamformer_enabled(beamformer_enabled) {}
// Only the rate and samples fields of fwd_proc_format_ are used because the
// forward processing number of channels is mutable and is tracked by the
// capture_audio_.
StreamConfig fwd_proc_format;
int split_rate;
int stream_delay_ms;
bool beamformer_enabled;
} capture_nonlocked_;
struct ApmRenderState {

2
webrtc/modules/audio_processing/echo_cancellation_impl.cc
View File

@ -174,7 +174,7 @@ int EchoCancellationImpl::ProcessCaptureAudio(AudioBuffer* audio) {
}
assert(audio->num_frames_per_band() <= 160);
assert(audio->num_channels() == apm_->num_output_channels());
assert(audio->num_channels() == apm_->num_proc_channels());
int err = AudioProcessing::kNoError;

2
webrtc/modules/audio_processing/gain_control_impl.cc
View File

@ -435,7 +435,7 @@ int GainControlImpl::ConfigureHandle(void* handle) const {
int GainControlImpl::num_handles_required() const {
// Not locked as it only relies on APM public API which is threadsafe.
return apm_->num_output_channels();
return apm_->num_proc_channels();
}
int GainControlImpl::GetHandleError(void* handle) const {

1
webrtc/modules/audio_processing/include/audio_processing.h
View File

@ -288,6 +288,7 @@ class AudioProcessing {
virtual int proc_sample_rate_hz() const = 0;
virtual int proc_split_sample_rate_hz() const = 0;
virtual int num_input_channels() const = 0;
virtual int num_proc_channels() const = 0;
virtual int num_output_channels() const = 0;
virtual int num_reverse_channels() const = 0;