zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove nonlinear beamformer API from APM

Browse Source 
This CL removes the remaining beamformer parts from the APM.

Bug: webrtc:9402
Change-Id: I9ab2795bd2813d17166ed0925125257b82d98a74
Reviewed-on: https://webrtc-review.googlesource.com/83340
Reviewed-by: Henrik Lundin <henrik.lundin@webrtc.org>
Reviewed-by: Minyue Li <minyue@webrtc.org>
Commit-Queue: Sam Zackrisson <saza@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23694}
This commit is contained in:
Sam Zackrisson
2018-06-21 10:12:24 +02:00
committed by Commit Bot
parent 7b55c73d31
commit db38972eda
23 changed files with 8 additions and 3236 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
modules/audio_processing/BUILD.gn
View File

@ -48,14 +48,6 @@ rtc_static_library("audio_processing") {
"audio_buffer.h",
"audio_processing_impl.cc",
"audio_processing_impl.h",
"beamformer/array_util.cc",
"beamformer/array_util.h",
"beamformer/complex_matrix.h",
"beamformer/covariance_matrix_generator.cc",
"beamformer/covariance_matrix_generator.h",
"beamformer/matrix.h",
"beamformer/nonlinear_beamformer.cc",
"beamformer/nonlinear_beamformer.h",
"common.h",
"echo_cancellation_impl.cc",
"echo_cancellation_impl.h",
@ -437,7 +429,6 @@ if (rtc_include_tests) {
deps = [
":audioproc_test_utils",
":click_annotate",
":nonlinear_beamformer_test",
":transient_suppression_test",
]
@ -468,12 +459,6 @@ if (rtc_include_tests) {
"agc/mock_agc.h",
"audio_buffer_unittest.cc",
"audio_frame_view_unittest.cc",
"beamformer/array_util_unittest.cc",
"beamformer/complex_matrix_unittest.cc",
"beamformer/covariance_matrix_generator_unittest.cc",
"beamformer/matrix_test_helpers.h",
"beamformer/matrix_unittest.cc",
"beamformer/mock_nonlinear_beamformer.h",
"config_unittest.cc",
"echo_cancellation_impl_unittest.cc",
"gain_controller2_unittest.cc",
@ -566,7 +551,6 @@ if (rtc_include_tests) {
"audio_processing_impl_locking_unittest.cc",
"audio_processing_impl_unittest.cc",
"audio_processing_unittest.cc",
"beamformer/nonlinear_beamformer_unittest.cc",
"echo_cancellation_bit_exact_unittest.cc",
"echo_control_mobile_unittest.cc",
"echo_detector/circular_buffer_unittest.cc",
@ -773,21 +757,6 @@ if (rtc_include_tests) {
]
}
rtc_executable("nonlinear_beamformer_test") {
testonly = true
sources = [
"beamformer/nonlinear_beamformer_test.cc",
]
deps = [
":audio_processing",
":audioproc_test_utils",
"../../common_audio:common_audio",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"../../system_wrappers:metrics_default",
]
}
if (rtc_enable_intelligibility_enhancer) {
rtc_executable("intelligibility_proc") {
testonly = true

25
modules/audio_processing/audio_processing_impl.cc
View File

@ -22,7 +22,6 @@
#include "modules/audio_processing/agc/agc_manager_direct.h"
#include "modules/audio_processing/agc2/gain_applier.h"
#include "modules/audio_processing/audio_buffer.h"
#include "modules/audio_processing/beamformer/nonlinear_beamformer.h"
#include "modules/audio_processing/common.h"
#include "modules/audio_processing/echo_cancellation_impl.h"
#include "modules/audio_processing/echo_control_mobile_impl.h"
@ -278,16 +277,13 @@ struct AudioProcessingImpl::ApmPublicSubmodules {
};
struct AudioProcessingImpl::ApmPrivateSubmodules {
ApmPrivateSubmodules(NonlinearBeamformer* beamformer,
std::unique_ptr<CustomProcessing> capture_post_processor,
ApmPrivateSubmodules(std::unique_ptr<CustomProcessing> capture_post_processor,
std::unique_ptr<CustomProcessing> render_pre_processor,
rtc::scoped_refptr<EchoDetector> echo_detector)
: beamformer(beamformer),
echo_detector(std::move(echo_detector)),
: echo_detector(std::move(echo_detector)),
capture_post_processor(std::move(capture_post_processor)),
render_pre_processor(std::move(render_pre_processor)) {}
// Accessed internally from capture or during initialization
std::unique_ptr<NonlinearBeamformer> beamformer;
std::unique_ptr<AgcManagerDirect> agc_manager;
std::unique_ptr<GainController2> gain_controller2;
std::unique_ptr<LowCutFilter> low_cut_filter;
@ -319,12 +315,6 @@ AudioProcessingBuilder& AudioProcessingBuilder::SetEchoControlFactory(
return *this;
}
AudioProcessingBuilder& AudioProcessingBuilder::SetNonlinearBeamformer(
std::unique_ptr<NonlinearBeamformer> nonlinear_beamformer) {
nonlinear_beamformer_ = std::move(nonlinear_beamformer);
return *this;
}
AudioProcessingBuilder& AudioProcessingBuilder::SetEchoDetector(
rtc::scoped_refptr<EchoDetector> echo_detector) {
echo_detector_ = std::move(echo_detector);
@ -340,7 +330,7 @@ AudioProcessing* AudioProcessingBuilder::Create(const webrtc::Config& config) {
AudioProcessingImpl* apm = new rtc::RefCountedObject<AudioProcessingImpl>(
config, std::move(capture_post_processing_),
std::move(render_pre_processing_), std::move(echo_control_factory_),
std::move(echo_detector_), nonlinear_beamformer_.release());
std::move(echo_detector_));
if (apm->Initialize() != AudioProcessing::kNoError) {
delete apm;
apm = nullptr;
@ -349,8 +339,7 @@ AudioProcessing* AudioProcessingBuilder::Create(const webrtc::Config& config) {
}
AudioProcessingImpl::AudioProcessingImpl(const webrtc::Config& config)
: AudioProcessingImpl(config, nullptr, nullptr, nullptr, nullptr, nullptr) {
}
: AudioProcessingImpl(config, nullptr, nullptr, nullptr, nullptr) {}
int AudioProcessingImpl::instance_count_ = 0;
@ -359,8 +348,7 @@ AudioProcessingImpl::AudioProcessingImpl(
std::unique_ptr<CustomProcessing> capture_post_processor,
std::unique_ptr<CustomProcessing> render_pre_processor,
std::unique_ptr<EchoControlFactory> echo_control_factory,
rtc::scoped_refptr<EchoDetector> echo_detector,
NonlinearBeamformer* beamformer)
rtc::scoped_refptr<EchoDetector> echo_detector)
: data_dumper_(
new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
capture_runtime_settings_(kRuntimeSettingQueueSize),
@ -372,8 +360,7 @@ AudioProcessingImpl::AudioProcessingImpl(
submodule_states_(!!capture_post_processor, !!render_pre_processor),
public_submodules_(new ApmPublicSubmodules()),
private_submodules_(
new ApmPrivateSubmodules(beamformer,
std::move(capture_post_processor),
new ApmPrivateSubmodules(std::move(capture_post_processor),
std::move(render_pre_processor),
std::move(echo_detector))),
constants_(config.Get<ExperimentalAgc>().startup_min_volume,

7
modules/audio_processing/audio_processing_impl.h
View File

@ -31,21 +31,18 @@ namespace webrtc {
class ApmDataDumper;
class AudioConverter;
class NonlinearBeamformer;
class AudioProcessingImpl : public AudioProcessing {
public:
// Methods forcing APM to run in a single-threaded manner.
// Acquires both the render and capture locks.
explicit AudioProcessingImpl(const webrtc::Config& config);
// AudioProcessingImpl takes ownership of capture post processor and
// beamformer.
// AudioProcessingImpl takes ownership of capture post processor.
AudioProcessingImpl(const webrtc::Config& config,
std::unique_ptr<CustomProcessing> capture_post_processor,
std::unique_ptr<CustomProcessing> render_pre_processor,
std::unique_ptr<EchoControlFactory> echo_control_factory,
rtc::scoped_refptr<EchoDetector> echo_detector,
NonlinearBeamformer* beamformer);
rtc::scoped_refptr<EchoDetector> echo_detector);
~AudioProcessingImpl() override;
int Initialize() override;
int Initialize(int capture_input_sample_rate_hz,

119
modules/audio_processing/beamformer/array_util.cc
View File

@ -1,119 +0,0 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/audio_processing/beamformer/array_util.h"
#include <algorithm>
#include <limits>
#include "rtc_base/checks.h"
namespace webrtc {
namespace {
const float kMaxDotProduct = 1e-6f;
} // namespace
float GetMinimumSpacing(const std::vector<Point>& array_geometry) {
RTC_CHECK_GT(array_geometry.size(), 1);
float mic_spacing = std::numeric_limits<float>::max();
for (size_t i = 0; i < (array_geometry.size() - 1); ++i) {
for (size_t j = i + 1; j < array_geometry.size(); ++j) {
mic_spacing =
std::min(mic_spacing, Distance(array_geometry[i], array_geometry[j]));
}
}
return mic_spacing;
}
Point PairDirection(const Point& a, const Point& b) {
return {b.x() - a.x(), b.y() - a.y(), b.z() - a.z()};
}
float DotProduct(const Point& a, const Point& b) {
return a.x() * b.x() + a.y() * b.y() + a.z() * b.z();
}
Point CrossProduct(const Point& a, const Point& b) {
return {a.y() * b.z() - a.z() * b.y(), a.z() * b.x() - a.x() * b.z(),
a.x() * b.y() - a.y() * b.x()};
}
bool AreParallel(const Point& a, const Point& b) {
Point cross_product = CrossProduct(a, b);
return DotProduct(cross_product, cross_product) < kMaxDotProduct;
}
bool ArePerpendicular(const Point& a, const Point& b) {
return std::abs(DotProduct(a, b)) < kMaxDotProduct;
}
absl::optional<Point> GetDirectionIfLinear(
const std::vector<Point>& array_geometry) {
RTC_DCHECK_GT(array_geometry.size(), 1);
const Point first_pair_direction =
PairDirection(array_geometry[0], array_geometry[1]);
for (size_t i = 2u; i < array_geometry.size(); ++i) {
const Point pair_direction =
PairDirection(array_geometry[i - 1], array_geometry[i]);
if (!AreParallel(first_pair_direction, pair_direction)) {
return absl::nullopt;
}
}
return first_pair_direction;
}
absl::optional<Point> GetNormalIfPlanar(
const std::vector<Point>& array_geometry) {
RTC_DCHECK_GT(array_geometry.size(), 1);
const Point first_pair_direction =
PairDirection(array_geometry[0], array_geometry[1]);
Point pair_direction(0.f, 0.f, 0.f);
size_t i = 2u;
bool is_linear = true;
for (; i < array_geometry.size() && is_linear; ++i) {
pair_direction = PairDirection(array_geometry[i - 1], array_geometry[i]);
if (!AreParallel(first_pair_direction, pair_direction)) {
is_linear = false;
}
}
if (is_linear) {
return absl::nullopt;
}
const Point normal_direction =
CrossProduct(first_pair_direction, pair_direction);
for (; i < array_geometry.size(); ++i) {
pair_direction = PairDirection(array_geometry[i - 1], array_geometry[i]);
if (!ArePerpendicular(normal_direction, pair_direction)) {
return absl::nullopt;
}
}
return normal_direction;
}
absl::optional<Point> GetArrayNormalIfExists(
const std::vector<Point>& array_geometry) {
const absl::optional<Point> direction = GetDirectionIfLinear(array_geometry);
if (direction) {
return Point(direction->y(), -direction->x(), 0.f);
}
const absl::optional<Point> normal = GetNormalIfPlanar(array_geometry);
if (normal && normal->z() < kMaxDotProduct) {
return normal;
}
return absl::nullopt;
}
Point AzimuthToPoint(float azimuth) {
return Point(std::cos(azimuth), std::sin(azimuth), 0.f);
}
} // namespace webrtc

117
modules/audio_processing/beamformer/array_util.h
View File

@ -1,117 +0,0 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_ARRAY_UTIL_H_
#define MODULES_AUDIO_PROCESSING_BEAMFORMER_ARRAY_UTIL_H_
#include <cmath>
#include <vector>
#include "absl/types/optional.h"
namespace webrtc {
// Coordinates in meters. The convention used is:
// x: the horizontal dimension, with positive to the right from the camera's
// perspective.
// y: the depth dimension, with positive forward from the camera's
// perspective.
// z: the vertical dimension, with positive upwards.
template <typename T>
struct CartesianPoint {
CartesianPoint() {
c[0] = 0;
c[1] = 0;
c[2] = 0;
}
CartesianPoint(T x, T y, T z) {
c[0] = x;
c[1] = y;
c[2] = z;
}
T x() const { return c[0]; }
T y() const { return c[1]; }
T z() const { return c[2]; }
T c[3];
};
using Point = CartesianPoint<float>;
// Calculates the direction from a to b.
Point PairDirection(const Point& a, const Point& b);
float DotProduct(const Point& a, const Point& b);
Point CrossProduct(const Point& a, const Point& b);
bool AreParallel(const Point& a, const Point& b);
bool ArePerpendicular(const Point& a, const Point& b);
// Returns the minimum distance between any two Points in the given
// |array_geometry|.
float GetMinimumSpacing(const std::vector<Point>& array_geometry);
// If the given array geometry is linear it returns the direction without
// normalizing.
absl::optional<Point> GetDirectionIfLinear(
const std::vector<Point>& array_geometry);
// If the given array geometry is planar it returns the normal without
// normalizing.
absl::optional<Point> GetNormalIfPlanar(
const std::vector<Point>& array_geometry);
// Returns the normal of an array if it has one and it is in the xy-plane.
absl::optional<Point> GetArrayNormalIfExists(
const std::vector<Point>& array_geometry);
// The resulting Point will be in the xy-plane.
Point AzimuthToPoint(float azimuth);
template <typename T>
float Distance(CartesianPoint<T> a, CartesianPoint<T> b) {
return std::sqrt((a.x() - b.x()) * (a.x() - b.x()) +
(a.y() - b.y()) * (a.y() - b.y()) +
(a.z() - b.z()) * (a.z() - b.z()));
}
// The convention used:
// azimuth: zero is to the right from the camera's perspective, with positive
// angles in radians counter-clockwise.
// elevation: zero is horizontal, with positive angles in radians upwards.
// radius: distance from the camera in meters.
template <typename T>
struct SphericalPoint {
SphericalPoint(T azimuth, T elevation, T radius) {
s[0] = azimuth;
s[1] = elevation;
s[2] = radius;
}
T azimuth() const { return s[0]; }
T elevation() const { return s[1]; }
T distance() const { return s[2]; }
T s[3];
};
using SphericalPointf = SphericalPoint<float>;
// Helper functions to transform degrees to radians and the inverse.
template <typename T>
T DegreesToRadians(T angle_degrees) {
return M_PI * angle_degrees / 180;
}
template <typename T>
T RadiansToDegrees(T angle_radians) {
return 180 * angle_radians / M_PI;
}
} // namespace webrtc
#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_ARRAY_UTIL_H_

185
modules/audio_processing/beamformer/array_util_unittest.cc
View File

@ -1,185 +0,0 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
// MSVC++ requires this to be set before any other includes to get M_PI.
#define _USE_MATH_DEFINES
#include "modules/audio_processing/beamformer/array_util.h"
#include <math.h>
#include <vector>
#include "test/gtest.h"
namespace webrtc {
bool operator==(const Point& lhs, const Point& rhs) {
return lhs.x() == rhs.x() && lhs.y() == rhs.y() && lhs.z() == rhs.z();
}
TEST(ArrayUtilTest, PairDirection) {
EXPECT_EQ(Point(1.f, 2.f, 3.f),
PairDirection(Point(0.f, 0.f, 0.f), Point(1.f, 2.f, 3.f)));
EXPECT_EQ(Point(-1.f, -2.f, -3.f),
PairDirection(Point(1.f, 2.f, 3.f), Point(0.f, 0.f, 0.f)));
EXPECT_EQ(Point(0.f, 0.f, 0.f),
PairDirection(Point(1.f, 0.f, 0.f), Point(1.f, 0.f, 0.f)));
EXPECT_EQ(Point(-1.f, 2.f, 0.f),
PairDirection(Point(1.f, 0.f, 0.f), Point(0.f, 2.f, 0.f)));
EXPECT_EQ(Point(-4.f, 4.f, -4.f),
PairDirection(Point(1.f, -2.f, 3.f), Point(-3.f, 2.f, -1.f)));
}
TEST(ArrayUtilTest, DotProduct) {
EXPECT_FLOAT_EQ(0.f, DotProduct(Point(0.f, 0.f, 0.f), Point(1.f, 2.f, 3.f)));
EXPECT_FLOAT_EQ(0.f, DotProduct(Point(1.f, 0.f, 2.f), Point(0.f, 3.f, 0.f)));
EXPECT_FLOAT_EQ(0.f, DotProduct(Point(1.f, 1.f, 0.f), Point(1.f, -1.f, 0.f)));
EXPECT_FLOAT_EQ(2.f, DotProduct(Point(1.f, 0.f, 0.f), Point(2.f, 0.f, 0.f)));
EXPECT_FLOAT_EQ(-6.f,
DotProduct(Point(-2.f, 0.f, 0.f), Point(3.f, 0.f, 0.f)));
EXPECT_FLOAT_EQ(-10.f,
DotProduct(Point(1.f, -2.f, 3.f), Point(-3.f, 2.f, -1.f)));
}
TEST(ArrayUtilTest, CrossProduct) {
EXPECT_EQ(Point(0.f, 0.f, 0.f),
CrossProduct(Point(0.f, 0.f, 0.f), Point(1.f, 2.f, 3.f)));
EXPECT_EQ(Point(0.f, 0.f, 1.f),
CrossProduct(Point(1.f, 0.f, 0.f), Point(0.f, 1.f, 0.f)));
EXPECT_EQ(Point(1.f, 0.f, 0.f),
CrossProduct(Point(0.f, 1.f, 0.f), Point(0.f, 0.f, 1.f)));
EXPECT_EQ(Point(0.f, -1.f, 0.f),
CrossProduct(Point(1.f, 0.f, 0.f), Point(0.f, 0.f, 1.f)));
EXPECT_EQ(Point(-4.f, -8.f, -4.f),
CrossProduct(Point(1.f, -2.f, 3.f), Point(-3.f, 2.f, -1.f)));
}
TEST(ArrayUtilTest, AreParallel) {
EXPECT_TRUE(AreParallel(Point(0.f, 0.f, 0.f), Point(1.f, 2.f, 3.f)));
EXPECT_FALSE(AreParallel(Point(1.f, 0.f, 2.f), Point(0.f, 3.f, 0.f)));
EXPECT_FALSE(AreParallel(Point(1.f, 2.f, 0.f), Point(1.f, -0.5f, 0.f)));
EXPECT_FALSE(AreParallel(Point(1.f, -2.f, 3.f), Point(-3.f, 2.f, -1.f)));
EXPECT_TRUE(AreParallel(Point(1.f, 0.f, 0.f), Point(2.f, 0.f, 0.f)));
EXPECT_TRUE(AreParallel(Point(1.f, 2.f, 3.f), Point(-2.f, -4.f, -6.f)));
}
TEST(ArrayUtilTest, ArePerpendicular) {
EXPECT_TRUE(ArePerpendicular(Point(0.f, 0.f, 0.f), Point(1.f, 2.f, 3.f)));
EXPECT_TRUE(ArePerpendicular(Point(1.f, 0.f, 2.f), Point(0.f, 3.f, 0.f)));
EXPECT_TRUE(ArePerpendicular(Point(1.f, 2.f, 0.f), Point(1.f, -0.5f, 0.f)));
EXPECT_FALSE(ArePerpendicular(Point(1.f, -2.f, 3.f), Point(-3.f, 2.f, -1.f)));
EXPECT_FALSE(ArePerpendicular(Point(1.f, 0.f, 0.f), Point(2.f, 0.f, 0.f)));
EXPECT_FALSE(ArePerpendicular(Point(1.f, 2.f, 3.f), Point(-2.f, -4.f, -6.f)));
}
TEST(ArrayUtilTest, GetMinimumSpacing) {
std::vector<Point> geometry;
geometry.push_back(Point(0.f, 0.f, 0.f));
geometry.push_back(Point(0.1f, 0.f, 0.f));
EXPECT_FLOAT_EQ(0.1f, GetMinimumSpacing(geometry));
geometry.push_back(Point(0.f, 0.05f, 0.f));
EXPECT_FLOAT_EQ(0.05f, GetMinimumSpacing(geometry));
geometry.push_back(Point(0.f, 0.f, 0.02f));
EXPECT_FLOAT_EQ(0.02f, GetMinimumSpacing(geometry));
geometry.push_back(Point(-0.003f, -0.004f, 0.02f));
EXPECT_FLOAT_EQ(0.005f, GetMinimumSpacing(geometry));
}
TEST(ArrayUtilTest, GetDirectionIfLinear) {
std::vector<Point> geometry;
geometry.push_back(Point(0.f, 0.f, 0.f));
geometry.push_back(Point(0.1f, 0.f, 0.f));
EXPECT_TRUE(
AreParallel(Point(1.f, 0.f, 0.f), *GetDirectionIfLinear(geometry)));
geometry.push_back(Point(0.15f, 0.f, 0.f));
EXPECT_TRUE(
AreParallel(Point(1.f, 0.f, 0.f), *GetDirectionIfLinear(geometry)));
geometry.push_back(Point(-0.2f, 0.f, 0.f));
EXPECT_TRUE(
AreParallel(Point(1.f, 0.f, 0.f), *GetDirectionIfLinear(geometry)));
geometry.push_back(Point(0.05f, 0.f, 0.f));
EXPECT_TRUE(
AreParallel(Point(1.f, 0.f, 0.f), *GetDirectionIfLinear(geometry)));
geometry.push_back(Point(0.1f, 0.1f, 0.f));
EXPECT_FALSE(GetDirectionIfLinear(geometry));
geometry.push_back(Point(0.f, 0.f, -0.2f));
EXPECT_FALSE(GetDirectionIfLinear(geometry));
}
TEST(ArrayUtilTest, GetNormalIfPlanar) {
std::vector<Point> geometry;
geometry.push_back(Point(0.f, 0.f, 0.f));
geometry.push_back(Point(0.1f, 0.f, 0.f));
EXPECT_FALSE(GetNormalIfPlanar(geometry));
geometry.push_back(Point(0.15f, 0.f, 0.f));
EXPECT_FALSE(GetNormalIfPlanar(geometry));
geometry.push_back(Point(0.1f, 0.2f, 0.f));
EXPECT_TRUE(AreParallel(Point(0.f, 0.f, 1.f), *GetNormalIfPlanar(geometry)));
geometry.push_back(Point(0.f, -0.15f, 0.f));
EXPECT_TRUE(AreParallel(Point(0.f, 0.f, 1.f), *GetNormalIfPlanar(geometry)));
geometry.push_back(Point(0.f, 0.1f, 0.2f));
EXPECT_FALSE(GetNormalIfPlanar(geometry));
geometry.push_back(Point(0.f, 0.f, -0.15f));
EXPECT_FALSE(GetNormalIfPlanar(geometry));
geometry.push_back(Point(0.1f, 0.2f, 0.f));
EXPECT_FALSE(GetNormalIfPlanar(geometry));
}
TEST(ArrayUtilTest, GetArrayNormalIfExists) {
std::vector<Point> geometry;
geometry.push_back(Point(0.f, 0.f, 0.f));
geometry.push_back(Point(0.1f, 0.f, 0.f));
EXPECT_TRUE(
AreParallel(Point(0.f, 1.f, 0.f), *GetArrayNormalIfExists(geometry)));
geometry.push_back(Point(0.15f, 0.f, 0.f));
EXPECT_TRUE(
AreParallel(Point(0.f, 1.f, 0.f), *GetArrayNormalIfExists(geometry)));
geometry.push_back(Point(0.1f, 0.f, 0.2f));
EXPECT_TRUE(
AreParallel(Point(0.f, 1.f, 0.f), *GetArrayNormalIfExists(geometry)));
geometry.push_back(Point(0.f, 0.f, -0.1f));
EXPECT_TRUE(
AreParallel(Point(0.f, 1.f, 0.f), *GetArrayNormalIfExists(geometry)));
geometry.push_back(Point(0.1f, 0.2f, 0.3f));
EXPECT_FALSE(GetArrayNormalIfExists(geometry));
geometry.push_back(Point(0.f, -0.1f, 0.f));
EXPECT_FALSE(GetArrayNormalIfExists(geometry));
geometry.push_back(Point(1.f, 0.f, -0.2f));
EXPECT_FALSE(GetArrayNormalIfExists(geometry));
}
TEST(ArrayUtilTest, DegreesToRadians) {
EXPECT_FLOAT_EQ(0.f, DegreesToRadians(0.f));
EXPECT_FLOAT_EQ(static_cast<float>(M_PI) / 6.f, DegreesToRadians(30.f));
EXPECT_FLOAT_EQ(-static_cast<float>(M_PI) / 4.f, DegreesToRadians(-45.f));
EXPECT_FLOAT_EQ(static_cast<float>(M_PI) / 3.f, DegreesToRadians(60.f));
EXPECT_FLOAT_EQ(-static_cast<float>(M_PI) / 2.f, DegreesToRadians(-90.f));
EXPECT_FLOAT_EQ(2.f * static_cast<float>(M_PI) / 3.f,
DegreesToRadians(120.f));
EXPECT_FLOAT_EQ(-3.f * static_cast<float>(M_PI) / 4.f,
DegreesToRadians(-135.f));
EXPECT_FLOAT_EQ(5.f * static_cast<float>(M_PI) / 6.f,
DegreesToRadians(150.f));
EXPECT_FLOAT_EQ(-static_cast<float>(M_PI), DegreesToRadians(-180.f));
}
TEST(ArrayUtilTest, RadiansToDegrees) {
EXPECT_FLOAT_EQ(0.f, RadiansToDegrees(0.f));
EXPECT_FLOAT_EQ(30.f, RadiansToDegrees(M_PI / 6.f));
EXPECT_FLOAT_EQ(-45.f, RadiansToDegrees(-M_PI / 4.f));
EXPECT_FLOAT_EQ(60.f, RadiansToDegrees(M_PI / 3.f));
EXPECT_FLOAT_EQ(-90.f, RadiansToDegrees(-M_PI / 2.f));
EXPECT_FLOAT_EQ(120.f, RadiansToDegrees(2.f * M_PI / 3.f));
EXPECT_FLOAT_EQ(-135.f, RadiansToDegrees(-3.f * M_PI / 4.f));
EXPECT_FLOAT_EQ(150.f, RadiansToDegrees(5.f * M_PI / 6.f));
EXPECT_FLOAT_EQ(-180.f, RadiansToDegrees(-M_PI));
}
} // namespace webrtc

96
modules/audio_processing/beamformer/complex_matrix.h
View File

@ -1,96 +0,0 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_COMPLEX_MATRIX_H_
#define MODULES_AUDIO_PROCESSING_BEAMFORMER_COMPLEX_MATRIX_H_
#include <complex>
#include "modules/audio_processing/beamformer/matrix.h"
#include "rtc_base/checks.h"
namespace webrtc {
using std::complex;
// An extension of Matrix for operations that only work on a complex type.
template <typename T>
class ComplexMatrix : public Matrix<complex<T> > {
public:
ComplexMatrix() : Matrix<complex<T> >() {}
ComplexMatrix(size_t num_rows, size_t num_columns)
: Matrix<complex<T> >(num_rows, num_columns) {}
ComplexMatrix(const complex<T>* data, size_t num_rows, size_t num_columns)
: Matrix<complex<T> >(data, num_rows, num_columns) {}
// Complex Matrix operations.
ComplexMatrix& PointwiseConjugate() {
complex<T>* const data = this->data();
size_t size = this->num_rows() * this->num_columns();
for (size_t i = 0; i < size; ++i) {
data[i] = conj(data[i]);
}
return *this;
}
ComplexMatrix& PointwiseConjugate(const ComplexMatrix& operand) {
this->CopyFrom(operand);
return PointwiseConjugate();
}
ComplexMatrix& ConjugateTranspose() {
this->CopyDataToScratch();
size_t num_rows = this->num_rows();
this->SetNumRows(this->num_columns());
this->SetNumColumns(num_rows);
this->Resize();
return ConjugateTranspose(this->scratch_elements());
}
ComplexMatrix& ConjugateTranspose(const ComplexMatrix& operand) {
RTC_CHECK_EQ(operand.num_rows(), this->num_columns());
RTC_CHECK_EQ(operand.num_columns(), this->num_rows());
return ConjugateTranspose(operand.elements());
}
ComplexMatrix& ZeroImag() {
complex<T>* const data = this->data();
size_t size = this->num_rows() * this->num_columns();
for (size_t i = 0; i < size; ++i) {
data[i] = complex<T>(data[i].real(), 0);
}
return *this;
}
ComplexMatrix& ZeroImag(const ComplexMatrix& operand) {
this->CopyFrom(operand);
return ZeroImag();
}
private:
ComplexMatrix& ConjugateTranspose(const complex<T>* const* src) {
complex<T>* const* elements = this->elements();
for (size_t i = 0; i < this->num_rows(); ++i) {
for (size_t j = 0; j < this->num_columns(); ++j) {
elements[i][j] = conj(src[j][i]);
}
}
return *this;
}
};
} // namespace webrtc
#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_COMPLEX_MATRIX_H_

102
modules/audio_processing/beamformer/complex_matrix_unittest.cc
View File

@ -1,102 +0,0 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/audio_processing/beamformer/complex_matrix.h"
#include "modules/audio_processing/beamformer/matrix_test_helpers.h"
#include "test/gtest.h"
namespace webrtc {
TEST(ComplexMatrixTest, TestPointwiseConjugate) {
const int kNumRows = 2;
const int kNumCols = 4;
const complex<float> kValuesInitial[kNumRows][kNumCols] = {
{complex<float>(1.1f, 1.1f), complex<float>(2.2f, -2.2f),
complex<float>(3.3f, 3.3f), complex<float>(4.4f, -4.4f)},
{complex<float>(5.5f, 5.5f), complex<float>(6.6f, -6.6f),
complex<float>(7.7f, 7.7f), complex<float>(8.8f, -8.8f)}};
const complex<float> kValuesExpected[kNumRows][kNumCols] = {
{complex<float>(1.1f, -1.1f), complex<float>(2.2f, 2.2f),
complex<float>(3.3f, -3.3f), complex<float>(4.4f, 4.4f)},
{complex<float>(5.5f, -5.5f), complex<float>(6.6f, 6.6f),
complex<float>(7.7f, -7.7f), complex<float>(8.8f, 8.8f)}};
ComplexMatrix<float> initial_mat(*kValuesInitial, kNumRows, kNumCols);
ComplexMatrix<float> expected_result(*kValuesExpected, kNumRows, kNumCols);
ComplexMatrix<float> actual_result(kNumRows, kNumCols);
actual_result.PointwiseConjugate(initial_mat);
MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(expected_result,
actual_result);
initial_mat.PointwiseConjugate();
MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(initial_mat,
actual_result);
}
TEST(ComplexMatrixTest, TestConjugateTranspose) {
const int kNumInitialRows = 2;
const int kNumInitialCols = 4;
const int kNumResultRows = 4;
const int kNumResultCols = 2;
const complex<float> kValuesInitial[kNumInitialRows][kNumInitialCols] = {
{complex<float>(1.1f, 1.1f), complex<float>(2.2f, 2.2f),
complex<float>(3.3f, 3.3f), complex<float>(4.4f, 4.4f)},
{complex<float>(5.5f, 5.5f), complex<float>(6.6f, 6.6f),
complex<float>(7.7f, 7.7f), complex<float>(8.8f, 8.8f)}};
const complex<float> kValuesExpected[kNumResultRows][kNumResultCols] = {
{complex<float>(1.1f, -1.1f), complex<float>(5.5f, -5.5f)},
{complex<float>(2.2f, -2.2f), complex<float>(6.6f, -6.6f)},
{complex<float>(3.3f, -3.3f), complex<float>(7.7f, -7.7f)},
{complex<float>(4.4f, -4.4f), complex<float>(8.8f, -8.8f)}};
ComplexMatrix<float> initial_mat(*kValuesInitial, kNumInitialRows,
kNumInitialCols);
ComplexMatrix<float> expected_result(*kValuesExpected, kNumResultRows,
kNumResultCols);
ComplexMatrix<float> actual_result(kNumResultRows, kNumResultCols);
actual_result.ConjugateTranspose(initial_mat);
MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(expected_result,
actual_result);
initial_mat.ConjugateTranspose();
MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(initial_mat,
actual_result);
}
TEST(ComplexMatrixTest, TestZeroImag) {
const int kNumRows = 2;
const int kNumCols = 2;
const complex<float> kValuesInitial[kNumRows][kNumCols] = {
{complex<float>(1.1f, 1.1f), complex<float>(2.2f, 2.2f)},
{complex<float>(3.3f, 3.3f), complex<float>(4.4f, 4.4f)}};
const complex<float> kValuesExpected[kNumRows][kNumCols] = {
{complex<float>(1.1f, 0.f), complex<float>(2.2f, 0.f)},
{complex<float>(3.3f, 0.f), complex<float>(4.4f, 0.f)}};
ComplexMatrix<float> initial_mat(*kValuesInitial, kNumRows, kNumCols);
ComplexMatrix<float> expected_result(*kValuesExpected, kNumRows, kNumCols);
ComplexMatrix<float> actual_result;
actual_result.ZeroImag(initial_mat);
MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(expected_result,
actual_result);
initial_mat.ZeroImag();
MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(initial_mat,
actual_result);
}
} // namespace webrtc

109
modules/audio_processing/beamformer/covariance_matrix_generator.cc
View File

@ -1,109 +0,0 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#define _USE_MATH_DEFINES
#include "modules/audio_processing/beamformer/covariance_matrix_generator.h"
#include <cmath>
namespace webrtc {
namespace {
float BesselJ0(float x) {
#ifdef WEBRTC_WIN
return _j0(x);
#else
return j0(x);
#endif
}
// Calculates the Euclidean norm for a row vector.
float Norm(const ComplexMatrix<float>& x) {
RTC_CHECK_EQ(1, x.num_rows());
const size_t length = x.num_columns();
const complex<float>* elems = x.elements()[0];
float result = 0.f;
for (size_t i = 0u; i < length; ++i) {
result += std::norm(elems[i]);
}
return std::sqrt(result);
}
} // namespace
void CovarianceMatrixGenerator::UniformCovarianceMatrix(
float wave_number,
const std::vector<Point>& geometry,
ComplexMatrix<float>* mat) {
RTC_CHECK_EQ(geometry.size(), mat->num_rows());
RTC_CHECK_EQ(geometry.size(), mat->num_columns());
complex<float>* const* mat_els = mat->elements();
for (size_t i = 0; i < geometry.size(); ++i) {
for (size_t j = 0; j < geometry.size(); ++j) {
if (wave_number > 0.f) {
mat_els[i][j] =
BesselJ0(wave_number * Distance(geometry[i], geometry[j]));
} else {
mat_els[i][j] = i == j ? 1.f : 0.f;
}
}
}
}
void CovarianceMatrixGenerator::AngledCovarianceMatrix(
float sound_speed,
float angle,
size_t frequency_bin,
size_t fft_size,
size_t num_freq_bins,
int sample_rate,
const std::vector<Point>& geometry,
ComplexMatrix<float>* mat) {
RTC_CHECK_EQ(geometry.size(), mat->num_rows());
RTC_CHECK_EQ(geometry.size(), mat->num_columns());
ComplexMatrix<float> interf_cov_vector(1, geometry.size());
ComplexMatrix<float> interf_cov_vector_transposed(geometry.size(), 1);
PhaseAlignmentMasks(frequency_bin, fft_size, sample_rate, sound_speed,
geometry, angle, &interf_cov_vector);
interf_cov_vector.Scale(1.f / Norm(interf_cov_vector));
interf_cov_vector_transposed.Transpose(interf_cov_vector);
interf_cov_vector.PointwiseConjugate();
mat->Multiply(interf_cov_vector_transposed, interf_cov_vector);
}
void CovarianceMatrixGenerator::PhaseAlignmentMasks(
size_t frequency_bin,
size_t fft_size,
int sample_rate,
float sound_speed,
const std::vector<Point>& geometry,
float angle,
ComplexMatrix<float>* mat) {
RTC_CHECK_EQ(1, mat->num_rows());
RTC_CHECK_EQ(geometry.size(), mat->num_columns());
float freq_in_hertz =
(static_cast<float>(frequency_bin) / fft_size) * sample_rate;
complex<float>* const* mat_els = mat->elements();
for (size_t c_ix = 0; c_ix < geometry.size(); ++c_ix) {
float distance = std::cos(angle) * geometry[c_ix].x() +
std::sin(angle) * geometry[c_ix].y();
float phase_shift = -2.f * M_PI * distance * freq_in_hertz / sound_speed;
// Euler's formula for mat[0][c_ix] = e^(j * phase_shift).
mat_els[0][c_ix] = complex<float>(cos(phase_shift), sin(phase_shift));
}
}
} // namespace webrtc

54
modules/audio_processing/beamformer/covariance_matrix_generator.h
View File

@ -1,54 +0,0 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_COVARIANCE_MATRIX_GENERATOR_H_
#define MODULES_AUDIO_PROCESSING_BEAMFORMER_COVARIANCE_MATRIX_GENERATOR_H_
#include "modules/audio_processing/beamformer/array_util.h"
#include "modules/audio_processing/beamformer/complex_matrix.h"
namespace webrtc {
// Helper class for Beamformer in charge of generating covariance matrices. For
// each function, the passed-in ComplexMatrix is expected to be of size
// |num_input_channels| x |num_input_channels|.
class CovarianceMatrixGenerator {
public:
// A uniform covariance matrix with a gap at the target location. WARNING:
// The target angle is assumed to be 0.
static void UniformCovarianceMatrix(float wave_number,
const std::vector<Point>& geometry,
ComplexMatrix<float>* mat);
// The covariance matrix of a source at the given angle.
static void AngledCovarianceMatrix(float sound_speed,
float angle,
size_t frequency_bin,
size_t fft_size,
size_t num_freq_bins,
int sample_rate,
const std::vector<Point>& geometry,
ComplexMatrix<float>* mat);
// Calculates phase shifts that, when applied to a multichannel signal and
// added together, cause constructive interferernce for sources located at
// the given angle.
static void PhaseAlignmentMasks(size_t frequency_bin,
size_t fft_size,
int sample_rate,
float sound_speed,
const std::vector<Point>& geometry,
float angle,
ComplexMatrix<float>* mat);
};
} // namespace webrtc
#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_BF_HELPERS_H_

217
modules/audio_processing/beamformer/covariance_matrix_generator_unittest.cc
View File

@ -1,217 +0,0 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#define _USE_MATH_DEFINES
#include "modules/audio_processing/beamformer/covariance_matrix_generator.h"
#include <cmath>
#include "modules/audio_processing/beamformer/matrix_test_helpers.h"
#include "test/gtest.h"
namespace webrtc {
using std::complex;
TEST(CovarianceMatrixGeneratorTest, TestUniformCovarianceMatrix2Mics) {
const float kWaveNumber = 0.5775f;
const int kNumberMics = 2;
const float kMicSpacing = 0.05f;
const float kTolerance = 0.0001f;
std::vector<Point> geometry;
float first_mic = (kNumberMics - 1) * kMicSpacing / 2.f;
for (int i = 0; i < kNumberMics; ++i) {
geometry.push_back(Point(i * kMicSpacing - first_mic, 0.f, 0.f));
}
ComplexMatrix<float> actual_covariance_matrix(kNumberMics, kNumberMics);
CovarianceMatrixGenerator::UniformCovarianceMatrix(kWaveNumber, geometry,
&actual_covariance_matrix);
complex<float>* const* actual_els = actual_covariance_matrix.elements();
EXPECT_NEAR(actual_els[0][0].real(), 1.f, kTolerance);
EXPECT_NEAR(actual_els[0][1].real(), 0.9998f, kTolerance);
EXPECT_NEAR(actual_els[1][0].real(), 0.9998f, kTolerance);
EXPECT_NEAR(actual_els[1][1].real(), 1.f, kTolerance);
EXPECT_NEAR(actual_els[0][0].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[0][1].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[1][0].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[1][1].imag(), 0.f, kTolerance);
}
TEST(CovarianceMatrixGeneratorTest, TestUniformCovarianceMatrix3Mics) {
const float kWaveNumber = 10.3861f;
const int kNumberMics = 3;
const float kMicSpacing = 0.04f;
const float kTolerance = 0.0001f;
std::vector<Point> geometry;
float first_mic = (kNumberMics - 1) * kMicSpacing / 2.f;
for (int i = 0; i < kNumberMics; ++i) {
geometry.push_back(Point(i * kMicSpacing - first_mic, 0.f, 0.f));
}
ComplexMatrix<float> actual_covariance_matrix(kNumberMics, kNumberMics);
CovarianceMatrixGenerator::UniformCovarianceMatrix(kWaveNumber, geometry,
&actual_covariance_matrix);
complex<float>* const* actual_els = actual_covariance_matrix.elements();
EXPECT_NEAR(actual_els[0][0].real(), 1.f, kTolerance);
EXPECT_NEAR(actual_els[0][1].real(), 0.9573f, kTolerance);
EXPECT_NEAR(actual_els[0][2].real(), 0.8347f, kTolerance);
EXPECT_NEAR(actual_els[1][0].real(), 0.9573f, kTolerance);
EXPECT_NEAR(actual_els[1][1].real(), 1.f, kTolerance);
EXPECT_NEAR(actual_els[1][2].real(), 0.9573f, kTolerance);
EXPECT_NEAR(actual_els[2][0].real(), 0.8347f, kTolerance);
EXPECT_NEAR(actual_els[2][1].real(), 0.9573f, kTolerance);
EXPECT_NEAR(actual_els[2][2].real(), 1.f, kTolerance);
EXPECT_NEAR(actual_els[0][0].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[0][1].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[0][2].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[1][0].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[1][1].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[1][2].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[2][0].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[2][1].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[2][2].imag(), 0.f, kTolerance);
}
TEST(CovarianceMatrixGeneratorTest, TestUniformCovarianceMatrix3DArray) {
const float kWaveNumber = 1.2345f;
const int kNumberMics = 4;
const float kTolerance = 0.0001f;
std::vector<Point> geometry;
geometry.push_back(Point(-0.025f, -0.05f, -0.075f));
geometry.push_back(Point(0.075f, -0.05f, -0.075f));
geometry.push_back(Point(-0.025f, 0.15f, -0.075f));
geometry.push_back(Point(-0.025f, -0.05f, 0.225f));
ComplexMatrix<float> actual_covariance_matrix(kNumberMics, kNumberMics);
CovarianceMatrixGenerator::UniformCovarianceMatrix(kWaveNumber, geometry,
&actual_covariance_matrix);
complex<float>* const* actual_els = actual_covariance_matrix.elements();
EXPECT_NEAR(actual_els[0][0].real(), 1.f, kTolerance);
EXPECT_NEAR(actual_els[0][1].real(), 0.9962f, kTolerance);
EXPECT_NEAR(actual_els[0][2].real(), 0.9848f, kTolerance);
EXPECT_NEAR(actual_els[0][3].real(), 0.9660f, kTolerance);
EXPECT_NEAR(actual_els[1][0].real(), 0.9962f, kTolerance);
EXPECT_NEAR(actual_els[1][1].real(), 1.f, kTolerance);
EXPECT_NEAR(actual_els[1][2].real(), 0.9810f, kTolerance);
EXPECT_NEAR(actual_els[1][3].real(), 0.9623f, kTolerance);
EXPECT_NEAR(actual_els[2][0].real(), 0.9848f, kTolerance);
EXPECT_NEAR(actual_els[2][1].real(), 0.9810f, kTolerance);
EXPECT_NEAR(actual_els[2][2].real(), 1.f, kTolerance);
EXPECT_NEAR(actual_els[2][3].real(), 0.9511f, kTolerance);
EXPECT_NEAR(actual_els[3][0].real(), 0.9660f, kTolerance);
EXPECT_NEAR(actual_els[3][1].real(), 0.9623f, kTolerance);
EXPECT_NEAR(actual_els[3][2].real(), 0.9511f, kTolerance);
EXPECT_NEAR(actual_els[3][3].real(), 1.f, kTolerance);
EXPECT_NEAR(actual_els[0][0].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[0][1].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[0][2].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[0][3].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[1][0].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[1][1].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[1][2].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[1][3].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[2][0].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[2][1].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[2][2].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[2][3].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[3][0].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[3][1].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[3][2].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[3][3].imag(), 0.f, kTolerance);
}
TEST(CovarianceMatrixGeneratorTest, TestAngledCovarianceMatrix2Mics) {
const float kSpeedOfSound = 340;
const float kAngle = static_cast<float>(M_PI) / 4.f;
const float kFrequencyBin = 6;
const float kFftSize = 512;
const int kNumberFrequencyBins = 257;
const int kSampleRate = 16000;
const int kNumberMics = 2;
const float kMicSpacing = 0.04f;
const float kTolerance = 0.0001f;
std::vector<Point> geometry;
float first_mic = (kNumberMics - 1) * kMicSpacing / 2.f;
for (int i = 0; i < kNumberMics; ++i) {
geometry.push_back(Point(i * kMicSpacing - first_mic, 0.f, 0.f));
}
ComplexMatrix<float> actual_covariance_matrix(kNumberMics, kNumberMics);
CovarianceMatrixGenerator::AngledCovarianceMatrix(
kSpeedOfSound, kAngle, kFrequencyBin, kFftSize, kNumberFrequencyBins,
kSampleRate, geometry, &actual_covariance_matrix);
complex<float>* const* actual_els = actual_covariance_matrix.elements();
EXPECT_NEAR(actual_els[0][0].real(), 0.5f, kTolerance);
EXPECT_NEAR(actual_els[0][1].real(), 0.4976f, kTolerance);
EXPECT_NEAR(actual_els[1][0].real(), 0.4976f, kTolerance);
EXPECT_NEAR(actual_els[1][1].real(), 0.5f, kTolerance);
EXPECT_NEAR(actual_els[0][0].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[0][1].imag(), 0.0489f, kTolerance);
EXPECT_NEAR(actual_els[1][0].imag(), -0.0489f, kTolerance);
EXPECT_NEAR(actual_els[1][1].imag(), 0.f, kTolerance);
}
TEST(CovarianceMatrixGeneratorTest, TestAngledCovarianceMatrix3Mics) {
const float kSpeedOfSound = 340;
const float kAngle = static_cast<float>(M_PI) / 4.f;
const float kFrequencyBin = 9;
const float kFftSize = 512;
const int kNumberFrequencyBins = 257;
const int kSampleRate = 42000;
const int kNumberMics = 3;
const float kMicSpacing = 0.05f;
const float kTolerance = 0.0001f;
std::vector<Point> geometry;
float first_mic = (kNumberMics - 1) * kMicSpacing / 2.f;
for (int i = 0; i < kNumberMics; ++i) {
geometry.push_back(Point(i * kMicSpacing - first_mic, 0.f, 0.f));
}
ComplexMatrix<float> actual_covariance_matrix(kNumberMics, kNumberMics);
CovarianceMatrixGenerator::AngledCovarianceMatrix(
kSpeedOfSound, kAngle, kFrequencyBin, kFftSize, kNumberFrequencyBins,
kSampleRate, geometry, &actual_covariance_matrix);
complex<float>* const* actual_els = actual_covariance_matrix.elements();
EXPECT_NEAR(actual_els[0][0].real(), 0.3333f, kTolerance);
EXPECT_NEAR(actual_els[0][1].real(), 0.2953f, kTolerance);
EXPECT_NEAR(actual_els[0][2].real(), 0.1899f, kTolerance);
EXPECT_NEAR(actual_els[1][0].real(), 0.2953f, kTolerance);
EXPECT_NEAR(actual_els[1][1].real(), 0.3333f, kTolerance);
EXPECT_NEAR(actual_els[1][2].real(), 0.2953f, kTolerance);
EXPECT_NEAR(actual_els[2][0].real(), 0.1899f, kTolerance);
EXPECT_NEAR(actual_els[2][1].real(), 0.2953f, kTolerance);
EXPECT_NEAR(actual_els[2][2].real(), 0.3333f, kTolerance);
EXPECT_NEAR(actual_els[0][0].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[0][1].imag(), 0.1546f, kTolerance);
EXPECT_NEAR(actual_els[0][2].imag(), 0.274f, kTolerance);
EXPECT_NEAR(actual_els[1][0].imag(), -0.1546f, kTolerance);
EXPECT_NEAR(actual_els[1][1].imag(), 0.f, kTolerance);
EXPECT_NEAR(actual_els[1][2].imag(), 0.1546f, kTolerance);
EXPECT_NEAR(actual_els[2][0].imag(), -0.274f, kTolerance);
EXPECT_NEAR(actual_els[2][1].imag(), -0.1546f, kTolerance);
EXPECT_NEAR(actual_els[2][2].imag(), 0.f, kTolerance);
}
// PhaseAlignmentMasks is tested by AngledCovarianceMatrix and by
// InitBeamformerWeights in BeamformerUnittest.
} // namespace webrtc

369
modules/audio_processing/beamformer/matrix.h
View File

@ -1,369 +0,0 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_MATRIX_H_
#define MODULES_AUDIO_PROCESSING_BEAMFORMER_MATRIX_H_
#include <algorithm>
#include <cstring>
#include <string>
#include <vector>
#include "rtc_base/checks.h"
#include "rtc_base/constructormagic.h"
namespace {
// Wrappers to get around the compiler warning resulting from the fact that
// there's no std::sqrt overload for ints. We cast all non-complex types to
// a double for the sqrt method.
template <typename T>
T sqrt_wrapper(T x) {
return sqrt(static_cast<double>(x));
}
template <typename S>
std::complex<S> sqrt_wrapper(std::complex<S> x) {
return sqrt(x);
}
} // namespace
namespace webrtc {
// Matrix is a class for doing standard matrix operations on 2 dimensional
// matrices of any size. Results of matrix operations are stored in the
// calling object. Function overloads exist for both in-place (the calling
// object is used as both an operand and the result) and out-of-place (all
// operands are passed in as parameters) operations. If operand dimensions
// mismatch, the program crashes. Out-of-place operations change the size of
// the calling object, if necessary, before operating.
//
// 'In-place' operations that inherently change the size of the matrix (eg.
// Transpose, Multiply on different-sized matrices) must make temporary copies
// (|scratch_elements_| and |scratch_data_|) of existing data to complete the
// operations.
//
// The data is stored contiguously. Data can be accessed internally as a flat
// array, |data_|, or as an array of row pointers, |elements_|, but is
// available to users only as an array of row pointers through |elements()|.
// Memory for storage is allocated when a matrix is resized only if the new
// size overflows capacity. Memory needed temporarily for any operations is
// similarly resized only if the new size overflows capacity.
//
// If you pass in storage through the ctor, that storage is copied into the
// matrix. TODO(claguna): albeit tricky, allow for data to be referenced
// instead of copied, and owned by the user.
template <typename T>
class Matrix {
public:
Matrix() : num_rows_(0), num_columns_(0) {}
// Allocates space for the elements and initializes all values to zero.
Matrix(size_t num_rows, size_t num_columns)
: num_rows_(num_rows), num_columns_(num_columns) {
Resize();
scratch_data_.resize(num_rows_ * num_columns_);
scratch_elements_.resize(num_rows_);
}
// Copies |data| into the new Matrix.
Matrix(const T* data, size_t num_rows, size_t num_columns)
: num_rows_(0), num_columns_(0) {
CopyFrom(data, num_rows, num_columns);
scratch_data_.resize(num_rows_ * num_columns_);
scratch_elements_.resize(num_rows_);
}
virtual ~Matrix() {}
// Deep copy an existing matrix.
void CopyFrom(const Matrix& other) {
CopyFrom(&other.data_[0], other.num_rows_, other.num_columns_);
}
// Copy |data| into the Matrix. The current data is lost.
void CopyFrom(const T* const data, size_t num_rows, size_t num_columns) {
Resize(num_rows, num_columns);
memcpy(&data_[0], data, num_rows_ * num_columns_ * sizeof(data_[0]));
}
Matrix& CopyFromColumn(const T* const* src,
size_t column_index,
size_t num_rows) {
Resize(1, num_rows);
for (size_t i = 0; i < num_columns_; ++i) {
data_[i] = src[i][column_index];
}
return *this;
}
void Resize(size_t num_rows, size_t num_columns) {
if (num_rows != num_rows_ || num_columns != num_columns_) {
num_rows_ = num_rows;
num_columns_ = num_columns;
Resize();
}
}
// Accessors and mutators.
size_t num_rows() const { return num_rows_; }
size_t num_columns() const { return num_columns_; }
T* const* elements() { return &elements_[0]; }
const T* const* elements() const { return &elements_[0]; }
T Trace() {
RTC_CHECK_EQ(num_rows_, num_columns_);
T trace = 0;
for (size_t i = 0; i < num_rows_; ++i) {
trace += elements_[i][i];
}
return trace;
}
// Matrix Operations. Returns *this to support method chaining.
Matrix& Transpose() {
CopyDataToScratch();
Resize(num_columns_, num_rows_);
return Transpose(scratch_elements());
}
Matrix& Transpose(const Matrix& operand) {
RTC_CHECK_EQ(operand.num_rows_, num_columns_);
RTC_CHECK_EQ(operand.num_columns_, num_rows_);
return Transpose(operand.elements());
}
template <typename S>
Matrix& Scale(const S& scalar) {
for (size_t i = 0; i < data_.size(); ++i) {
data_[i] *= scalar;
}
return *this;
}
template <typename S>
Matrix& Scale(const Matrix& operand, const S& scalar) {
CopyFrom(operand);
return Scale(scalar);
}
Matrix& Add(const Matrix& operand) {
RTC_CHECK_EQ(num_rows_, operand.num_rows_);
RTC_CHECK_EQ(num_columns_, operand.num_columns_);
for (size_t i = 0; i < data_.size(); ++i) {
data_[i] += operand.data_[i];
}
return *this;
}
Matrix& Add(const Matrix& lhs, const Matrix& rhs) {
CopyFrom(lhs);
return Add(rhs);
}
Matrix& Subtract(const Matrix& operand) {
RTC_CHECK_EQ(num_rows_, operand.num_rows_);
RTC_CHECK_EQ(num_columns_, operand.num_columns_);
for (size_t i = 0; i < data_.size(); ++i) {
data_[i] -= operand.data_[i];
}
return *this;
}
Matrix& Subtract(const Matrix& lhs, const Matrix& rhs) {
CopyFrom(lhs);
return Subtract(rhs);
}
Matrix& PointwiseMultiply(const Matrix& operand) {
RTC_CHECK_EQ(num_rows_, operand.num_rows_);
RTC_CHECK_EQ(num_columns_, operand.num_columns_);
for (size_t i = 0; i < data_.size(); ++i) {
data_[i] *= operand.data_[i];
}
return *this;
}
Matrix& PointwiseMultiply(const Matrix& lhs, const Matrix& rhs) {
CopyFrom(lhs);
return PointwiseMultiply(rhs);
}
Matrix& PointwiseDivide(const Matrix& operand) {
RTC_CHECK_EQ(num_rows_, operand.num_rows_);
RTC_CHECK_EQ(num_columns_, operand.num_columns_);
for (size_t i = 0; i < data_.size(); ++i) {
data_[i] /= operand.data_[i];
}
return *this;
}
Matrix& PointwiseDivide(const Matrix& lhs, const Matrix& rhs) {
CopyFrom(lhs);
return PointwiseDivide(rhs);
}
Matrix& PointwiseSquareRoot() {
for (size_t i = 0; i < data_.size(); ++i) {
data_[i] = sqrt_wrapper(data_[i]);
}
return *this;
}
Matrix& PointwiseSquareRoot(const Matrix& operand) {
CopyFrom(operand);
return PointwiseSquareRoot();
}
Matrix& PointwiseAbsoluteValue() {
for (size_t i = 0; i < data_.size(); ++i) {
data_[i] = abs(data_[i]);
}
return *this;
}
Matrix& PointwiseAbsoluteValue(const Matrix& operand) {
CopyFrom(operand);
return PointwiseAbsoluteValue();
}
Matrix& PointwiseSquare() {
for (size_t i = 0; i < data_.size(); ++i) {
data_[i] *= data_[i];
}
return *this;
}
Matrix& PointwiseSquare(const Matrix& operand) {
CopyFrom(operand);
return PointwiseSquare();
}
Matrix& Multiply(const Matrix& lhs, const Matrix& rhs) {
RTC_CHECK_EQ(lhs.num_columns_, rhs.num_rows_);
RTC_CHECK_EQ(num_rows_, lhs.num_rows_);
RTC_CHECK_EQ(num_columns_, rhs.num_columns_);
return Multiply(lhs.elements(), rhs.num_rows_, rhs.elements());
}
Matrix& Multiply(const Matrix& rhs) {
RTC_CHECK_EQ(num_columns_, rhs.num_rows_);
CopyDataToScratch();
Resize(num_rows_, rhs.num_columns_);
return Multiply(scratch_elements(), rhs.num_rows_, rhs.elements());
}
std::string ToString() const {
std::ostringstream ss;
ss << std::endl << "Matrix" << std::endl;
for (size_t i = 0; i < num_rows_; ++i) {
for (size_t j = 0; j < num_columns_; ++j) {
ss << elements_[i][j] << " ";
}
ss << std::endl;
}
ss << std::endl;
return ss.str();
}
protected:
void SetNumRows(const size_t num_rows) { num_rows_ = num_rows; }
void SetNumColumns(const size_t num_columns) { num_columns_ = num_columns; }
T* data() { return &data_[0]; }
const T* data() const { return &data_[0]; }
const T* const* scratch_elements() const { return &scratch_elements_[0]; }
// Resize the matrix. If an increase in capacity is required, the current
// data is lost.
void Resize() {
size_t size = num_rows_ * num_columns_;
data_.resize(size);
elements_.resize(num_rows_);
for (size_t i = 0; i < num_rows_; ++i) {
elements_[i] = &data_[i * num_columns_];
}
}
// Copies data_ into scratch_data_ and updates scratch_elements_ accordingly.
void CopyDataToScratch() {
scratch_data_ = data_;
scratch_elements_.resize(num_rows_);
for (size_t i = 0; i < num_rows_; ++i) {
scratch_elements_[i] = &scratch_data_[i * num_columns_];
}
}
private:
size_t num_rows_;
size_t num_columns_;
std::vector<T> data_;
std::vector<T*> elements_;
// Stores temporary copies of |data_| and |elements_| for in-place operations
// where referring to original data is necessary.
std::vector<T> scratch_data_;
std::vector<T*> scratch_elements_;
// Helpers for Transpose and Multiply operations that unify in-place and
// out-of-place solutions.
Matrix& Transpose(const T* const* src) {
for (size_t i = 0; i < num_rows_; ++i) {
for (size_t j = 0; j < num_columns_; ++j) {
elements_[i][j] = src[j][i];
}
}
return *this;
}
Matrix& Multiply(const T* const* lhs,
size_t num_rows_rhs,
const T* const* rhs) {
for (size_t row = 0; row < num_rows_; ++row) {
for (size_t col = 0; col < num_columns_; ++col) {
T cur_element = 0;
for (size_t i = 0; i < num_rows_rhs; ++i) {
cur_element += lhs[row][i] * rhs[i][col];
}
elements_[row][col] = cur_element;
}
}
return *this;
}
RTC_DISALLOW_COPY_AND_ASSIGN(Matrix);
};
} // namespace webrtc
#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_MATRIX_H_

98
modules/audio_processing/beamformer/matrix_test_helpers.h
View File

@ -1,98 +0,0 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_MATRIX_TEST_HELPERS_H_
#define MODULES_AUDIO_PROCESSING_BEAMFORMER_MATRIX_TEST_HELPERS_H_
#include "modules/audio_processing/beamformer/complex_matrix.h"
#include "modules/audio_processing/beamformer/matrix.h"
#include "test/gtest.h"
namespace {
const float kTolerance = 0.001f;
}
namespace webrtc {
using std::complex;
// Functions used in both matrix_unittest and complex_matrix_unittest.
class MatrixTestHelpers {
public:
template <typename T>
static void ValidateMatrixEquality(const Matrix<T>& expected,
const Matrix<T>& actual) {
EXPECT_EQ(expected.num_rows(), actual.num_rows());
EXPECT_EQ(expected.num_columns(), actual.num_columns());
const T* const* expected_elements = expected.elements();
const T* const* actual_elements = actual.elements();
for (size_t i = 0; i < expected.num_rows(); ++i) {
for (size_t j = 0; j < expected.num_columns(); ++j) {
EXPECT_EQ(expected_elements[i][j], actual_elements[i][j]);
}
}
}
static void ValidateMatrixEqualityFloat(const Matrix<float>& expected,
const Matrix<float>& actual) {
EXPECT_EQ(expected.num_rows(), actual.num_rows());
EXPECT_EQ(expected.num_columns(), actual.num_columns());
const float* const* expected_elements = expected.elements();
const float* const* actual_elements = actual.elements();
for (size_t i = 0; i < expected.num_rows(); ++i) {
for (size_t j = 0; j < expected.num_columns(); ++j) {
EXPECT_NEAR(expected_elements[i][j], actual_elements[i][j], kTolerance);
}
}
}
static void ValidateMatrixEqualityComplexFloat(
const Matrix<complex<float> >& expected,
const Matrix<complex<float> >& actual) {
EXPECT_EQ(expected.num_rows(), actual.num_rows());
EXPECT_EQ(expected.num_columns(), actual.num_columns());
const complex<float>* const* expected_elements = expected.elements();
const complex<float>* const* actual_elements = actual.elements();
for (size_t i = 0; i < expected.num_rows(); ++i) {
for (size_t j = 0; j < expected.num_columns(); ++j) {
EXPECT_NEAR(expected_elements[i][j].real(),
actual_elements[i][j].real(), kTolerance);
EXPECT_NEAR(expected_elements[i][j].imag(),
actual_elements[i][j].imag(), kTolerance);
}
}
}
static void ValidateMatrixNearEqualityComplexFloat(
const Matrix<complex<float> >& expected,
const Matrix<complex<float> >& actual,
float tolerance) {
EXPECT_EQ(expected.num_rows(), actual.num_rows());
EXPECT_EQ(expected.num_columns(), actual.num_columns());
const complex<float>* const* expected_elements = expected.elements();
const complex<float>* const* actual_elements = actual.elements();
for (size_t i = 0; i < expected.num_rows(); ++i) {
for (size_t j = 0; j < expected.num_columns(); ++j) {
EXPECT_NEAR(expected_elements[i][j].real(),
actual_elements[i][j].real(), tolerance);
EXPECT_NEAR(expected_elements[i][j].imag(),
actual_elements[i][j].imag(), tolerance);
}
}
}
};
} // namespace webrtc
#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_MATRIX_TEST_HELPERS_H_

326
modules/audio_processing/beamformer/matrix_unittest.cc
View File

@ -1,326 +0,0 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <complex>
#include "modules/audio_processing/beamformer/matrix.h"
#include "modules/audio_processing/beamformer/matrix_test_helpers.h"
#include "test/gtest.h"
namespace webrtc {
using std::complex;
TEST(MatrixTest, TestMultiplySameSize) {
const int kNumRows = 2;
const int kNumCols = 2;
const float kValuesLeft[kNumRows][kNumCols] = {{1.1f, 2.2f}, {3.3f, 4.4f}};
const float kValuesRight[kNumRows][kNumCols] = {{5.4f, 127.f},
{4600.f, -555.f}};
const float kValuesExpected[kNumRows][kNumCols] = {{10125.94f, -1081.3f},
{20257.82f, -2022.9f}};
Matrix<float> lh_mat(*kValuesLeft, kNumRows, kNumCols);
Matrix<float> rh_mat(*kValuesRight, kNumRows, kNumCols);
Matrix<float> expected_result(*kValuesExpected, kNumRows, kNumCols);
Matrix<float> actual_result(kNumRows, kNumCols);
actual_result.Multiply(lh_mat, rh_mat);
MatrixTestHelpers::ValidateMatrixEquality(expected_result, actual_result);
lh_mat.Multiply(rh_mat);
MatrixTestHelpers::ValidateMatrixEquality(lh_mat, actual_result);
}
TEST(MatrixTest, TestMultiplyDifferentSize) {
const int kNumRowsLeft = 2;
const int kNumColsLeft = 3;
const int kNumRowsRight = 3;
const int kNumColsRight = 2;
const int kValuesLeft[kNumRowsLeft][kNumColsLeft] = {{35, 466, -15},
{-3, 3422, 9}};
const int kValuesRight[kNumRowsRight][kNumColsRight] = {
{765, -42}, {0, 194}, {625, 66321}};
const int kValuesExpected[kNumRowsLeft][kNumColsRight] = {{17400, -905881},
{3330, 1260883}};
Matrix<int> lh_mat(*kValuesLeft, kNumRowsLeft, kNumColsLeft);
Matrix<int> rh_mat(*kValuesRight, kNumRowsRight, kNumColsRight);
Matrix<int> expected_result(*kValuesExpected, kNumRowsLeft, kNumColsRight);
Matrix<int> actual_result(kNumRowsLeft, kNumColsRight);
actual_result.Multiply(lh_mat, rh_mat);
MatrixTestHelpers::ValidateMatrixEquality(expected_result, actual_result);
lh_mat.Multiply(rh_mat);
MatrixTestHelpers::ValidateMatrixEquality(lh_mat, actual_result);
}
TEST(MatrixTest, TestTranspose) {
const int kNumInitialRows = 2;
const int kNumInitialCols = 4;
const int kNumResultRows = 4;
const int kNumResultCols = 2;
const float kValuesInitial[kNumInitialRows][kNumInitialCols] = {
{1.1f, 2.2f, 3.3f, 4.4f}, {5.5f, 6.6f, 7.7f, 8.8f}};
const float kValuesExpected[kNumResultRows][kNumResultCols] = {
{1.1f, 5.5f}, {2.2f, 6.6f}, {3.3f, 7.7f}, {4.4f, 8.8f}};
Matrix<float> initial_mat(*kValuesInitial, kNumInitialRows, kNumInitialCols);
Matrix<float> expected_result(*kValuesExpected, kNumResultRows,
kNumResultCols);
Matrix<float> actual_result(kNumResultRows, kNumResultCols);
actual_result.Transpose(initial_mat);
MatrixTestHelpers::ValidateMatrixEqualityFloat(expected_result,
actual_result);
initial_mat.Transpose();
MatrixTestHelpers::ValidateMatrixEqualityFloat(initial_mat, actual_result);
}
TEST(MatrixTest, TestScale) {
const int kNumRows = 3;
const int kNumCols = 3;
const int kScaleFactor = -9;
const int kValuesInitial[kNumRows][kNumCols] = {
{1, 20, 5000}, {-3, -29, 66}, {7654, 0, -23455}};
const int kValuesExpected[kNumRows][kNumCols] = {
{-9, -180, -45000}, {27, 261, -594}, {-68886, 0, 211095}};
Matrix<int> initial_mat(*kValuesInitial, kNumRows, kNumCols);
Matrix<int> expected_result(*kValuesExpected, kNumRows, kNumCols);
Matrix<int> actual_result;
actual_result.Scale(initial_mat, kScaleFactor);
MatrixTestHelpers::ValidateMatrixEquality(expected_result, actual_result);
initial_mat.Scale(kScaleFactor);
MatrixTestHelpers::ValidateMatrixEquality(initial_mat, actual_result);
}
TEST(MatrixTest, TestPointwiseAdd) {
const int kNumRows = 2;
const int kNumCols = 3;
const float kValuesLeft[kNumRows][kNumCols] = {{1.1f, 210.45f, -549.2f},
{11.876f, 586.7f, -64.35f}};
const float kValuesRight[kNumRows][kNumCols] = {{-50.4f, 1.f, 0.5f},
{460.f, -554.2f, 4566.f}};
const float kValuesExpected[kNumRows][kNumCols] = {
{-49.3f, 211.45f, -548.7f}, {471.876f, 32.5f, 4501.65f}};
Matrix<float> lh_mat(*kValuesLeft, kNumRows, kNumCols);
Matrix<float> rh_mat(*kValuesRight, kNumRows, kNumCols);
Matrix<float> expected_result(*kValuesExpected, kNumRows, kNumCols);
Matrix<float> actual_result;
actual_result.Add(lh_mat, rh_mat);
MatrixTestHelpers::ValidateMatrixEqualityFloat(expected_result,
actual_result);
lh_mat.Add(rh_mat);
MatrixTestHelpers::ValidateMatrixEqualityFloat(lh_mat, actual_result);
}
TEST(MatrixTest, TestPointwiseSubtract) {
const int kNumRows = 3;
const int kNumCols = 2;
const float kValuesLeft[kNumRows][kNumCols] = {
{1.1f, 210.45f}, {-549.2f, 11.876f}, {586.7f, -64.35f}};
const float kValuesRight[kNumRows][kNumCols] = {
{-50.4f, 1.f}, {0.5f, 460.f}, {-554.2f, 4566.f}};
const float kValuesExpected[kNumRows][kNumCols] = {
{51.5f, 209.45f}, {-549.7f, -448.124f}, {1140.9f, -4630.35f}};
Matrix<float> lh_mat(*kValuesLeft, kNumRows, kNumCols);
Matrix<float> rh_mat(*kValuesRight, kNumRows, kNumCols);
Matrix<float> expected_result(*kValuesExpected, kNumRows, kNumCols);
Matrix<float> actual_result;
actual_result.Subtract(lh_mat, rh_mat);
MatrixTestHelpers::ValidateMatrixEqualityFloat(expected_result,
actual_result);
lh_mat.Subtract(rh_mat);
MatrixTestHelpers::ValidateMatrixEqualityFloat(lh_mat, actual_result);
}
TEST(MatrixTest, TestPointwiseMultiply) {
const int kNumRows = 1;
const int kNumCols = 5;
const float kValuesLeft[kNumRows][kNumCols] = {
{1.1f, 6.4f, 0.f, -1.f, -88.3f}};
const float kValuesRight[kNumRows][kNumCols] = {
{53.2f, -210.45f, -549.2f, 99.99f, -45.2f}};
const float kValuesExpected[kNumRows][kNumCols] = {
{58.52f, -1346.88f, 0.f, -99.99f, 3991.16f}};
Matrix<float> lh_mat(*kValuesLeft, kNumRows, kNumCols);
Matrix<float> rh_mat(*kValuesRight, kNumRows, kNumCols);
Matrix<float> expected_result(*kValuesExpected, kNumRows, kNumCols);
Matrix<float> actual_result;
actual_result.PointwiseMultiply(lh_mat, rh_mat);
MatrixTestHelpers::ValidateMatrixEqualityFloat(expected_result,
actual_result);
lh_mat.PointwiseMultiply(rh_mat);
MatrixTestHelpers::ValidateMatrixEqualityFloat(lh_mat, actual_result);
}
TEST(MatrixTest, TestPointwiseDivide) {
const int kNumRows = 5;
const int kNumCols = 1;
const float kValuesLeft[kNumRows][kNumCols] = {
{1.1f}, {6.4f}, {0.f}, {-1.f}, {-88.3f}};
const float kValuesRight[kNumRows][kNumCols] = {
{53.2f}, {-210.45f}, {-549.2f}, {99.99f}, {-45.2f}};
const float kValuesExpected[kNumRows][kNumCols] = {
{0.020676691f}, {-0.03041102399f}, {0.f}, {-0.010001f}, {1.9535398f}};
Matrix<float> lh_mat(*kValuesLeft, kNumRows, kNumCols);
Matrix<float> rh_mat(*kValuesRight, kNumRows, kNumCols);
Matrix<float> expected_result(*kValuesExpected, kNumRows, kNumCols);
Matrix<float> actual_result;
actual_result.PointwiseDivide(lh_mat, rh_mat);
MatrixTestHelpers::ValidateMatrixEqualityFloat(expected_result,
actual_result);
lh_mat.PointwiseDivide(rh_mat);
MatrixTestHelpers::ValidateMatrixEqualityFloat(lh_mat, actual_result);
}
TEST(MatrixTest, TestPointwiseSquareRoot) {
const int kNumRows = 2;
const int kNumCols = 2;
const int kValues[kNumRows][kNumCols] = {{4, 9}, {16, 0}};
const int kValuesExpected[kNumRows][kNumCols] = {{2, 3}, {4, 0}};
Matrix<int> operand_mat(*kValues, kNumRows, kNumCols);
Matrix<int> expected_result(*kValuesExpected, kNumRows, kNumCols);
Matrix<int> actual_result;
actual_result.PointwiseSquareRoot(operand_mat);
MatrixTestHelpers::ValidateMatrixEquality(expected_result, actual_result);
operand_mat.PointwiseSquareRoot();
MatrixTestHelpers::ValidateMatrixEquality(operand_mat, actual_result);
}
TEST(MatrixTest, TestPointwiseSquareRootComplex) {
const int kNumRows = 1;
const int kNumCols = 3;
const complex<float> kValues[kNumRows][kNumCols] = {
{complex<float>(-4.f, 0), complex<float>(0, 9), complex<float>(3, -4)}};
const complex<float> kValuesExpected[kNumRows][kNumCols] = {
{complex<float>(0.f, 2.f), complex<float>(2.1213202f, 2.1213202f),
complex<float>(2.f, -1.f)}};
Matrix<complex<float> > operand_mat(*kValues, kNumRows, kNumCols);
Matrix<complex<float> > expected_result(*kValuesExpected, kNumRows, kNumCols);
Matrix<complex<float> > actual_result;
actual_result.PointwiseSquareRoot(operand_mat);
MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(expected_result,
actual_result);
operand_mat.PointwiseSquareRoot();
MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(operand_mat,
actual_result);
}
TEST(MatrixTest, TestPointwiseAbsoluteValue) {
const int kNumRows = 1;
const int kNumCols = 3;
const complex<float> kValues[kNumRows][kNumCols] = {
{complex<float>(-4.f, 0), complex<float>(0, 9), complex<float>(3, -4)}};
const complex<float> kValuesExpected[kNumRows][kNumCols] = {
{complex<float>(4.f, 0), complex<float>(9.f, 0), complex<float>(5.f, 0)}};
Matrix<complex<float> > operand_mat(*kValues, kNumRows, kNumCols);
Matrix<complex<float> > expected_result(*kValuesExpected, kNumRows, kNumCols);
Matrix<complex<float> > actual_result;
actual_result.PointwiseAbsoluteValue(operand_mat);
MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(expected_result,
actual_result);
operand_mat.PointwiseAbsoluteValue();
MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(operand_mat,
actual_result);
}
TEST(MatrixTest, TestPointwiseSquare) {
const int kNumRows = 1;
const int kNumCols = 3;
const float kValues[kNumRows][kNumCols] = {{2.4f, -4.f, 3.3f}};
const float kValuesExpected[kNumRows][kNumCols] = {{5.76f, 16.f, 10.89f}};
Matrix<float> operand_mat(*kValues, kNumRows, kNumCols);
Matrix<float> expected_result(*kValuesExpected, kNumRows, kNumCols);
Matrix<float> actual_result;
actual_result.PointwiseSquare(operand_mat);
MatrixTestHelpers::ValidateMatrixEqualityFloat(expected_result,
actual_result);
operand_mat.PointwiseSquare();
MatrixTestHelpers::ValidateMatrixEqualityFloat(operand_mat, actual_result);
}
TEST(MatrixTest, TestComplexOperations) {
const int kNumRows = 2;
const int kNumCols = 2;
const complex<float> kValuesLeft[kNumRows][kNumCols] = {
{complex<float>(1.f, 1.f), complex<float>(2.f, 2.f)},
{complex<float>(3.f, 3.f), complex<float>(4.f, 4.f)}};
const complex<float> kValuesRight[kNumRows][kNumCols] = {
{complex<float>(5.f, 5.f), complex<float>(6.f, 6.f)},
{complex<float>(7.f, 7.f), complex<float>(8.f, 8.f)}};
const complex<float> kValuesExpectedAdd[kNumRows][kNumCols] = {
{complex<float>(6.f, 6.f), complex<float>(8.f, 8.f)},
{complex<float>(10.f, 10.f), complex<float>(12.f, 12.f)}};
const complex<float> kValuesExpectedMultiply[kNumRows][kNumCols] = {
{complex<float>(0.f, 38.f), complex<float>(0.f, 44.f)},
{complex<float>(0.f, 86.f), complex<float>(0.f, 100.f)}};
const complex<float> kValuesExpectedPointwiseDivide[kNumRows][kNumCols] = {
{complex<float>(0.2f, 0.f), complex<float>(0.33333333f, 0.f)},
{complex<float>(0.42857143f, 0.f), complex<float>(0.5f, 0.f)}};
Matrix<complex<float> > lh_mat(*kValuesLeft, kNumRows, kNumCols);
Matrix<complex<float> > rh_mat(*kValuesRight, kNumRows, kNumCols);
Matrix<complex<float> > expected_result_add(*kValuesExpectedAdd, kNumRows,
kNumCols);
Matrix<complex<float> > expected_result_multiply(*kValuesExpectedMultiply,
kNumRows, kNumCols);
Matrix<complex<float> > expected_result_pointwise_divide(
*kValuesExpectedPointwiseDivide, kNumRows, kNumCols);
Matrix<complex<float> > actual_result_add;
Matrix<complex<float> > actual_result_multiply(kNumRows, kNumCols);
Matrix<complex<float> > actual_result_pointwise_divide;
actual_result_add.Add(lh_mat, rh_mat);
MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(expected_result_add,
actual_result_add);
actual_result_multiply.Multiply(lh_mat, rh_mat);
MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(
expected_result_multiply, actual_result_multiply);
actual_result_pointwise_divide.PointwiseDivide(lh_mat, rh_mat);
MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(
expected_result_pointwise_divide, actual_result_pointwise_divide);
}
} // namespace webrtc

39
modules/audio_processing/beamformer/mock_nonlinear_beamformer.h
View File

@ -1,39 +0,0 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_MOCK_BEAMFORMER_H_
#define MODULES_AUDIO_PROCESSING_BEAMFORMER_MOCK_BEAMFORMER_H_
#include <vector>
#include "modules/audio_processing/beamformer/nonlinear_beamformer.h"
#include "test/gmock.h"
namespace webrtc {
class MockNonlinearBeamformer : public NonlinearBeamformer {
public:
MockNonlinearBeamformer(const std::vector<Point>& array_geometry,
size_t num_postfilter_channels)
: NonlinearBeamformer(array_geometry, num_postfilter_channels) {}
MockNonlinearBeamformer(const std::vector<Point>& array_geometry)
: NonlinearBeamformer(array_geometry, 1u) {}
MOCK_METHOD2(Initialize, void(int chunk_size_ms, int sample_rate_hz));
MOCK_METHOD1(AnalyzeChunk, void(const ChannelBuffer<float>& data));
MOCK_METHOD1(PostFilter, void(ChannelBuffer<float>* data));
MOCK_METHOD1(IsInBeam, bool(const SphericalPointf& spherical_point));
MOCK_METHOD0(is_target_present, bool());
};
} // namespace webrtc
#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_MOCK_BEAMFORMER_H_

587
modules/audio_processing/beamformer/nonlinear_beamformer.cc
View File

@ -1,587 +0,0 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#define _USE_MATH_DEFINES
#include "modules/audio_processing/beamformer/nonlinear_beamformer.h"
#include <algorithm>
#include <cmath>
#include <numeric>
#include <vector>
#include "common_audio/window_generator.h"
#include "modules/audio_processing/beamformer/covariance_matrix_generator.h"
#include "rtc_base/arraysize.h"
namespace webrtc {
namespace {
// Alpha for the Kaiser Bessel Derived window.
const float kKbdAlpha = 1.5f;
const float kSpeedOfSoundMeterSeconds = 343;
// The minimum separation in radians between the target direction and an
// interferer scenario.
const float kMinAwayRadians = 0.2f;
// The separation between the target direction and the closest interferer
// scenario is proportional to this constant.
const float kAwaySlope = 0.008f;
// When calculating the interference covariance matrix, this is the weight for
// the weighted average between the uniform covariance matrix and the angled
// covariance matrix.
// Rpsi = Rpsi_angled * kBalance + Rpsi_uniform * (1 - kBalance)
const float kBalance = 0.95f;
// Alpha coefficients for mask smoothing.
const float kMaskTimeSmoothAlpha = 0.2f;
const float kMaskFrequencySmoothAlpha = 0.6f;
// The average mask is computed from masks in this mid-frequency range. If these
// ranges are changed |kMaskQuantile| might need to be adjusted.
const int kLowMeanStartHz = 200;
const int kLowMeanEndHz = 400;
// Range limiter for subtractive terms in the nominator and denominator of the
// postfilter expression. It handles the scenario mismatch between the true and
// model sources (target and interference).
const float kCutOffConstant = 0.9999f;
// Quantile of mask values which is used to estimate target presence.
const float kMaskQuantile = 0.7f;
// Mask threshold over which the data is considered signal and not interference.
// It has to be updated every time the postfilter calculation is changed
// significantly.
// TODO(aluebs): Write a tool to tune the target threshold automatically based
// on files annotated with target and interference ground truth.
const float kMaskTargetThreshold = 0.01f;
// Time in seconds after which the data is considered interference if the mask
// does not pass |kMaskTargetThreshold|.
const float kHoldTargetSeconds = 0.25f;
// To compensate for the attenuation this algorithm introduces to the target
// signal. It was estimated empirically from a low-noise low-reverberation
// recording from broadside.
const float kCompensationGain = 2.f;
// Does conjugate(|norm_mat|) * |mat| * transpose(|norm_mat|). No extra space is
// used; to accomplish this, we compute both multiplications in the same loop.
// The returned norm is clamped to be non-negative.
float Norm(const ComplexMatrix<float>& mat,
const ComplexMatrix<float>& norm_mat) {
RTC_CHECK_EQ(1, norm_mat.num_rows());
RTC_CHECK_EQ(norm_mat.num_columns(), mat.num_rows());
RTC_CHECK_EQ(norm_mat.num_columns(), mat.num_columns());
complex<float> first_product = complex<float>(0.f, 0.f);
complex<float> second_product = complex<float>(0.f, 0.f);
const complex<float>* const* mat_els = mat.elements();
const complex<float>* const* norm_mat_els = norm_mat.elements();
for (size_t i = 0; i < norm_mat.num_columns(); ++i) {
for (size_t j = 0; j < norm_mat.num_columns(); ++j) {
first_product += conj(norm_mat_els[0][j]) * mat_els[j][i];
}
second_product += first_product * norm_mat_els[0][i];
first_product = 0.f;
}
return std::max(second_product.real(), 0.f);
}
// Does conjugate(|lhs|) * |rhs| for row vectors |lhs| and |rhs|.
complex<float> ConjugateDotProduct(const ComplexMatrix<float>& lhs,
const ComplexMatrix<float>& rhs) {
RTC_CHECK_EQ(1, lhs.num_rows());
RTC_CHECK_EQ(1, rhs.num_rows());
RTC_CHECK_EQ(lhs.num_columns(), rhs.num_columns());
const complex<float>* const* lhs_elements = lhs.elements();
const complex<float>* const* rhs_elements = rhs.elements();
complex<float> result = complex<float>(0.f, 0.f);
for (size_t i = 0; i < lhs.num_columns(); ++i) {
result += conj(lhs_elements[0][i]) * rhs_elements[0][i];
}
return result;
}
// Works for positive numbers only.
size_t Round(float x) {
return static_cast<size_t>(std::floor(x + 0.5f));
}
// Calculates the sum of squares of a complex matrix.
float SumSquares(const ComplexMatrix<float>& mat) {
float sum_squares = 0.f;
const complex<float>* const* mat_els = mat.elements();
for (size_t i = 0; i < mat.num_rows(); ++i) {
for (size_t j = 0; j < mat.num_columns(); ++j) {
float abs_value = std::abs(mat_els[i][j]);
sum_squares += abs_value * abs_value;
}
}
return sum_squares;
}
// Does |out| = |in|.' * conj(|in|) for row vector |in|.
void TransposedConjugatedProduct(const ComplexMatrix<float>& in,
ComplexMatrix<float>* out) {
RTC_CHECK_EQ(1, in.num_rows());
RTC_CHECK_EQ(out->num_rows(), in.num_columns());
RTC_CHECK_EQ(out->num_columns(), in.num_columns());
const complex<float>* in_elements = in.elements()[0];
complex<float>* const* out_elements = out->elements();
for (size_t i = 0; i < out->num_rows(); ++i) {
for (size_t j = 0; j < out->num_columns(); ++j) {
out_elements[i][j] = in_elements[i] * conj(in_elements[j]);
}
}
}
std::vector<Point> GetCenteredArray(std::vector<Point> array_geometry) {
for (size_t dim = 0; dim < 3; ++dim) {
float center = 0.f;
for (size_t i = 0; i < array_geometry.size(); ++i) {
center += array_geometry[i].c[dim];
}
center /= array_geometry.size();
for (size_t i = 0; i < array_geometry.size(); ++i) {
array_geometry[i].c[dim] -= center;
}
}
return array_geometry;
}
} // namespace
const float NonlinearBeamformer::kHalfBeamWidthRadians = DegreesToRadians(20.f);
// static
const size_t NonlinearBeamformer::kNumFreqBins;
PostFilterTransform::PostFilterTransform(size_t num_channels,
size_t chunk_length,
float* window,
size_t fft_size)
: transform_(num_channels,
num_channels,
chunk_length,
window,
fft_size,
fft_size / 2,
this),
num_freq_bins_(fft_size / 2 + 1) {}
void PostFilterTransform::ProcessChunk(float* const* data, float* final_mask) {
final_mask_ = final_mask;
transform_.ProcessChunk(data, data);
}
void PostFilterTransform::ProcessAudioBlock(const complex<float>* const* input,
size_t num_input_channels,
size_t num_freq_bins,
size_t num_output_channels,
complex<float>* const* output) {
RTC_DCHECK_EQ(num_freq_bins_, num_freq_bins);
RTC_DCHECK_EQ(num_input_channels, num_output_channels);
for (size_t ch = 0; ch < num_input_channels; ++ch) {
for (size_t f_ix = 0; f_ix < num_freq_bins_; ++f_ix) {
output[ch][f_ix] =
kCompensationGain * final_mask_[f_ix] * input[ch][f_ix];
}
}
}
NonlinearBeamformer::NonlinearBeamformer(
const std::vector<Point>& array_geometry,
size_t num_postfilter_channels,
SphericalPointf target_direction)
: num_input_channels_(array_geometry.size()),
num_postfilter_channels_(num_postfilter_channels),
array_geometry_(GetCenteredArray(array_geometry)),
array_normal_(GetArrayNormalIfExists(array_geometry)),
min_mic_spacing_(GetMinimumSpacing(array_geometry)),
target_angle_radians_(target_direction.azimuth()),
away_radians_(std::min(
static_cast<float>(M_PI),
std::max(kMinAwayRadians,
kAwaySlope * static_cast<float>(M_PI) / min_mic_spacing_))) {
WindowGenerator::KaiserBesselDerived(kKbdAlpha, kFftSize, window_);
}
NonlinearBeamformer::~NonlinearBeamformer() = default;
void NonlinearBeamformer::Initialize(int chunk_size_ms, int sample_rate_hz) {
chunk_length_ =
static_cast<size_t>(sample_rate_hz / (1000.f / chunk_size_ms));
sample_rate_hz_ = sample_rate_hz;
high_pass_postfilter_mask_ = 1.f;
is_target_present_ = false;
hold_target_blocks_ = kHoldTargetSeconds * 2 * sample_rate_hz / kFftSize;
interference_blocks_count_ = hold_target_blocks_;
process_transform_.reset(new LappedTransform(num_input_channels_, 0u,
chunk_length_, window_, kFftSize,
kFftSize / 2, this));
postfilter_transform_.reset(new PostFilterTransform(
num_postfilter_channels_, chunk_length_, window_, kFftSize));
const float wave_number_step =
(2.f * M_PI * sample_rate_hz_) / (kFftSize * kSpeedOfSoundMeterSeconds);
for (size_t i = 0; i < kNumFreqBins; ++i) {
time_smooth_mask_[i] = 1.f;
final_mask_[i] = 1.f;
wave_numbers_[i] = i * wave_number_step;
}
InitLowFrequencyCorrectionRanges();
InitDiffuseCovMats();
AimAt(SphericalPointf(target_angle_radians_, 0.f, 1.f));
}
// These bin indexes determine the regions over which a mean is taken. This is
// applied as a constant value over the adjacent end "frequency correction"
// regions.
//
// low_mean_start_bin_ high_mean_start_bin_
// v v constant
// |----------------|--------|----------------|-------|----------------|
// constant ^ ^
// low_mean_end_bin_ high_mean_end_bin_
//
void NonlinearBeamformer::InitLowFrequencyCorrectionRanges() {
low_mean_start_bin_ =
Round(static_cast<float>(kLowMeanStartHz) * kFftSize / sample_rate_hz_);
low_mean_end_bin_ =
Round(static_cast<float>(kLowMeanEndHz) * kFftSize / sample_rate_hz_);
RTC_DCHECK_GT(low_mean_start_bin_, 0U);
RTC_DCHECK_LT(low_mean_start_bin_, low_mean_end_bin_);
}
void NonlinearBeamformer::InitHighFrequencyCorrectionRanges() {
const float kAliasingFreqHz =
kSpeedOfSoundMeterSeconds /
(min_mic_spacing_ * (1.f + std::abs(std::cos(target_angle_radians_))));
const float kHighMeanStartHz =
std::min(0.5f * kAliasingFreqHz, sample_rate_hz_ / 2.f);
const float kHighMeanEndHz =
std::min(0.75f * kAliasingFreqHz, sample_rate_hz_ / 2.f);
high_mean_start_bin_ = Round(kHighMeanStartHz * kFftSize / sample_rate_hz_);
high_mean_end_bin_ = Round(kHighMeanEndHz * kFftSize / sample_rate_hz_);
RTC_DCHECK_LT(low_mean_end_bin_, high_mean_end_bin_);
RTC_DCHECK_LT(high_mean_start_bin_, high_mean_end_bin_);
RTC_DCHECK_LT(high_mean_end_bin_, kNumFreqBins - 1);
}
void NonlinearBeamformer::InitInterfAngles() {
interf_angles_radians_.clear();
const Point target_direction = AzimuthToPoint(target_angle_radians_);
const Point clockwise_interf_direction =
AzimuthToPoint(target_angle_radians_ - away_radians_);
if (!array_normal_ ||
DotProduct(*array_normal_, target_direction) *
DotProduct(*array_normal_, clockwise_interf_direction) >=
0.f) {
// The target and clockwise interferer are in the same half-plane defined
// by the array.
interf_angles_radians_.push_back(target_angle_radians_ - away_radians_);
} else {
// Otherwise, the interferer will begin reflecting back at the target.
// Instead rotate it away 180 degrees.
interf_angles_radians_.push_back(target_angle_radians_ - away_radians_ +
M_PI);
}
const Point counterclock_interf_direction =
AzimuthToPoint(target_angle_radians_ + away_radians_);
if (!array_normal_ ||
DotProduct(*array_normal_, target_direction) *
DotProduct(*array_normal_, counterclock_interf_direction) >=
0.f) {
// The target and counter-clockwise interferer are in the same half-plane
// defined by the array.
interf_angles_radians_.push_back(target_angle_radians_ + away_radians_);
} else {
// Otherwise, the interferer will begin reflecting back at the target.
// Instead rotate it away 180 degrees.
interf_angles_radians_.push_back(target_angle_radians_ + away_radians_ -
M_PI);
}
}
void NonlinearBeamformer::InitDelaySumMasks() {
for (size_t f_ix = 0; f_ix < kNumFreqBins; ++f_ix) {
delay_sum_masks_[f_ix].Resize(1, num_input_channels_);
CovarianceMatrixGenerator::PhaseAlignmentMasks(
f_ix, kFftSize, sample_rate_hz_, kSpeedOfSoundMeterSeconds,
array_geometry_, target_angle_radians_, &delay_sum_masks_[f_ix]);
complex_f norm_factor = sqrt(
ConjugateDotProduct(delay_sum_masks_[f_ix], delay_sum_masks_[f_ix]));
delay_sum_masks_[f_ix].Scale(1.f / norm_factor);
}
}
void NonlinearBeamformer::InitTargetCovMats() {
for (size_t i = 0; i < kNumFreqBins; ++i) {
target_cov_mats_[i].Resize(num_input_channels_, num_input_channels_);
TransposedConjugatedProduct(delay_sum_masks_[i], &target_cov_mats_[i]);
}
}
void NonlinearBeamformer::InitDiffuseCovMats() {
for (size_t i = 0; i < kNumFreqBins; ++i) {
uniform_cov_mat_[i].Resize(num_input_channels_, num_input_channels_);
CovarianceMatrixGenerator::UniformCovarianceMatrix(
wave_numbers_[i], array_geometry_, &uniform_cov_mat_[i]);
complex_f normalization_factor = uniform_cov_mat_[i].elements()[0][0];
uniform_cov_mat_[i].Scale(1.f / normalization_factor);
uniform_cov_mat_[i].Scale(1 - kBalance);
}
}
void NonlinearBeamformer::InitInterfCovMats() {
for (size_t i = 0; i < kNumFreqBins; ++i) {
interf_cov_mats_[i].clear();
for (size_t j = 0; j < interf_angles_radians_.size(); ++j) {
interf_cov_mats_[i].push_back(std::unique_ptr<ComplexMatrixF>(
new ComplexMatrixF(num_input_channels_, num_input_channels_)));
ComplexMatrixF angled_cov_mat(num_input_channels_, num_input_channels_);
CovarianceMatrixGenerator::AngledCovarianceMatrix(
kSpeedOfSoundMeterSeconds, interf_angles_radians_[j], i, kFftSize,
kNumFreqBins, sample_rate_hz_, array_geometry_, &angled_cov_mat);
// Normalize matrices before averaging them.
complex_f normalization_factor = angled_cov_mat.elements()[0][0];
angled_cov_mat.Scale(1.f / normalization_factor);
// Weighted average of matrices.
angled_cov_mat.Scale(kBalance);
interf_cov_mats_[i][j]->Add(uniform_cov_mat_[i], angled_cov_mat);
}
}
}
void NonlinearBeamformer::NormalizeCovMats() {
for (size_t i = 0; i < kNumFreqBins; ++i) {
rxiws_[i] = Norm(target_cov_mats_[i], delay_sum_masks_[i]);
rpsiws_[i].clear();
for (size_t j = 0; j < interf_angles_radians_.size(); ++j) {
rpsiws_[i].push_back(Norm(*interf_cov_mats_[i][j], delay_sum_masks_[i]));
}
}
}
void NonlinearBeamformer::AnalyzeChunk(const ChannelBuffer<float>& data) {
RTC_DCHECK_EQ(data.num_channels(), num_input_channels_);
RTC_DCHECK_EQ(data.num_frames_per_band(), chunk_length_);
old_high_pass_mask_ = high_pass_postfilter_mask_;
process_transform_->ProcessChunk(data.channels(0), nullptr);
}
void NonlinearBeamformer::PostFilter(ChannelBuffer<float>* data) {
RTC_DCHECK_EQ(data->num_frames_per_band(), chunk_length_);
// TODO(aluebs): Change to RTC_CHECK_EQ once the ChannelBuffer is updated.
RTC_DCHECK_GE(data->num_channels(), num_postfilter_channels_);
postfilter_transform_->ProcessChunk(data->channels(0), final_mask_);
// Ramp up/down for smoothing is needed in order to avoid discontinuities in
// the transitions between 10 ms frames.
const float ramp_increment =
(high_pass_postfilter_mask_ - old_high_pass_mask_) /
data->num_frames_per_band();
for (size_t i = 1; i < data->num_bands(); ++i) {
float smoothed_mask = old_high_pass_mask_;
for (size_t j = 0; j < data->num_frames_per_band(); ++j) {
smoothed_mask += ramp_increment;
for (size_t k = 0; k < num_postfilter_channels_; ++k) {
data->channels(i)[k][j] *= smoothed_mask;
}
}
}
}
void NonlinearBeamformer::AimAt(const SphericalPointf& target_direction) {
target_angle_radians_ = target_direction.azimuth();
InitHighFrequencyCorrectionRanges();
InitInterfAngles();
InitDelaySumMasks();
InitTargetCovMats();
InitInterfCovMats();
NormalizeCovMats();
}
bool NonlinearBeamformer::IsInBeam(const SphericalPointf& spherical_point) {
// If more than half-beamwidth degrees away from the beam's center,
// you are out of the beam.
return fabs(spherical_point.azimuth() - target_angle_radians_) <
kHalfBeamWidthRadians;
}
bool NonlinearBeamformer::is_target_present() {
return is_target_present_;
}
void NonlinearBeamformer::ProcessAudioBlock(const complex_f* const* input,
size_t num_input_channels,
size_t num_freq_bins,
size_t num_output_channels,
complex_f* const* output) {
RTC_CHECK_EQ(kNumFreqBins, num_freq_bins);
RTC_CHECK_EQ(num_input_channels_, num_input_channels);
RTC_CHECK_EQ(0, num_output_channels);
// Calculating the post-filter masks. Note that we need two for each
// frequency bin to account for the positive and negative interferer
// angle.
for (size_t i = low_mean_start_bin_; i <= high_mean_end_bin_; ++i) {
eig_m_.CopyFromColumn(input, i, num_input_channels_);
float eig_m_norm_factor = std::sqrt(SumSquares(eig_m_));
if (eig_m_norm_factor != 0.f) {
eig_m_.Scale(1.f / eig_m_norm_factor);
}
float rxim = Norm(target_cov_mats_[i], eig_m_);
float ratio_rxiw_rxim = 0.f;
if (rxim > 0.f) {
ratio_rxiw_rxim = rxiws_[i] / rxim;
}
complex_f rmw = abs(ConjugateDotProduct(delay_sum_masks_[i], eig_m_));
rmw *= rmw;
float rmw_r = rmw.real();
new_mask_[i] = CalculatePostfilterMask(
*interf_cov_mats_[i][0], rpsiws_[i][0], ratio_rxiw_rxim, rmw_r);
for (size_t j = 1; j < interf_angles_radians_.size(); ++j) {
float tmp_mask = CalculatePostfilterMask(
*interf_cov_mats_[i][j], rpsiws_[i][j], ratio_rxiw_rxim, rmw_r);
if (tmp_mask < new_mask_[i]) {
new_mask_[i] = tmp_mask;
}
}
}
ApplyMaskTimeSmoothing();
EstimateTargetPresence();
ApplyLowFrequencyCorrection();
ApplyHighFrequencyCorrection();
ApplyMaskFrequencySmoothing();
}
float NonlinearBeamformer::CalculatePostfilterMask(
const ComplexMatrixF& interf_cov_mat,
float rpsiw,
float ratio_rxiw_rxim,
float rmw_r) {
float rpsim = Norm(interf_cov_mat, eig_m_);
float ratio = 0.f;
if (rpsim > 0.f) {
ratio = rpsiw / rpsim;
}
float numerator = 1.f - kCutOffConstant;
if (rmw_r > 0.f) {
numerator = 1.f - std::min(kCutOffConstant, ratio / rmw_r);
}
float denominator = 1.f - kCutOffConstant;
if (ratio_rxiw_rxim > 0.f) {
denominator = 1.f - std::min(kCutOffConstant, ratio / ratio_rxiw_rxim);
}
return numerator / denominator;
}
// Smooth new_mask_ into time_smooth_mask_.
void NonlinearBeamformer::ApplyMaskTimeSmoothing() {
for (size_t i = low_mean_start_bin_; i <= high_mean_end_bin_; ++i) {
time_smooth_mask_[i] = kMaskTimeSmoothAlpha * new_mask_[i] +
(1 - kMaskTimeSmoothAlpha) * time_smooth_mask_[i];
}
}
// Copy time_smooth_mask_ to final_mask_ and smooth over frequency.
void NonlinearBeamformer::ApplyMaskFrequencySmoothing() {
// Smooth over frequency in both directions. The "frequency correction"
// regions have constant value, but we enter them to smooth over the jump
// that exists at the boundary. However, this does mean when smoothing "away"
// from the region that we only need to use the last element.
//
// Upward smoothing:
// low_mean_start_bin_
// v
// |------|------------|------|
// ^------------------>^
//
// Downward smoothing:
// high_mean_end_bin_
// v
// |------|------------|------|
// ^<------------------^
std::copy(time_smooth_mask_, time_smooth_mask_ + kNumFreqBins, final_mask_);
for (size_t i = low_mean_start_bin_; i < kNumFreqBins; ++i) {
final_mask_[i] = kMaskFrequencySmoothAlpha * final_mask_[i] +
(1 - kMaskFrequencySmoothAlpha) * final_mask_[i - 1];
}
for (size_t i = high_mean_end_bin_ + 1; i > 0; --i) {
final_mask_[i - 1] = kMaskFrequencySmoothAlpha * final_mask_[i - 1] +
(1 - kMaskFrequencySmoothAlpha) * final_mask_[i];
}
}
// Apply low frequency correction to time_smooth_mask_.
void NonlinearBeamformer::ApplyLowFrequencyCorrection() {
const float low_frequency_mask =
MaskRangeMean(low_mean_start_bin_, low_mean_end_bin_ + 1);
std::fill(time_smooth_mask_, time_smooth_mask_ + low_mean_start_bin_,
low_frequency_mask);
}
// Apply high frequency correction to time_smooth_mask_. Update
// high_pass_postfilter_mask_ to use for the high frequency time-domain bands.
void NonlinearBeamformer::ApplyHighFrequencyCorrection() {
high_pass_postfilter_mask_ =
MaskRangeMean(high_mean_start_bin_, high_mean_end_bin_ + 1);
std::fill(time_smooth_mask_ + high_mean_end_bin_ + 1,
time_smooth_mask_ + kNumFreqBins, high_pass_postfilter_mask_);
}
// Compute mean over the given range of time_smooth_mask_, [first, last).
float NonlinearBeamformer::MaskRangeMean(size_t first, size_t last) {
RTC_DCHECK_GT(last, first);
const float sum =
std::accumulate(time_smooth_mask_ + first, time_smooth_mask_ + last, 0.f);
return sum / (last - first);
}
void NonlinearBeamformer::EstimateTargetPresence() {
const size_t quantile = static_cast<size_t>(
(high_mean_end_bin_ - low_mean_start_bin_) * kMaskQuantile +
low_mean_start_bin_);
std::nth_element(new_mask_ + low_mean_start_bin_, new_mask_ + quantile,
new_mask_ + high_mean_end_bin_ + 1);
if (new_mask_[quantile] > kMaskTargetThreshold) {
is_target_present_ = true;
interference_blocks_count_ = 0;
} else {
is_target_present_ = interference_blocks_count_++ < hold_target_blocks_;
}
}
} // namespace webrtc

230
modules/audio_processing/beamformer/nonlinear_beamformer.h
View File

@ -1,230 +0,0 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_NONLINEAR_BEAMFORMER_H_
#define MODULES_AUDIO_PROCESSING_BEAMFORMER_NONLINEAR_BEAMFORMER_H_
// MSVC++ requires this to be set before any other includes to get M_PI.
#ifndef _USE_MATH_DEFINES
#define _USE_MATH_DEFINES
#endif
#include <math.h>
#include <memory>
#include <vector>
#include "common_audio/channel_buffer.h"
#include "common_audio/lapped_transform.h"
#include "modules/audio_processing/beamformer/array_util.h"
#include "modules/audio_processing/beamformer/complex_matrix.h"
namespace webrtc {
class PostFilterTransform : public LappedTransform::Callback {
public:
PostFilterTransform(size_t num_channels,
size_t chunk_length,
float* window,
size_t fft_size);
void ProcessChunk(float* const* data, float* final_mask);
protected:
void ProcessAudioBlock(const complex<float>* const* input,
size_t num_input_channels,
size_t num_freq_bins,
size_t num_output_channels,
complex<float>* const* output) override;
private:
LappedTransform transform_;
const size_t num_freq_bins_;
float* final_mask_;
};
// Enhances sound sources coming directly in front of a uniform linear array
// and suppresses sound sources coming from all other directions. Operates on
// multichannel signals and produces single-channel output.
//
// The implemented nonlinear postfilter algorithm taken from "A Robust Nonlinear
// Beamforming Postprocessor" by Bastiaan Kleijn.
class NonlinearBeamformer : public LappedTransform::Callback {
public:
static const float kHalfBeamWidthRadians;
explicit NonlinearBeamformer(
const std::vector<Point>& array_geometry,
size_t num_postfilter_channels = 1u,
SphericalPointf target_direction =
SphericalPointf(static_cast<float>(M_PI) / 2.f, 0.f, 1.f));
~NonlinearBeamformer() override;
// Sample rate corresponds to the lower band.
// Needs to be called before the NonlinearBeamformer can be used.
virtual void Initialize(int chunk_size_ms, int sample_rate_hz);
// Analyzes one time-domain chunk of audio. The audio is expected to be split
// into frequency bands inside the ChannelBuffer. The number of frames and
// channels must correspond to the constructor parameters.
virtual void AnalyzeChunk(const ChannelBuffer<float>& data);
// Applies the postfilter mask to one chunk of audio. The audio is expected to
// be split into frequency bands inside the ChannelBuffer. The number of
// frames and channels must correspond to the constructor parameters.
virtual void PostFilter(ChannelBuffer<float>* data);
virtual void AimAt(const SphericalPointf& target_direction);
virtual bool IsInBeam(const SphericalPointf& spherical_point);
// After processing each block |is_target_present_| is set to true if the
// target signal es present and to false otherwise. This methods can be called
// to know if the data is target signal or interference and process it
// accordingly.
virtual bool is_target_present();
protected:
// Process one frequency-domain block of audio. This is where the fun
// happens. Implements LappedTransform::Callback.
void ProcessAudioBlock(const complex<float>* const* input,
size_t num_input_channels,
size_t num_freq_bins,
size_t num_output_channels,
complex<float>* const* output) override;
private:
FRIEND_TEST_ALL_PREFIXES(NonlinearBeamformerTest,
InterfAnglesTakeAmbiguityIntoAccount);
typedef Matrix<float> MatrixF;
typedef ComplexMatrix<float> ComplexMatrixF;
typedef complex<float> complex_f;
void InitLowFrequencyCorrectionRanges();
void InitHighFrequencyCorrectionRanges();
void InitInterfAngles();
void InitDelaySumMasks();
void InitTargetCovMats();
void InitDiffuseCovMats();
void InitInterfCovMats();
void NormalizeCovMats();
// Calculates postfilter masks that minimize the mean squared error of our
// estimation of the desired signal.
float CalculatePostfilterMask(const ComplexMatrixF& interf_cov_mat,
float rpsiw,
float ratio_rxiw_rxim,
float rmxi_r);
// Prevents the postfilter masks from degenerating too quickly (a cause of
// musical noise).
void ApplyMaskTimeSmoothing();
void ApplyMaskFrequencySmoothing();
// The postfilter masks are unreliable at low frequencies. Calculates a better
// mask by averaging mid-low frequency values.
void ApplyLowFrequencyCorrection();
// Postfilter masks are also unreliable at high frequencies. Average mid-high
// frequency masks to calculate a single mask per block which can be applied
// in the time-domain. Further, we average these block-masks over a chunk,
// resulting in one postfilter mask per audio chunk. This allows us to skip
// both transforming and blocking the high-frequency signal.
void ApplyHighFrequencyCorrection();
// Compute the means needed for the above frequency correction.
float MaskRangeMean(size_t start_bin, size_t end_bin);
// Applies post-filter mask to |input| and store in |output|.
void ApplyPostFilter(const complex_f* input, complex_f* output);
void EstimateTargetPresence();
static const size_t kFftSize = 256;
static const size_t kNumFreqBins = kFftSize / 2 + 1;
// Deals with the fft transform and blocking.
size_t chunk_length_;
std::unique_ptr<LappedTransform> process_transform_;
std::unique_ptr<PostFilterTransform> postfilter_transform_;
float window_[kFftSize];
// Parameters exposed to the user.
const size_t num_input_channels_;
const size_t num_postfilter_channels_;
int sample_rate_hz_;
const std::vector<Point> array_geometry_;
// The normal direction of the array if it has one and it is in the xy-plane.
const absl::optional<Point> array_normal_;
// Minimum spacing between microphone pairs.
const float min_mic_spacing_;
// Calculated based on user-input and constants in the .cc file.
size_t low_mean_start_bin_;
size_t low_mean_end_bin_;
size_t high_mean_start_bin_;
size_t high_mean_end_bin_;
// Quickly varying mask updated every block.
float new_mask_[kNumFreqBins];
// Time smoothed mask.
float time_smooth_mask_[kNumFreqBins];
// Time and frequency smoothed mask.
float final_mask_[kNumFreqBins];
float target_angle_radians_;
// Angles of the interferer scenarios.
std::vector<float> interf_angles_radians_;
// The angle between the target and the interferer scenarios.
const float away_radians_;
// Array of length |kNumFreqBins|, Matrix of size |1| x |num_channels_|.
ComplexMatrixF delay_sum_masks_[kNumFreqBins];
// Arrays of length |kNumFreqBins|, Matrix of size |num_input_channels_| x
// |num_input_channels_|.
ComplexMatrixF target_cov_mats_[kNumFreqBins];
ComplexMatrixF uniform_cov_mat_[kNumFreqBins];
// Array of length |kNumFreqBins|, Matrix of size |num_input_channels_| x
// |num_input_channels_|. The vector has a size equal to the number of
// interferer scenarios.
std::vector<std::unique_ptr<ComplexMatrixF>> interf_cov_mats_[kNumFreqBins];
// Of length |kNumFreqBins|.
float wave_numbers_[kNumFreqBins];
// Preallocated for ProcessAudioBlock()
// Of length |kNumFreqBins|.
float rxiws_[kNumFreqBins];
// The vector has a size equal to the number of interferer scenarios.
std::vector<float> rpsiws_[kNumFreqBins];
// The microphone normalization factor.
ComplexMatrixF eig_m_;
// For processing the high-frequency input signal.
float high_pass_postfilter_mask_;
float old_high_pass_mask_;
// True when the target signal is present.
bool is_target_present_;
// Number of blocks after which the data is considered interference if the
// mask does not pass |kMaskSignalThreshold|.
size_t hold_target_blocks_;
// Number of blocks since the last mask that passed |kMaskSignalThreshold|.
size_t interference_blocks_count_;
};
} // namespace webrtc
#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_NONLINEAR_BEAMFORMER_H_

120
modules/audio_processing/beamformer/nonlinear_beamformer_test.cc
View File

@ -1,120 +0,0 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <vector>
#include "common_audio/channel_buffer.h"
#include "common_audio/wav_file.h"
#include "modules/audio_processing/beamformer/nonlinear_beamformer.h"
#include "modules/audio_processing/test/test_utils.h"
#include "rtc_base/checks.h"
#include "rtc_base/flags.h"
#include "rtc_base/format_macros.h"
DEFINE_string(i, "", "The name of the input file to read from.");
DEFINE_string(o, "out.wav", "Name of the output file to write to.");
DEFINE_string(mic_positions,
"",
"Space delimited cartesian coordinates of microphones in meters. "
"The coordinates of each point are contiguous. "
"For a two element array: \"x1 y1 z1 x2 y2 z2\"");
DEFINE_bool(help, false, "Prints this message.");
namespace webrtc {
namespace {
const int kChunksPerSecond = 100;
const int kChunkSizeMs = 1000 / kChunksPerSecond;
const char kUsage[] =
"Command-line tool to run beamforming on WAV files. The signal is passed\n"
"in as a single band, unlike the audio processing interface which splits\n"
"signals into multiple bands.\n";
std::vector<Point> ParseArrayGeometry(const std::string& mic_positions) {
const std::vector<float> values = ParseList<float>(mic_positions);
const size_t num_mics =
rtc::CheckedDivExact(values.size(), static_cast<size_t>(3));
RTC_CHECK_GT(num_mics, 0) << "mic_positions is not large enough.";
std::vector<Point> result;
result.reserve(num_mics);
for (size_t i = 0; i < values.size(); i += 3) {
result.push_back(Point(values[i + 0], values[i + 1], values[i + 2]));
}
return result;
}
std::vector<Point> ParseArrayGeometry(const std::string& mic_positions,
size_t num_mics) {
std::vector<Point> result = ParseArrayGeometry(mic_positions);
RTC_CHECK_EQ(result.size(), num_mics)
<< "Could not parse mic_positions or incorrect number of points.";
return result;
}
} // namespace
int main(int argc, char* argv[]) {
if (rtc::FlagList::SetFlagsFromCommandLine(&argc, argv, true) || FLAG_help ||
argc != 1) {
printf("%s", kUsage);
if (FLAG_help) {
rtc::FlagList::Print(nullptr, false);
return 0;
}
return 1;
}
WavReader in_file(FLAG_i);
WavWriter out_file(FLAG_o, in_file.sample_rate(), in_file.num_channels());
const size_t num_mics = in_file.num_channels();
const std::vector<Point> array_geometry =
ParseArrayGeometry(FLAG_mic_positions, num_mics);
RTC_CHECK_EQ(array_geometry.size(), num_mics);
NonlinearBeamformer bf(array_geometry, array_geometry.size());
bf.Initialize(kChunkSizeMs, in_file.sample_rate());
printf("Input file: %s\nChannels: %" PRIuS ", Sample rate: %d Hz\n\n", FLAG_i,
in_file.num_channels(), in_file.sample_rate());
printf("Output file: %s\nChannels: %" PRIuS ", Sample rate: %d Hz\n\n",
FLAG_o, out_file.num_channels(), out_file.sample_rate());
ChannelBuffer<float> buf(
rtc::CheckedDivExact(in_file.sample_rate(), kChunksPerSecond),
in_file.num_channels());
std::vector<float> interleaved(buf.size());
while (in_file.ReadSamples(interleaved.size(), &interleaved[0]) ==
interleaved.size()) {
FloatS16ToFloat(&interleaved[0], interleaved.size(), &interleaved[0]);
Deinterleave(&interleaved[0], buf.num_frames(), buf.num_channels(),
buf.channels());
bf.AnalyzeChunk(buf);
bf.PostFilter(&buf);
Interleave(buf.channels(), buf.num_frames(), buf.num_channels(),
&interleaved[0]);
FloatToFloatS16(&interleaved[0], interleaved.size(), &interleaved[0]);
out_file.WriteSamples(&interleaved[0], interleaved.size());
}
return 0;
}
} // namespace webrtc
int main(int argc, char* argv[]) {
return webrtc::main(argc, argv);
}

365
modules/audio_processing/beamformer/nonlinear_beamformer_unittest.cc
View File

@ -1,365 +0,0 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
// MSVC++ requires this to be set before any other includes to get M_PI.
#define _USE_MATH_DEFINES
#include "modules/audio_processing/beamformer/nonlinear_beamformer.h"
#include <math.h>
#include "api/array_view.h"
#include "modules/audio_processing/audio_buffer.h"
#include "modules/audio_processing/test/audio_buffer_tools.h"
#include "modules/audio_processing/test/bitexactness_tools.h"
#include "test/gtest.h"
namespace webrtc {
namespace {
const int kChunkSizeMs = 10;
const int kSampleRateHz = 16000;
SphericalPointf AzimuthToSphericalPoint(float azimuth_radians) {
return SphericalPointf(azimuth_radians, 0.f, 1.f);
}
void Verify(NonlinearBeamformer* bf, float target_azimuth_radians) {
EXPECT_TRUE(bf->IsInBeam(AzimuthToSphericalPoint(target_azimuth_radians)));
EXPECT_TRUE(bf->IsInBeam(AzimuthToSphericalPoint(
target_azimuth_radians - NonlinearBeamformer::kHalfBeamWidthRadians +
0.001f)));
EXPECT_TRUE(bf->IsInBeam(AzimuthToSphericalPoint(
target_azimuth_radians + NonlinearBeamformer::kHalfBeamWidthRadians -
0.001f)));
EXPECT_FALSE(bf->IsInBeam(AzimuthToSphericalPoint(
target_azimuth_radians - NonlinearBeamformer::kHalfBeamWidthRadians -
0.001f)));
EXPECT_FALSE(bf->IsInBeam(AzimuthToSphericalPoint(
target_azimuth_radians + NonlinearBeamformer::kHalfBeamWidthRadians +
0.001f)));
}
void AimAndVerify(NonlinearBeamformer* bf, float target_azimuth_radians) {
bf->AimAt(AzimuthToSphericalPoint(target_azimuth_radians));
Verify(bf, target_azimuth_radians);
}
// Bitexactness test code.
const size_t kNumFramesToProcess = 1000;
void ProcessOneFrame(int sample_rate_hz,
AudioBuffer* capture_audio_buffer,
NonlinearBeamformer* beamformer) {
if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) {
capture_audio_buffer->SplitIntoFrequencyBands();
}
beamformer->AnalyzeChunk(*capture_audio_buffer->split_data_f());
capture_audio_buffer->set_num_channels(1);
beamformer->PostFilter(capture_audio_buffer->split_data_f());
if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) {
capture_audio_buffer->MergeFrequencyBands();
}
}
int BeamformerSampleRate(int sample_rate_hz) {
return (sample_rate_hz > AudioProcessing::kSampleRate16kHz
? AudioProcessing::kSampleRate16kHz
: sample_rate_hz);
}
void RunBitExactnessTest(int sample_rate_hz,
const std::vector<Point>& array_geometry,
const SphericalPointf& target_direction,
rtc::ArrayView<const float> output_reference) {
NonlinearBeamformer beamformer(array_geometry, 1u, target_direction);
beamformer.Initialize(AudioProcessing::kChunkSizeMs,
BeamformerSampleRate(sample_rate_hz));
const StreamConfig capture_config(sample_rate_hz, array_geometry.size(),
false);
AudioBuffer capture_buffer(
capture_config.num_frames(), capture_config.num_channels(),
capture_config.num_frames(), capture_config.num_channels(),
capture_config.num_frames());
test::InputAudioFile capture_file(
test::GetApmCaptureTestVectorFileName(sample_rate_hz));
std::vector<float> capture_input(capture_config.num_frames() *
capture_config.num_channels());
for (size_t frame_no = 0u; frame_no < kNumFramesToProcess; ++frame_no) {
ReadFloatSamplesFromStereoFile(capture_config.num_frames(),
capture_config.num_channels(), &capture_file,
capture_input);
test::CopyVectorToAudioBuffer(capture_config, capture_input,
&capture_buffer);
ProcessOneFrame(sample_rate_hz, &capture_buffer, &beamformer);
}
// Extract and verify the test results.
std::vector<float> capture_output;
test::ExtractVectorFromAudioBuffer(capture_config, &capture_buffer,
&capture_output);
const float kElementErrorBound = 1.f / static_cast<float>(1 << 15);
// Compare the output with the reference. Only the first values of the output
// from last frame processed are compared in order not having to specify all
// preceeding frames as testvectors. As the algorithm being tested has a
// memory, testing only the last frame implicitly also tests the preceeding
// frames.
EXPECT_TRUE(test::VerifyDeinterleavedArray(
capture_config.num_frames(), capture_config.num_channels(),
output_reference, capture_output, kElementErrorBound));
}
// TODO(peah): Add bitexactness tests for scenarios with more than 2 input
// channels.
std::vector<Point> CreateArrayGeometry(int variant) {
std::vector<Point> array_geometry;
switch (variant) {
case 1:
array_geometry.push_back(Point(-0.025f, 0.f, 0.f));
array_geometry.push_back(Point(0.025f, 0.f, 0.f));
break;
case 2:
array_geometry.push_back(Point(-0.035f, 0.f, 0.f));
array_geometry.push_back(Point(0.035f, 0.f, 0.f));
break;
case 3:
array_geometry.push_back(Point(-0.5f, 0.f, 0.f));
array_geometry.push_back(Point(0.5f, 0.f, 0.f));
break;
default:
RTC_CHECK(false);
}
return array_geometry;
}
const SphericalPointf TargetDirection1(0.4f * static_cast<float>(M_PI) / 2.f,
0.f,
1.f);
const SphericalPointf TargetDirection2(static_cast<float>(M_PI) / 2.f,
1.f,
2.f);
} // namespace
TEST(NonlinearBeamformerTest, AimingModifiesBeam) {
std::vector<Point> array_geometry;
array_geometry.push_back(Point(-0.025f, 0.f, 0.f));
array_geometry.push_back(Point(0.025f, 0.f, 0.f));
NonlinearBeamformer bf(array_geometry, 1u);
bf.Initialize(kChunkSizeMs, kSampleRateHz);
// The default constructor parameter sets the target angle to PI / 2.
Verify(&bf, static_cast<float>(M_PI) / 2.f);
AimAndVerify(&bf, static_cast<float>(M_PI) / 3.f);
AimAndVerify(&bf, 3.f * static_cast<float>(M_PI) / 4.f);
AimAndVerify(&bf, static_cast<float>(M_PI) / 6.f);
AimAndVerify(&bf, static_cast<float>(M_PI));
}
TEST(NonlinearBeamformerTest, InterfAnglesTakeAmbiguityIntoAccount) {
{
// For linear arrays there is ambiguity.
std::vector<Point> array_geometry;
array_geometry.push_back(Point(-0.1f, 0.f, 0.f));
array_geometry.push_back(Point(0.f, 0.f, 0.f));
array_geometry.push_back(Point(0.2f, 0.f, 0.f));
NonlinearBeamformer bf(array_geometry, 1u);
bf.Initialize(kChunkSizeMs, kSampleRateHz);
EXPECT_EQ(2u, bf.interf_angles_radians_.size());
EXPECT_FLOAT_EQ(M_PI / 2.f - bf.away_radians_,
bf.interf_angles_radians_[0]);
EXPECT_FLOAT_EQ(M_PI / 2.f + bf.away_radians_,
bf.interf_angles_radians_[1]);
bf.AimAt(AzimuthToSphericalPoint(bf.away_radians_ / 2.f));
EXPECT_EQ(2u, bf.interf_angles_radians_.size());
EXPECT_FLOAT_EQ(M_PI - bf.away_radians_ / 2.f,
bf.interf_angles_radians_[0]);
EXPECT_FLOAT_EQ(3.f * bf.away_radians_ / 2.f, bf.interf_angles_radians_[1]);
}
{
// For planar arrays with normal in the xy-plane there is ambiguity.
std::vector<Point> array_geometry;
array_geometry.push_back(Point(-0.1f, 0.f, 0.f));
array_geometry.push_back(Point(0.f, 0.f, 0.f));
array_geometry.push_back(Point(0.2f, 0.f, 0.f));
array_geometry.push_back(Point(0.1f, 0.f, 0.2f));
array_geometry.push_back(Point(0.f, 0.f, -0.1f));
NonlinearBeamformer bf(array_geometry, 1u);
bf.Initialize(kChunkSizeMs, kSampleRateHz);
EXPECT_EQ(2u, bf.interf_angles_radians_.size());
EXPECT_FLOAT_EQ(M_PI / 2.f - bf.away_radians_,
bf.interf_angles_radians_[0]);
EXPECT_FLOAT_EQ(M_PI / 2.f + bf.away_radians_,
bf.interf_angles_radians_[1]);
bf.AimAt(AzimuthToSphericalPoint(bf.away_radians_ / 2.f));
EXPECT_EQ(2u, bf.interf_angles_radians_.size());
EXPECT_FLOAT_EQ(M_PI - bf.away_radians_ / 2.f,
bf.interf_angles_radians_[0]);
EXPECT_FLOAT_EQ(3.f * bf.away_radians_ / 2.f, bf.interf_angles_radians_[1]);
}
{
// For planar arrays with normal not in the xy-plane there is no ambiguity.
std::vector<Point> array_geometry;
array_geometry.push_back(Point(0.f, 0.f, 0.f));
array_geometry.push_back(Point(0.2f, 0.f, 0.f));
array_geometry.push_back(Point(0.f, 0.1f, -0.2f));
NonlinearBeamformer bf(array_geometry, 1u);
bf.Initialize(kChunkSizeMs, kSampleRateHz);
EXPECT_EQ(2u, bf.interf_angles_radians_.size());
EXPECT_FLOAT_EQ(M_PI / 2.f - bf.away_radians_,
bf.interf_angles_radians_[0]);
EXPECT_FLOAT_EQ(M_PI / 2.f + bf.away_radians_,
bf.interf_angles_radians_[1]);
bf.AimAt(AzimuthToSphericalPoint(bf.away_radians_ / 2.f));
EXPECT_EQ(2u, bf.interf_angles_radians_.size());
EXPECT_FLOAT_EQ(-bf.away_radians_ / 2.f, bf.interf_angles_radians_[0]);
EXPECT_FLOAT_EQ(3.f * bf.away_radians_ / 2.f, bf.interf_angles_radians_[1]);
}
{
// For arrays which are not linear or planar there is no ambiguity.
std::vector<Point> array_geometry;
array_geometry.push_back(Point(0.f, 0.f, 0.f));
array_geometry.push_back(Point(0.1f, 0.f, 0.f));
array_geometry.push_back(Point(0.f, 0.2f, 0.f));
array_geometry.push_back(Point(0.f, 0.f, 0.3f));
NonlinearBeamformer bf(array_geometry, 1u);
bf.Initialize(kChunkSizeMs, kSampleRateHz);
EXPECT_EQ(2u, bf.interf_angles_radians_.size());
EXPECT_FLOAT_EQ(M_PI / 2.f - bf.away_radians_,
bf.interf_angles_radians_[0]);
EXPECT_FLOAT_EQ(M_PI / 2.f + bf.away_radians_,
bf.interf_angles_radians_[1]);
bf.AimAt(AzimuthToSphericalPoint(bf.away_radians_ / 2.f));
EXPECT_EQ(2u, bf.interf_angles_radians_.size());
EXPECT_FLOAT_EQ(-bf.away_radians_ / 2.f, bf.interf_angles_radians_[0]);
EXPECT_FLOAT_EQ(3.f * bf.away_radians_ / 2.f, bf.interf_angles_radians_[1]);
}
}
// TODO(peah): Investigate why the nonlinear_beamformer.cc causes a DCHECK in
// this setup.
TEST(BeamformerBitExactnessTest,
DISABLED_Stereo8kHz_ArrayGeometry1_TargetDirection1) {
const float kOutputReference[] = {0.001318f, -0.001091f, 0.000990f,
0.001318f, -0.001091f, 0.000990f};
RunBitExactnessTest(AudioProcessing::kSampleRate8kHz, CreateArrayGeometry(1),
TargetDirection1, kOutputReference);
}
TEST(BeamformerBitExactnessTest, Stereo16kHz_ArrayGeometry1_TargetDirection1) {
const float kOutputReference[] = {-0.000077f, -0.000147f, -0.000138f,
-0.000077f, -0.000147f, -0.000138f};
RunBitExactnessTest(AudioProcessing::kSampleRate16kHz, CreateArrayGeometry(1),
TargetDirection1, kOutputReference);
}
TEST(BeamformerBitExactnessTest, Stereo32kHz_ArrayGeometry1_TargetDirection1) {
const float kOutputReference[] = {-0.000061f, -0.000061f, -0.000061f,
-0.000061f, -0.000061f, -0.000061f};
RunBitExactnessTest(AudioProcessing::kSampleRate32kHz, CreateArrayGeometry(1),
TargetDirection1, kOutputReference);
}
TEST(BeamformerBitExactnessTest, Stereo48kHz_ArrayGeometry1_TargetDirection1) {
const float kOutputReference[] = {0.000450f, 0.000436f, 0.000433f,
0.000450f, 0.000436f, 0.000433f};
RunBitExactnessTest(AudioProcessing::kSampleRate48kHz, CreateArrayGeometry(1),
TargetDirection1, kOutputReference);
}
// TODO(peah): Investigate why the nonlinear_beamformer.cc causes a DCHECK in
// this setup.
TEST(BeamformerBitExactnessTest,
DISABLED_Stereo8kHz_ArrayGeometry1_TargetDirection2) {
const float kOutputReference[] = {0.001144f, -0.001026f, 0.001074f,
-0.016205f, -0.007324f, -0.015656f};
RunBitExactnessTest(AudioProcessing::kSampleRate8kHz, CreateArrayGeometry(1),
TargetDirection2, kOutputReference);
}
TEST(BeamformerBitExactnessTest, Stereo16kHz_ArrayGeometry1_TargetDirection2) {
const float kOutputReference[] = {0.000221f, -0.000249f, 0.000140f,
0.000221f, -0.000249f, 0.000140f};
RunBitExactnessTest(AudioProcessing::kSampleRate16kHz, CreateArrayGeometry(1),
TargetDirection2, kOutputReference);
}
TEST(BeamformerBitExactnessTest, Stereo32kHz_ArrayGeometry1_TargetDirection2) {
const float kOutputReference[] = {0.000763f, -0.000336f, 0.000549f,
0.000763f, -0.000336f, 0.000549f};
RunBitExactnessTest(AudioProcessing::kSampleRate32kHz, CreateArrayGeometry(1),
TargetDirection2, kOutputReference);
}
TEST(BeamformerBitExactnessTest, Stereo48kHz_ArrayGeometry1_TargetDirection2) {
const float kOutputReference[] = {-0.000004f, -0.000494f, 0.000255f,
-0.000004f, -0.000494f, 0.000255f};
RunBitExactnessTest(AudioProcessing::kSampleRate48kHz, CreateArrayGeometry(1),
TargetDirection2, kOutputReference);
}
TEST(BeamformerBitExactnessTest, Stereo8kHz_ArrayGeometry2_TargetDirection2) {
const float kOutputReference[] = {-0.000914f, 0.002170f, -0.002382f,
-0.000914f, 0.002170f, -0.002382f};
RunBitExactnessTest(AudioProcessing::kSampleRate8kHz, CreateArrayGeometry(2),
TargetDirection2, kOutputReference);
}
TEST(BeamformerBitExactnessTest, Stereo16kHz_ArrayGeometry2_TargetDirection2) {
const float kOutputReference[] = {0.000179f, -0.000179f, 0.000081f,
0.000179f, -0.000179f, 0.000081f};
RunBitExactnessTest(AudioProcessing::kSampleRate16kHz, CreateArrayGeometry(2),
TargetDirection2, kOutputReference);
}
TEST(BeamformerBitExactnessTest, Stereo32kHz_ArrayGeometry2_TargetDirection2) {
const float kOutputReference[] = {0.000549f, -0.000214f, 0.000366f,
0.000549f, -0.000214f, 0.000366f};
RunBitExactnessTest(AudioProcessing::kSampleRate32kHz, CreateArrayGeometry(2),
TargetDirection2, kOutputReference);
}
TEST(BeamformerBitExactnessTest, Stereo48kHz_ArrayGeometry2_TargetDirection2) {
const float kOutputReference[] = {0.000019f, -0.000310f, 0.000182f,
0.000019f, -0.000310f, 0.000182f};
RunBitExactnessTest(AudioProcessing::kSampleRate48kHz, CreateArrayGeometry(2),
TargetDirection2, kOutputReference);
}
// TODO(peah): Investigate why the nonlinear_beamformer.cc causes a DCHECK in
// this setup.
TEST(BeamformerBitExactnessTest,
DISABLED_Stereo16kHz_ArrayGeometry3_TargetDirection1) {
const float kOutputReference[] = {-0.000161f, 0.000171f, -0.000096f,
0.001007f, 0.000427f, 0.000977f};
RunBitExactnessTest(AudioProcessing::kSampleRate16kHz, CreateArrayGeometry(3),
TargetDirection1, kOutputReference);
}
} // namespace webrtc

19
modules/audio_processing/include/audio_processing.cc
View File

@ -14,25 +14,6 @@
namespace webrtc {
Beamforming::Beamforming()
: enabled(false),
array_geometry(),
target_direction(
SphericalPointf(static_cast<float>(M_PI) / 2.f, 0.f, 1.f)) {}
Beamforming::Beamforming(bool enabled, const std::vector<Point>& array_geometry)
: Beamforming(enabled,
array_geometry,
SphericalPointf(static_cast<float>(M_PI) / 2.f, 0.f, 1.f)) {}
Beamforming::Beamforming(bool enabled,
const std::vector<Point>& array_geometry,
SphericalPointf target_direction)
: enabled(enabled),
array_geometry(array_geometry),
target_direction(target_direction) {}
Beamforming::~Beamforming() {}
void CustomProcessing::SetRuntimeSetting(
AudioProcessing::RuntimeSetting setting) {}

24
modules/audio_processing/include/audio_processing.h
View File

@ -25,7 +25,6 @@
#include "absl/types/optional.h"
#include "api/audio/echo_canceller3_config.h"
#include "api/audio/echo_control.h"
#include "modules/audio_processing/beamformer/array_util.h"
#include "modules/audio_processing/include/audio_generator.h"
#include "modules/audio_processing/include/audio_processing_statistics.h"
#include "modules/audio_processing/include/config.h"
@ -44,8 +43,6 @@ class AecDump;
class AudioBuffer;
class AudioFrame;
class NonlinearBeamformer;
class StreamConfig;
class ProcessingConfig;
@ -147,22 +144,6 @@ struct ExperimentalNs {
bool enabled;
};
// Use to enable beamforming. Must be provided through the constructor. It will
// have no impact if used with AudioProcessing::SetExtraOptions().
struct Beamforming {
Beamforming();
Beamforming(bool enabled, const std::vector<Point>& array_geometry);
Beamforming(bool enabled,
const std::vector<Point>& array_geometry,
SphericalPointf target_direction);
~Beamforming();
static const ConfigOptionID identifier = ConfigOptionID::kBeamforming;
const bool enabled;
const std::vector<Point> array_geometry;
const SphericalPointf target_direction;
};
// Use to enable intelligibility enhancer in audio processing.
//
// Note: If enabled and the reverse stream has more than one output channel,
@ -673,10 +654,6 @@ class AudioProcessingBuilder {
// The AudioProcessingBuilder takes ownership of the render_pre_processing.
AudioProcessingBuilder& SetRenderPreProcessing(
std::unique_ptr<CustomProcessing> render_pre_processing);
// The AudioProcessingBuilder takes ownership of the nonlinear beamformer.
RTC_DEPRECATED
AudioProcessingBuilder& SetNonlinearBeamformer(
std::unique_ptr<NonlinearBeamformer> nonlinear_beamformer);
// The AudioProcessingBuilder takes ownership of the echo_detector.
AudioProcessingBuilder& SetEchoDetector(
rtc::scoped_refptr<EchoDetector> echo_detector);
@ -689,7 +666,6 @@ class AudioProcessingBuilder {
std::unique_ptr<EchoControlFactory> echo_control_factory_;
std::unique_ptr<CustomProcessing> capture_post_processing_;
std::unique_ptr<CustomProcessing> render_pre_processing_;
std::unique_ptr<NonlinearBeamformer> nonlinear_beamformer_;
rtc::scoped_refptr<EchoDetector> echo_detector_;
RTC_DISALLOW_COPY_AND_ASSIGN(AudioProcessingBuilder);
};

1
modules/audio_processing/test/audio_processing_simulator.h
View File

@ -56,7 +56,6 @@ struct SimulationSettings {
absl::optional<bool> use_hpf;
absl::optional<bool> use_ns;
absl::optional<bool> use_ts;
absl::optional<bool> use_bf;
absl::optional<bool> use_ie;
absl::optional<bool> use_vad;
absl::optional<bool> use_le;

4
test/fuzzers/audio_processing_configs_fuzzer.cc
View File

@ -46,7 +46,6 @@ std::unique_ptr<AudioProcessing> CreateApm(test::FuzzDataHelper* fuzz_data,
// configurable public components of APM.
bool exp_agc = fuzz_data->ReadOrDefaultValue(true);
bool exp_ns = fuzz_data->ReadOrDefaultValue(true);
bool bf = fuzz_data->ReadOrDefaultValue(true);
bool ef = fuzz_data->ReadOrDefaultValue(true);
bool raf = fuzz_data->ReadOrDefaultValue(true);
static_cast<void>(fuzz_data->ReadOrDefaultValue(true));
@ -105,9 +104,6 @@ std::unique_ptr<AudioProcessing> CreateApm(test::FuzzDataHelper* fuzz_data,
config.Set<ExperimentalAgc>(new ExperimentalAgc(exp_agc));
config.Set<ExperimentalNs>(new ExperimentalNs(exp_ns));
if (bf) {
config.Set<Beamforming>(new Beamforming());
}
config.Set<ExtendedFilter>(new ExtendedFilter(ef));
config.Set<RefinedAdaptiveFilter>(new RefinedAdaptiveFilter(raf));
config.Set<DelayAgnostic>(new DelayAgnostic(true));