diff --git a/webrtc/common_audio/BUILD.gn b/webrtc/common_audio/BUILD.gn index 0103655ba9..b01b31816b 100644 --- a/webrtc/common_audio/BUILD.gn +++ b/webrtc/common_audio/BUILD.gn @@ -21,7 +21,11 @@ source_set("common_audio") { sources = [ "audio_converter.cc", "audio_converter.h", + "audio_ring_buffer.cc", + "audio_ring_buffer.h", "audio_util.cc", + "blocker.cc", + "blocker.h", "channel_buffer.cc", "channel_buffer.h", "fft4g.c", @@ -31,6 +35,8 @@ source_set("common_audio") { "fir_filter_neon.h", "fir_filter_sse.h", "include/audio_util.h", + "lapped_transform.cc", + "lapped_transform.h", "real_fourier.cc", "real_fourier.h", "real_fourier_ooura.cc", @@ -43,6 +49,8 @@ source_set("common_audio") { "resampler/resampler.cc", "resampler/sinc_resampler.cc", "resampler/sinc_resampler.h", + "ring_buffer.c", + "ring_buffer.h", "signal_processing/auto_corr_to_refl_coef.c", "signal_processing/auto_correlation.c", "signal_processing/complex_fft_tables.h", diff --git a/webrtc/common_audio/DEPS b/webrtc/common_audio/DEPS index 2b59d2365b..7df03ea3fe 100644 --- a/webrtc/common_audio/DEPS +++ b/webrtc/common_audio/DEPS @@ -1,6 +1,5 @@ include_rules = [ "+dl/sp/api", # For openmax_dl. "+webrtc/base", - "+webrtc/modules/audio_processing", "+webrtc/system_wrappers", ] diff --git a/webrtc/modules/audio_processing/utility/audio_ring_buffer.cc b/webrtc/common_audio/audio_ring_buffer.cc similarity index 94% rename from webrtc/modules/audio_processing/utility/audio_ring_buffer.cc rename to webrtc/common_audio/audio_ring_buffer.cc index 73f578f58b..a29e53a61c 100644 --- a/webrtc/modules/audio_processing/utility/audio_ring_buffer.cc +++ b/webrtc/common_audio/audio_ring_buffer.cc @@ -8,10 +8,10 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "webrtc/modules/audio_processing/utility/audio_ring_buffer.h" +#include "webrtc/common_audio/audio_ring_buffer.h" #include "webrtc/base/checks.h" -#include "webrtc/modules/audio_processing/utility/ring_buffer.h" +#include "webrtc/common_audio/ring_buffer.h" // This is a simple multi-channel wrapper over the ring_buffer.h C interface. diff --git a/webrtc/common_audio/audio_ring_buffer.h b/webrtc/common_audio/audio_ring_buffer.h index 820d1273b8..ae825a3cd0 100644 --- a/webrtc/common_audio/audio_ring_buffer.h +++ b/webrtc/common_audio/audio_ring_buffer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -10,7 +10,46 @@ #ifndef WEBRTC_COMMON_AUDIO_AUDIO_RING_BUFFER_H_ #define WEBRTC_COMMON_AUDIO_AUDIO_RING_BUFFER_H_ -// TODO(peah): Remove as soon as all downstream dependencies are resolved. -#include "webrtc/modules/audio_processing/utility/audio_ring_buffer.h" +#include +#include + +struct RingBuffer; + +namespace webrtc { + +// A ring buffer tailored for float deinterleaved audio. Any operation that +// cannot be performed as requested will cause a crash (e.g. insufficient data +// in the buffer to fulfill a read request.) +class AudioRingBuffer final { + public: + // Specify the number of channels and maximum number of frames the buffer will + // contain. + AudioRingBuffer(size_t channels, size_t max_frames); + ~AudioRingBuffer(); + + // Copies |data| to the buffer and advances the write pointer. |channels| must + // be the same as at creation time. + void Write(const float* const* data, size_t channels, size_t frames); + + // Copies from the buffer to |data| and advances the read pointer. |channels| + // must be the same as at creation time. + void Read(float* const* data, size_t channels, size_t frames); + + size_t ReadFramesAvailable() const; + size_t WriteFramesAvailable() const; + + // Moves the read position. The forward version advances the read pointer + // towards the write pointer and the backward verison withdraws the read + // pointer away from the write pointer (i.e. flushing and stuffing the buffer + // respectively.) + void MoveReadPositionForward(size_t frames); + void MoveReadPositionBackward(size_t frames); + + private: + // TODO(kwiberg): Use std::vector> instead. + std::vector buffers_; +}; + +} // namespace webrtc #endif // WEBRTC_COMMON_AUDIO_AUDIO_RING_BUFFER_H_ diff --git a/webrtc/modules/audio_processing/utility/audio_ring_buffer_unittest.cc b/webrtc/common_audio/audio_ring_buffer_unittest.cc similarity index 97% rename from webrtc/modules/audio_processing/utility/audio_ring_buffer_unittest.cc rename to webrtc/common_audio/audio_ring_buffer_unittest.cc index c2d5e7a64c..c5c38de56d 100644 --- a/webrtc/modules/audio_processing/utility/audio_ring_buffer_unittest.cc +++ b/webrtc/common_audio/audio_ring_buffer_unittest.cc @@ -9,9 +9,8 @@ */ #include -#include -#include "webrtc/modules/audio_processing/utility/audio_ring_buffer.h" +#include "webrtc/common_audio/audio_ring_buffer.h" #include "testing/gtest/include/gtest/gtest.h" #include "webrtc/common_audio/channel_buffer.h" diff --git a/webrtc/modules/audio_processing/utility/blocker.cc b/webrtc/common_audio/blocker.cc similarity index 99% rename from webrtc/modules/audio_processing/utility/blocker.cc rename to webrtc/common_audio/blocker.cc index a3661cc156..13432f2e7a 100644 --- a/webrtc/modules/audio_processing/utility/blocker.cc +++ b/webrtc/common_audio/blocker.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "webrtc/modules/audio_processing/utility/blocker.h" +#include "webrtc/common_audio/blocker.h" #include diff --git a/webrtc/common_audio/blocker.h b/webrtc/common_audio/blocker.h index 2ce2e930f3..edf81d337a 100644 --- a/webrtc/common_audio/blocker.h +++ b/webrtc/common_audio/blocker.h @@ -1,17 +1,124 @@ - /* - * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ -#ifndef WEBRTC_COMMON_AUDIO_BLOCKER_H_ -#define WEBRTC_COMMON_AUDIO_BLOCKER_H_ +#ifndef WEBRTC_INTERNAL_BEAMFORMER_BLOCKER_H_ +#define WEBRTC_INTERNAL_BEAMFORMER_BLOCKER_H_ -// TODO(peah): Remove as soon as all downstream dependencies are resolved. -#include "webrtc/modules/audio_processing/utility/blocker.h" +#include -#endif // WEBRTC_COMMON_AUDIO_BLOCKER_H_ +#include "webrtc/common_audio/audio_ring_buffer.h" +#include "webrtc/common_audio/channel_buffer.h" + +namespace webrtc { + +// The callback function to process audio in the time domain. Input has already +// been windowed, and output will be windowed. The number of input channels +// must be >= the number of output channels. +class BlockerCallback { + public: + virtual ~BlockerCallback() {} + + virtual void ProcessBlock(const float* const* input, + size_t num_frames, + size_t num_input_channels, + size_t num_output_channels, + float* const* output) = 0; +}; + +// The main purpose of Blocker is to abstract away the fact that often we +// receive a different number of audio frames than our transform takes. For +// example, most FFTs work best when the fft-size is a power of 2, but suppose +// we receive 20ms of audio at a sample rate of 48000. That comes to 960 frames +// of audio, which is not a power of 2. Blocker allows us to specify the +// transform and all other necessary processing via the Process() callback +// function without any constraints on the transform-size +// (read: |block_size_|) or received-audio-size (read: |chunk_size_|). +// We handle this for the multichannel audio case, allowing for different +// numbers of input and output channels (for example, beamforming takes 2 or +// more input channels and returns 1 output channel). Audio signals are +// represented as deinterleaved floats in the range [-1, 1]. +// +// Blocker is responsible for: +// - blocking audio while handling potential discontinuities on the edges +// of chunks +// - windowing blocks before sending them to Process() +// - windowing processed blocks, and overlap-adding them together before +// sending back a processed chunk +// +// To use blocker: +// 1. Impelment a BlockerCallback object |bc|. +// 2. Instantiate a Blocker object |b|, passing in |bc|. +// 3. As you receive audio, call b.ProcessChunk() to get processed audio. +// +// A small amount of delay is added to the first received chunk to deal with +// the difference in chunk/block sizes. This delay is <= chunk_size. +// +// Ownership of window is retained by the caller. That is, Blocker makes a +// copy of window and does not attempt to delete it. +class Blocker { + public: + Blocker(size_t chunk_size, + size_t block_size, + size_t num_input_channels, + size_t num_output_channels, + const float* window, + size_t shift_amount, + BlockerCallback* callback); + + void ProcessChunk(const float* const* input, + size_t chunk_size, + size_t num_input_channels, + size_t num_output_channels, + float* const* output); + + private: + const size_t chunk_size_; + const size_t block_size_; + const size_t num_input_channels_; + const size_t num_output_channels_; + + // The number of frames of delay to add at the beginning of the first chunk. + const size_t initial_delay_; + + // The frame index into the input buffer where the first block should be read + // from. This is necessary because shift_amount_ is not necessarily a + // multiple of chunk_size_, so blocks won't line up at the start of the + // buffer. + size_t frame_offset_; + + // Since blocks nearly always overlap, there are certain blocks that require + // frames from the end of one chunk and the beginning of the next chunk. The + // input and output buffers are responsible for saving those frames between + // calls to ProcessChunk(). + // + // Both contain |initial delay| + |chunk_size| frames. The input is a fairly + // standard FIFO, but due to the overlap-add it's harder to use an + // AudioRingBuffer for the output. + AudioRingBuffer input_buffer_; + ChannelBuffer output_buffer_; + + // Space for the input block (can't wrap because of windowing). + ChannelBuffer input_block_; + + // Space for the output block (can't wrap because of overlap/add). + ChannelBuffer output_block_; + + std::unique_ptr window_; + + // The amount of frames between the start of contiguous blocks. For example, + // |shift_amount_| = |block_size_| / 2 for a Hann window. + size_t shift_amount_; + + BlockerCallback* callback_; +}; + +} // namespace webrtc + +#endif // WEBRTC_INTERNAL_BEAMFORMER_BLOCKER_H_ diff --git a/webrtc/modules/audio_processing/utility/blocker_unittest.cc b/webrtc/common_audio/blocker_unittest.cc similarity index 99% rename from webrtc/modules/audio_processing/utility/blocker_unittest.cc rename to webrtc/common_audio/blocker_unittest.cc index 3f53f68660..eea3e2516a 100644 --- a/webrtc/modules/audio_processing/utility/blocker_unittest.cc +++ b/webrtc/common_audio/blocker_unittest.cc @@ -10,7 +10,7 @@ #include -#include "webrtc/modules/audio_processing/utility/blocker.h" +#include "webrtc/common_audio/blocker.h" #include "testing/gtest/include/gtest/gtest.h" #include "webrtc/base/arraysize.h" diff --git a/webrtc/common_audio/common_audio.gyp b/webrtc/common_audio/common_audio.gyp index 4a6377a4da..57d9f1ca64 100644 --- a/webrtc/common_audio/common_audio.gyp +++ b/webrtc/common_audio/common_audio.gyp @@ -31,8 +31,10 @@ 'sources': [ 'audio_converter.cc', 'audio_converter.h', + 'audio_ring_buffer.cc', 'audio_ring_buffer.h', 'audio_util.cc', + 'blocker.cc', 'blocker.h', 'channel_buffer.cc', 'channel_buffer.h', @@ -43,6 +45,7 @@ 'fir_filter_neon.h', 'fir_filter_sse.h', 'include/audio_util.h', + 'lapped_transform.cc', 'lapped_transform.h', 'real_fourier.cc', 'real_fourier.h', @@ -56,6 +59,8 @@ 'resampler/resampler.cc', 'resampler/sinc_resampler.cc', 'resampler/sinc_resampler.h', + 'ring_buffer.c', + 'ring_buffer.h', 'signal_processing/include/real_fft.h', 'signal_processing/include/signal_processing_library.h', 'signal_processing/include/spl_inl.h', @@ -235,14 +240,18 @@ ], 'sources': [ 'audio_converter_unittest.cc', + 'audio_ring_buffer_unittest.cc', 'audio_util_unittest.cc', + 'blocker_unittest.cc', 'fir_filter_unittest.cc', + 'lapped_transform_unittest.cc', 'real_fourier_unittest.cc', 'resampler/resampler_unittest.cc', 'resampler/push_resampler_unittest.cc', 'resampler/push_sinc_resampler_unittest.cc', 'resampler/sinusoidal_linear_chirp_source.cc', 'resampler/sinusoidal_linear_chirp_source.h', + 'ring_buffer_unittest.cc', 'signal_processing/real_fft_unittest.cc', 'signal_processing/signal_processing_unittest.cc', 'sparse_fir_filter_unittest.cc', diff --git a/webrtc/modules/audio_processing/utility/lapped_transform.cc b/webrtc/common_audio/lapped_transform.cc similarity index 98% rename from webrtc/modules/audio_processing/utility/lapped_transform.cc rename to webrtc/common_audio/lapped_transform.cc index cb5496dfc4..0edf586d78 100644 --- a/webrtc/modules/audio_processing/utility/lapped_transform.cc +++ b/webrtc/common_audio/lapped_transform.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "webrtc/modules/audio_processing/utility/lapped_transform.h" +#include "webrtc/common_audio/lapped_transform.h" #include #include diff --git a/webrtc/common_audio/lapped_transform.h b/webrtc/common_audio/lapped_transform.h index 18b77685a3..832735991b 100644 --- a/webrtc/common_audio/lapped_transform.h +++ b/webrtc/common_audio/lapped_transform.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -11,7 +11,115 @@ #ifndef WEBRTC_COMMON_AUDIO_LAPPED_TRANSFORM_H_ #define WEBRTC_COMMON_AUDIO_LAPPED_TRANSFORM_H_ -// TODO(peah): Remove as soon as all downstream dependencies are resolved. -#include "webrtc/modules/audio_processing/utility/lapped_transform.h +#include +#include + +#include "webrtc/common_audio/blocker.h" +#include "webrtc/common_audio/real_fourier.h" +#include "webrtc/system_wrappers/include/aligned_array.h" + +namespace webrtc { + +// Helper class for audio processing modules which operate on frequency domain +// input derived from the windowed time domain audio stream. +// +// The input audio chunk is sliced into possibly overlapping blocks, multiplied +// by a window and transformed with an FFT implementation. The transformed data +// is supplied to the given callback for processing. The processed output is +// then inverse transformed into the time domain and spliced back into a chunk +// which constitutes the final output of this processing module. +class LappedTransform { + public: + class Callback { + public: + virtual ~Callback() {} + + virtual void ProcessAudioBlock(const std::complex* const* in_block, + size_t num_in_channels, size_t frames, + size_t num_out_channels, + std::complex* const* out_block) = 0; + }; + + // Construct a transform instance. |chunk_length| is the number of samples in + // each channel. |window| defines the window, owned by the caller (a copy is + // made internally); |window| should have length equal to |block_length|. + // |block_length| defines the length of a block, in samples. + // |shift_amount| is in samples. |callback| is the caller-owned audio + // processing function called for each block of the input chunk. + LappedTransform(size_t num_in_channels, + size_t num_out_channels, + size_t chunk_length, + const float* window, + size_t block_length, + size_t shift_amount, + Callback* callback); + ~LappedTransform() {} + + // Main audio processing helper method. Internally slices |in_chunk| into + // blocks, transforms them to frequency domain, calls the callback for each + // block and returns a de-blocked time domain chunk of audio through + // |out_chunk|. Both buffers are caller-owned. + void ProcessChunk(const float* const* in_chunk, float* const* out_chunk); + + // Get the chunk length. + // + // The chunk length is the number of samples per channel that must be passed + // to ProcessChunk via the parameter in_chunk. + // + // Returns the same chunk_length passed to the LappedTransform constructor. + size_t chunk_length() const { return chunk_length_; } + + // Get the number of input channels. + // + // This is the number of arrays that must be passed to ProcessChunk via + // in_chunk. + // + // Returns the same num_in_channels passed to the LappedTransform constructor. + size_t num_in_channels() const { return num_in_channels_; } + + // Get the number of output channels. + // + // This is the number of arrays that must be passed to ProcessChunk via + // out_chunk. + // + // Returns the same num_out_channels passed to the LappedTransform + // constructor. + size_t num_out_channels() const { return num_out_channels_; } + + private: + // Internal middleware callback, given to the blocker. Transforms each block + // and hands it over to the processing method given at construction time. + class BlockThunk : public BlockerCallback { + public: + explicit BlockThunk(LappedTransform* parent) : parent_(parent) {} + + virtual void ProcessBlock(const float* const* input, + size_t num_frames, + size_t num_input_channels, + size_t num_output_channels, + float* const* output); + + private: + LappedTransform* const parent_; + } blocker_callback_; + + const size_t num_in_channels_; + const size_t num_out_channels_; + + const size_t block_length_; + const size_t chunk_length_; + + Callback* const block_processor_; + Blocker blocker_; + + std::unique_ptr fft_; + const size_t cplx_length_; + AlignedArray real_buf_; + AlignedArray > cplx_pre_; + AlignedArray > cplx_post_; +}; + +} // namespace webrtc #endif // WEBRTC_COMMON_AUDIO_LAPPED_TRANSFORM_H_ + diff --git a/webrtc/modules/audio_processing/utility/lapped_transform_unittest.cc b/webrtc/common_audio/lapped_transform_unittest.cc similarity index 98% rename from webrtc/modules/audio_processing/utility/lapped_transform_unittest.cc rename to webrtc/common_audio/lapped_transform_unittest.cc index f6c8ebd6d3..a78488e326 100644 --- a/webrtc/modules/audio_processing/utility/lapped_transform_unittest.cc +++ b/webrtc/common_audio/lapped_transform_unittest.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "webrtc/modules/audio_processing/utility/lapped_transform.h" +#include "webrtc/common_audio/lapped_transform.h" #include #include diff --git a/webrtc/modules/audio_processing/utility/ring_buffer.c b/webrtc/common_audio/ring_buffer.c similarity index 99% rename from webrtc/modules/audio_processing/utility/ring_buffer.c rename to webrtc/common_audio/ring_buffer.c index c71bac1862..60fb5dff20 100644 --- a/webrtc/modules/audio_processing/utility/ring_buffer.c +++ b/webrtc/common_audio/ring_buffer.c @@ -11,7 +11,7 @@ // A ring buffer to hold arbitrary data. Provides no thread safety. Unless // otherwise specified, functions return 0 on success and -1 on error. -#include "webrtc/modules/audio_processing/utility/ring_buffer.h" +#include "webrtc/common_audio/ring_buffer.h" #include // size_t #include diff --git a/webrtc/modules/audio_processing/utility/ring_buffer.h b/webrtc/common_audio/ring_buffer.h similarity index 92% rename from webrtc/modules/audio_processing/utility/ring_buffer.h rename to webrtc/common_audio/ring_buffer.h index a46c262229..4125c48d01 100644 --- a/webrtc/modules/audio_processing/utility/ring_buffer.h +++ b/webrtc/common_audio/ring_buffer.h @@ -11,8 +11,8 @@ // A ring buffer to hold arbitrary data. Provides no thread safety. Unless // otherwise specified, functions return 0 on success and -1 on error. -#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_UTILITY_RING_BUFFER_H_ -#define WEBRTC_MODULES_AUDIO_PROCESSING_UTILITY_RING_BUFFER_H_ +#ifndef WEBRTC_COMMON_AUDIO_RING_BUFFER_H_ +#define WEBRTC_COMMON_AUDIO_RING_BUFFER_H_ #ifdef __cplusplus extern "C" { @@ -63,4 +63,4 @@ size_t WebRtc_available_write(const RingBuffer* handle); } #endif -#endif // WEBRTC_MODULES_AUDIO_PROCESSING_UTILITY_RING_BUFFER_H_ +#endif // WEBRTC_COMMON_AUDIO_RING_BUFFER_H_ diff --git a/webrtc/modules/audio_processing/utility/ring_buffer_unittest.cc b/webrtc/common_audio/ring_buffer_unittest.cc similarity index 98% rename from webrtc/modules/audio_processing/utility/ring_buffer_unittest.cc rename to webrtc/common_audio/ring_buffer_unittest.cc index 7972657727..92c470a02d 100644 --- a/webrtc/modules/audio_processing/utility/ring_buffer_unittest.cc +++ b/webrtc/common_audio/ring_buffer_unittest.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "webrtc/modules/audio_processing/utility/ring_buffer.h" +#include "webrtc/common_audio/ring_buffer.h" #include #include diff --git a/webrtc/modules/audio_processing/BUILD.gn b/webrtc/modules/audio_processing/BUILD.gn index bf7bdc5d1c..22c904d52f 100644 --- a/webrtc/modules/audio_processing/BUILD.gn +++ b/webrtc/modules/audio_processing/BUILD.gn @@ -108,21 +108,13 @@ source_set("audio_processing") { "transient/wpd_tree.h", "typing_detection.cc", "typing_detection.h", - "utility/audio_ring_buffer.cc", - "utility/audio_ring_buffer.h", "utility/block_mean_calculator.cc", "utility/block_mean_calculator.h", - "utility/blocker.cc", - "utility/blocker.h", "utility/delay_estimator.c", "utility/delay_estimator.h", "utility/delay_estimator_internal.h", "utility/delay_estimator_wrapper.c", "utility/delay_estimator_wrapper.h", - "utility/lapped_transform.cc", - "utility/lapped_transform.h", - "utility/ring_buffer.c", - "utility/ring_buffer.h", "vad/common.h", "vad/gmm.cc", "vad/gmm.h", diff --git a/webrtc/modules/audio_processing/aec/aec_core.cc b/webrtc/modules/audio_processing/aec/aec_core.cc index 4fe635d2db..e23a79312b 100644 --- a/webrtc/modules/audio_processing/aec/aec_core.cc +++ b/webrtc/modules/audio_processing/aec/aec_core.cc @@ -24,6 +24,9 @@ #include #include +extern "C" { +#include "webrtc/common_audio/ring_buffer.h" +} #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h" #include "webrtc/modules/audio_processing/aec/aec_common.h" #include "webrtc/modules/audio_processing/aec/aec_core_internal.h" @@ -33,7 +36,6 @@ extern "C" { #include "webrtc/modules/audio_processing/logging/aec_logging.h" extern "C" { #include "webrtc/modules/audio_processing/utility/delay_estimator_wrapper.h" -#include "webrtc/modules/audio_processing/utility/ring_buffer.h" } #include "webrtc/system_wrappers/include/cpu_features_wrapper.h" #include "webrtc/typedefs.h" diff --git a/webrtc/modules/audio_processing/aec/aec_core_internal.h b/webrtc/modules/audio_processing/aec/aec_core_internal.h index 1f4f99be63..ea5889f503 100644 --- a/webrtc/modules/audio_processing/aec/aec_core_internal.h +++ b/webrtc/modules/audio_processing/aec/aec_core_internal.h @@ -11,14 +11,13 @@ #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_AEC_AEC_CORE_INTERNAL_H_ #define WEBRTC_MODULES_AUDIO_PROCESSING_AEC_AEC_CORE_INTERNAL_H_ +extern "C" { +#include "webrtc/common_audio/ring_buffer.h" +} #include "webrtc/common_audio/wav_file.h" #include "webrtc/modules/audio_processing/aec/aec_common.h" #include "webrtc/modules/audio_processing/aec/aec_core.h" #include "webrtc/modules/audio_processing/utility/block_mean_calculator.h" -extern "C" { -#include "webrtc/modules/audio_processing/utility/ring_buffer.h" -} - #include "webrtc/typedefs.h" namespace webrtc { diff --git a/webrtc/modules/audio_processing/aec/echo_cancellation.cc b/webrtc/modules/audio_processing/aec/echo_cancellation.cc index 493068d9bf..32496ca33c 100644 --- a/webrtc/modules/audio_processing/aec/echo_cancellation.cc +++ b/webrtc/modules/audio_processing/aec/echo_cancellation.cc @@ -21,14 +21,12 @@ #include extern "C" { +#include "webrtc/common_audio/ring_buffer.h" #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h" } #include "webrtc/modules/audio_processing/aec/aec_core.h" #include "webrtc/modules/audio_processing/aec/aec_resampler.h" #include "webrtc/modules/audio_processing/aec/echo_cancellation_internal.h" -extern "C" { -#include "webrtc/modules/audio_processing/utility/ring_buffer.h" -} #include "webrtc/typedefs.h" namespace webrtc { diff --git a/webrtc/modules/audio_processing/aec/echo_cancellation_internal.h b/webrtc/modules/audio_processing/aec/echo_cancellation_internal.h index 537ab5d904..b4a6fd8390 100644 --- a/webrtc/modules/audio_processing/aec/echo_cancellation_internal.h +++ b/webrtc/modules/audio_processing/aec/echo_cancellation_internal.h @@ -11,10 +11,10 @@ #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_AEC_ECHO_CANCELLATION_INTERNAL_H_ #define WEBRTC_MODULES_AUDIO_PROCESSING_AEC_ECHO_CANCELLATION_INTERNAL_H_ -#include "webrtc/modules/audio_processing/aec/aec_core.h" extern "C" { -#include "webrtc/modules/audio_processing/utility/ring_buffer.h" +#include "webrtc/common_audio/ring_buffer.h" } +#include "webrtc/modules/audio_processing/aec/aec_core.h" namespace webrtc { diff --git a/webrtc/modules/audio_processing/aecm/aecm_core.c b/webrtc/modules/audio_processing/aecm/aecm_core.c index 084feb858b..6bf1cf7f3e 100644 --- a/webrtc/modules/audio_processing/aecm/aecm_core.c +++ b/webrtc/modules/audio_processing/aecm/aecm_core.c @@ -14,10 +14,10 @@ #include #include +#include "webrtc/common_audio/ring_buffer.h" #include "webrtc/common_audio/signal_processing/include/real_fft.h" #include "webrtc/modules/audio_processing/aecm/echo_control_mobile.h" #include "webrtc/modules/audio_processing/utility/delay_estimator_wrapper.h" -#include "webrtc/modules/audio_processing/utility/ring_buffer.h" #include "webrtc/system_wrappers/include/compile_assert_c.h" #include "webrtc/system_wrappers/include/cpu_features_wrapper.h" #include "webrtc/typedefs.h" diff --git a/webrtc/modules/audio_processing/aecm/aecm_core.h b/webrtc/modules/audio_processing/aecm/aecm_core.h index 9b70f47503..b52bb62d2d 100644 --- a/webrtc/modules/audio_processing/aecm/aecm_core.h +++ b/webrtc/modules/audio_processing/aecm/aecm_core.h @@ -13,9 +13,9 @@ #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_AECM_AECM_CORE_H_ #define WEBRTC_MODULES_AUDIO_PROCESSING_AECM_AECM_CORE_H_ +#include "webrtc/common_audio/ring_buffer.h" #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h" #include "webrtc/modules/audio_processing/aecm/aecm_defines.h" -#include "webrtc/modules/audio_processing/utility/ring_buffer.h" #include "webrtc/typedefs.h" #ifdef _MSC_VER // visual c++ diff --git a/webrtc/modules/audio_processing/aecm/aecm_core_c.c b/webrtc/modules/audio_processing/aecm/aecm_core_c.c index ea5f2bc8cf..3a8fafa4ec 100644 --- a/webrtc/modules/audio_processing/aecm/aecm_core_c.c +++ b/webrtc/modules/audio_processing/aecm/aecm_core_c.c @@ -14,10 +14,10 @@ #include #include +#include "webrtc/common_audio/ring_buffer.h" #include "webrtc/common_audio/signal_processing/include/real_fft.h" #include "webrtc/modules/audio_processing/aecm/echo_control_mobile.h" #include "webrtc/modules/audio_processing/utility/delay_estimator_wrapper.h" -#include "webrtc/modules/audio_processing/utility/ring_buffer.h" #include "webrtc/system_wrappers/include/compile_assert_c.h" #include "webrtc/system_wrappers/include/cpu_features_wrapper.h" #include "webrtc/typedefs.h" @@ -768,3 +768,4 @@ static void ComfortNoise(AecmCore* aecm, out[i].imag = WebRtcSpl_AddSatW16(out[i].imag, uImag[i]); } } + diff --git a/webrtc/modules/audio_processing/aecm/echo_control_mobile.c b/webrtc/modules/audio_processing/aecm/echo_control_mobile.c index d58ee3e456..91e6f0e80c 100644 --- a/webrtc/modules/audio_processing/aecm/echo_control_mobile.c +++ b/webrtc/modules/audio_processing/aecm/echo_control_mobile.c @@ -15,9 +15,9 @@ #endif #include +#include "webrtc/common_audio/ring_buffer.h" #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h" #include "webrtc/modules/audio_processing/aecm/aecm_core.h" -#include "webrtc/modules/audio_processing/utility/ring_buffer.h" #define BUF_SIZE_FRAMES 50 // buffer size (frames) // Maximum length of resampled signal. Must be an integer multiple of frames diff --git a/webrtc/modules/audio_processing/audio_processing.gypi b/webrtc/modules/audio_processing/audio_processing.gypi index 75da59f3e0..264f3e5bef 100644 --- a/webrtc/modules/audio_processing/audio_processing.gypi +++ b/webrtc/modules/audio_processing/audio_processing.gypi @@ -118,21 +118,13 @@ 'transient/wpd_tree.h', 'typing_detection.cc', 'typing_detection.h', - 'utility/audio_ring_buffer.cc', - 'utility/audio_ring_buffer.h', 'utility/block_mean_calculator.cc', 'utility/block_mean_calculator.h', - 'utility/blocker.cc', - 'utility/blocker.h', 'utility/delay_estimator.c', 'utility/delay_estimator.h', 'utility/delay_estimator_internal.h', 'utility/delay_estimator_wrapper.c', 'utility/delay_estimator_wrapper.h', - 'utility/lapped_transform.cc', - 'utility/lapped_transform.h', - 'utility/ring_buffer.c', - 'utility/ring_buffer.h', 'vad/common.h', 'vad/gmm.cc', 'vad/gmm.h', diff --git a/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h b/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h index c1f9188bfc..b8953b0a4f 100644 --- a/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h +++ b/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h @@ -19,10 +19,11 @@ #include #include +#include "webrtc/common_audio/lapped_transform.h" #include "webrtc/common_audio/channel_buffer.h" #include "webrtc/modules/audio_processing/beamformer/beamformer.h" #include "webrtc/modules/audio_processing/beamformer/complex_matrix.h" -#include "webrtc/modules/audio_processing/utility/lapped_transform.h" + namespace webrtc { // Enhances sound sources coming directly in front of a uniform linear array diff --git a/webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h b/webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h index 728e4cfb70..1413212934 100644 --- a/webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h +++ b/webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h @@ -16,10 +16,10 @@ #include #include "webrtc/base/swap_queue.h" +#include "webrtc/common_audio/lapped_transform.h" #include "webrtc/common_audio/channel_buffer.h" #include "webrtc/modules/audio_processing/intelligibility/intelligibility_utils.h" #include "webrtc/modules/audio_processing/render_queue_item_verifier.h" -#include "webrtc/modules/audio_processing/utility/lapped_transform.h" #include "webrtc/modules/audio_processing/vad/voice_activity_detector.h" namespace webrtc { diff --git a/webrtc/modules/audio_processing/utility/audio_ring_buffer.h b/webrtc/modules/audio_processing/utility/audio_ring_buffer.h deleted file mode 100644 index 8f9758770e..0000000000 --- a/webrtc/modules/audio_processing/utility/audio_ring_buffer.h +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ -#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_UTILITY_AUDIO_RING_BUFFER_H_ -#define WEBRTC_MODULES_AUDIO_PROCESSING_UTILITY_AUDIO_RING_BUFFER_H_ - -#include -#include - -struct RingBuffer; - -namespace webrtc { - -// A ring buffer tailored for float deinterleaved audio. Any operation that -// cannot be performed as requested will cause a crash (e.g. insufficient data -// in the buffer to fulfill a read request.) -class AudioRingBuffer final { - public: - // Specify the number of channels and maximum number of frames the buffer will - // contain. - AudioRingBuffer(size_t channels, size_t max_frames); - ~AudioRingBuffer(); - - // Copies |data| to the buffer and advances the write pointer. |channels| must - // be the same as at creation time. - void Write(const float* const* data, size_t channels, size_t frames); - - // Copies from the buffer to |data| and advances the read pointer. |channels| - // must be the same as at creation time. - void Read(float* const* data, size_t channels, size_t frames); - - size_t ReadFramesAvailable() const; - size_t WriteFramesAvailable() const; - - // Moves the read position. The forward version advances the read pointer - // towards the write pointer and the backward verison withdraws the read - // pointer away from the write pointer (i.e. flushing and stuffing the buffer - // respectively.) - void MoveReadPositionForward(size_t frames); - void MoveReadPositionBackward(size_t frames); - - private: - // TODO(kwiberg): Use std::vector> instead. - std::vector buffers_; -}; - -} // namespace webrtc - -#endif // WEBRTC_MODULES_AUDIO_PROCESSING_UTILITY_AUDIO_RING_BUFFER_H_ diff --git a/webrtc/modules/audio_processing/utility/blocker.h b/webrtc/modules/audio_processing/utility/blocker.h deleted file mode 100644 index 7d9bf66e48..0000000000 --- a/webrtc/modules/audio_processing/utility/blocker.h +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_UTILITY_BLOCKER_H_ -#define WEBRTC_MODULES_AUDIO_PROCESSING_UTILITY_BLOCKER_H_ - -#include - -#include "webrtc/common_audio/channel_buffer.h" -#include "webrtc/modules/audio_processing/utility/audio_ring_buffer.h" - -namespace webrtc { - -// The callback function to process audio in the time domain. Input has already -// been windowed, and output will be windowed. The number of input channels -// must be >= the number of output channels. -class BlockerCallback { - public: - virtual ~BlockerCallback() {} - - virtual void ProcessBlock(const float* const* input, - size_t num_frames, - size_t num_input_channels, - size_t num_output_channels, - float* const* output) = 0; -}; - -// The main purpose of Blocker is to abstract away the fact that often we -// receive a different number of audio frames than our transform takes. For -// example, most FFTs work best when the fft-size is a power of 2, but suppose -// we receive 20ms of audio at a sample rate of 48000. That comes to 960 frames -// of audio, which is not a power of 2. Blocker allows us to specify the -// transform and all other necessary processing via the Process() callback -// function without any constraints on the transform-size -// (read: |block_size_|) or received-audio-size (read: |chunk_size_|). -// We handle this for the multichannel audio case, allowing for different -// numbers of input and output channels (for example, beamforming takes 2 or -// more input channels and returns 1 output channel). Audio signals are -// represented as deinterleaved floats in the range [-1, 1]. -// -// Blocker is responsible for: -// - blocking audio while handling potential discontinuities on the edges -// of chunks -// - windowing blocks before sending them to Process() -// - windowing processed blocks, and overlap-adding them together before -// sending back a processed chunk -// -// To use blocker: -// 1. Impelment a BlockerCallback object |bc|. -// 2. Instantiate a Blocker object |b|, passing in |bc|. -// 3. As you receive audio, call b.ProcessChunk() to get processed audio. -// -// A small amount of delay is added to the first received chunk to deal with -// the difference in chunk/block sizes. This delay is <= chunk_size. -// -// Ownership of window is retained by the caller. That is, Blocker makes a -// copy of window and does not attempt to delete it. -class Blocker { - public: - Blocker(size_t chunk_size, - size_t block_size, - size_t num_input_channels, - size_t num_output_channels, - const float* window, - size_t shift_amount, - BlockerCallback* callback); - - void ProcessChunk(const float* const* input, - size_t chunk_size, - size_t num_input_channels, - size_t num_output_channels, - float* const* output); - - private: - const size_t chunk_size_; - const size_t block_size_; - const size_t num_input_channels_; - const size_t num_output_channels_; - - // The number of frames of delay to add at the beginning of the first chunk. - const size_t initial_delay_; - - // The frame index into the input buffer where the first block should be read - // from. This is necessary because shift_amount_ is not necessarily a - // multiple of chunk_size_, so blocks won't line up at the start of the - // buffer. - size_t frame_offset_; - - // Since blocks nearly always overlap, there are certain blocks that require - // frames from the end of one chunk and the beginning of the next chunk. The - // input and output buffers are responsible for saving those frames between - // calls to ProcessChunk(). - // - // Both contain |initial delay| + |chunk_size| frames. The input is a fairly - // standard FIFO, but due to the overlap-add it's harder to use an - // AudioRingBuffer for the output. - AudioRingBuffer input_buffer_; - ChannelBuffer output_buffer_; - - // Space for the input block (can't wrap because of windowing). - ChannelBuffer input_block_; - - // Space for the output block (can't wrap because of overlap/add). - ChannelBuffer output_block_; - - std::unique_ptr window_; - - // The amount of frames between the start of contiguous blocks. For example, - // |shift_amount_| = |block_size_| / 2 for a Hann window. - size_t shift_amount_; - - BlockerCallback* callback_; -}; - -} // namespace webrtc - -#endif // WEBRTC_MODULES_AUDIO_PROCESSING_UTILITY_BLOCKER_H_ diff --git a/webrtc/modules/audio_processing/utility/lapped_transform.h b/webrtc/modules/audio_processing/utility/lapped_transform.h deleted file mode 100644 index 1286ecf2d8..0000000000 --- a/webrtc/modules/audio_processing/utility/lapped_transform.h +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_UTILITY_LAPPED_TRANSFORM_H_ -#define WEBRTC_MODULES_AUDIO_PROCESSING_UTILITY_LAPPED_TRANSFORM_H_ - -#include -#include - -#include "webrtc/common_audio/real_fourier.h" -#include "webrtc/modules/audio_processing/utility/blocker.h" -#include "webrtc/system_wrappers/include/aligned_array.h" - -namespace webrtc { - -// Helper class for audio processing modules which operate on frequency domain -// input derived from the windowed time domain audio stream. -// -// The input audio chunk is sliced into possibly overlapping blocks, multiplied -// by a window and transformed with an FFT implementation. The transformed data -// is supplied to the given callback for processing. The processed output is -// then inverse transformed into the time domain and spliced back into a chunk -// which constitutes the final output of this processing module. -class LappedTransform { - public: - class Callback { - public: - virtual ~Callback() {} - - virtual void ProcessAudioBlock(const std::complex* const* in_block, - size_t num_in_channels, size_t frames, - size_t num_out_channels, - std::complex* const* out_block) = 0; - }; - - // Construct a transform instance. |chunk_length| is the number of samples in - // each channel. |window| defines the window, owned by the caller (a copy is - // made internally); |window| should have length equal to |block_length|. - // |block_length| defines the length of a block, in samples. - // |shift_amount| is in samples. |callback| is the caller-owned audio - // processing function called for each block of the input chunk. - LappedTransform(size_t num_in_channels, - size_t num_out_channels, - size_t chunk_length, - const float* window, - size_t block_length, - size_t shift_amount, - Callback* callback); - ~LappedTransform() {} - - // Main audio processing helper method. Internally slices |in_chunk| into - // blocks, transforms them to frequency domain, calls the callback for each - // block and returns a de-blocked time domain chunk of audio through - // |out_chunk|. Both buffers are caller-owned. - void ProcessChunk(const float* const* in_chunk, float* const* out_chunk); - - // Get the chunk length. - // - // The chunk length is the number of samples per channel that must be passed - // to ProcessChunk via the parameter in_chunk. - // - // Returns the same chunk_length passed to the LappedTransform constructor. - size_t chunk_length() const { return chunk_length_; } - - // Get the number of input channels. - // - // This is the number of arrays that must be passed to ProcessChunk via - // in_chunk. - // - // Returns the same num_in_channels passed to the LappedTransform constructor. - size_t num_in_channels() const { return num_in_channels_; } - - // Get the number of output channels. - // - // This is the number of arrays that must be passed to ProcessChunk via - // out_chunk. - // - // Returns the same num_out_channels passed to the LappedTransform - // constructor. - size_t num_out_channels() const { return num_out_channels_; } - - private: - // Internal middleware callback, given to the blocker. Transforms each block - // and hands it over to the processing method given at construction time. - class BlockThunk : public BlockerCallback { - public: - explicit BlockThunk(LappedTransform* parent) : parent_(parent) {} - - virtual void ProcessBlock(const float* const* input, - size_t num_frames, - size_t num_input_channels, - size_t num_output_channels, - float* const* output); - - private: - LappedTransform* const parent_; - } blocker_callback_; - - const size_t num_in_channels_; - const size_t num_out_channels_; - - const size_t block_length_; - const size_t chunk_length_; - - Callback* const block_processor_; - Blocker blocker_; - - std::unique_ptr fft_; - const size_t cplx_length_; - AlignedArray real_buf_; - AlignedArray > cplx_pre_; - AlignedArray > cplx_post_; -}; - -} // namespace webrtc - -#endif // WEBRTC_MODULES_AUDIO_PROCESSING_UTILITY_LAPPED_TRANSFORM_H_ diff --git a/webrtc/modules/modules.gyp b/webrtc/modules/modules.gyp index eb886a9e4a..d5df853905 100644 --- a/webrtc/modules/modules.gyp +++ b/webrtc/modules/modules.gyp @@ -258,12 +258,8 @@ 'audio_processing/transient/transient_suppressor_unittest.cc', 'audio_processing/transient/wpd_node_unittest.cc', 'audio_processing/transient/wpd_tree_unittest.cc', - 'audio_processing/utility/audio_ring_buffer_unittest.cc', 'audio_processing/utility/block_mean_calculator_unittest.cc', - 'audio_processing/utility/blocker_unittest.cc', 'audio_processing/utility/delay_estimator_unittest.cc', - 'audio_processing/utility/lapped_transform_unittest.cc', - 'audio_processing/utility/ring_buffer_unittest.cc', 'audio_processing/vad/gmm_unittest.cc', 'audio_processing/vad/pitch_based_vad_unittest.cc', 'audio_processing/vad/pitch_internal_unittest.cc',