Revert "Reland "ObjC ADM: record/play implementation via RTCAudioDevice [3/3]""
This reverts commit 9a0a6a198e8e247884fe01d7e0aa6bd425721c14. Reason for revert: Breaks upstream project Original change's description: > Reland "ObjC ADM: record/play implementation via RTCAudioDevice [3/3]" > > This is a reland of commit 2b9aaad58f56744f5c573c3b918fe072566598a5 > > Original change's description: > > ObjC ADM: record/play implementation via RTCAudioDevice [3/3] > > > > # Overview > > This CL chain exposes new API from ObjC WebRTC SDK to inject custom > > means to play and record audio. The goal of CLs is achieved by having > > additional implementation of `webrtc::AudioDeviceModule` > > called `ObjCAudioDeviceModule`. The feature > > of `ObjCAudioDeviceModule` is that it does not directly use any > > of OS-provided audio APIs like AudioUnit, AVAudioEngine, AudioQueue, > > AVCaptureSession etc. Instead it delegates communication with specific > > system audio API to user-injectable audio device instance which > > implements `RTCAudioDevice` protocol. > > `RTCAudioDevice` is new API added to ObC WebRTC SDK in the CL chain. > > > > # AudioDeviceBuffer > > `ObjCAudioDeviceModule` does conform to heavy `AudioDeviceModule` > > interface providing stubs for unrelated methods. It also implements > > common low-level management of audio device buffer, which glues audio > > PCM flow to/from WebRTC. > > `ObjCAudioDeviceModule` owns single `webrtc::AudioDeviceBuffer` which > > with the help of two `FineAudioBuffer` (one for recording and one for > > playout) is exchanged audio PCMs with user-provided `RTCAudioDevice` > > instance. > > `webrtc::AudioDeviceBuffer` is configured to work with specific audio: > > it has to know sample rate and channels count of audio being played and > > recorded. These formats could be different between playout and > > recording. `ObjCAudioDeviceModule` stores current audio parameters > > applied to `webrtc::AudioDeviceBuffer` as fields of > > type `webrtc::AudioParameters`. `RTCAudioDevice` has it's own variable > > audio parameters like sample rate, channels count and IO buffer > > duration. The audio parameters of `RTCAudioDevice` must be kept in sync > > with audio parameters applied to `webrtc::AudioDeviceBuffer`, otherwise > > audio playout and recording will be corrupted: audio is sent only > > partially over the wire and/or audio is played with artifacts. > > `ObjCAudioDeviceModule` reads current `RTCAudioDevice` audio parameters > > when playout or recording is initialized. Whenever `RTCAudioDevice` > > audio parameters parameters are changed, there must be a notification to > > `ObjCAudioDeviceModule` to allow it to reconfigure > > it's `webrtc::AudioDeviceBuffer`. The notification is performed > > via `RTCAudioDeviceDelegate` object, which is provided > > by `ObjCAudioDeviceModule` during initialization of `RTCAudioDevice`. > > > > # Threading > > `ObjCAudioDeviceModule` is stick to same thread between initialization > > and termination. The only exception is two IO functions invoked by SDK > > user code presumably from real-time audio IO thread. > > Implementation of `RTCAudioDevice` may rely on the fact that all the > > methods of `RTCAudioDevice` are called on the same thread between > > initialization and termination. `ObjCAudioDeviceModule` is also expect > > that the implementation of `RTCAudioDevice` will call methods related > > to notification of audio parameters changes and audio interruption are > > invoked on `ObjCAudioDeviceModule` thread. To facilitate this > > requirement `RTCAudioDeviceDelegate` provides two functions to execute > > sync and async block on `ObjCAudioDeviceModule` thread. > > Async block could be useful when handling audio session notifications to > > dispatch whole block re-configuring audio objects used > > by `RTCAudioDevice` implementation. > > Sync block could be used to make sure changes to audio parameters > > of ADB owned by `ObjCAudioDeviceModule` are notified, before interrupted > > playout/recording restarted. > > > > Bug: webrtc:14193 > > Change-Id: I5587ec6bbee3cf02bad70dd59b822feb0ada7f86 > > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/269006 > > Reviewed-by: Henrik Andreasson <henrika@google.com> > > Commit-Queue: Yury Yarashevich <yura.yaroshevich@gmail.com> > > Reviewed-by: Peter Hanspers <peterhanspers@webrtc.org> > > Reviewed-by: Henrik Andreassson <henrika@webrtc.org> > > Reviewed-by: Tomas Gunnarsson <tommi@webrtc.org> > > Cr-Commit-Position: refs/heads/main@{#37928} > > Bug: webrtc:14193 > Change-Id: Iaf950d24bb2394a20e50421d5122f72ce46ae840 > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/273380 > Commit-Queue: Tomas Gunnarsson <tommi@webrtc.org> > Reviewed-by: Tomas Gunnarsson <tommi@webrtc.org> > Cr-Commit-Position: refs/heads/main@{#37946} Bug: webrtc:14193 Change-Id: I5e18cc919ca4bb1cef7d5a11489451a0907f0d66 No-Presubmit: true No-Tree-Checks: true No-Try: true Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/273486 Bot-Commit: rubber-stamper@appspot.gserviceaccount.com <rubber-stamper@appspot.gserviceaccount.com> Owners-Override: Andrey Logvin <landrey@google.com> Commit-Queue: Andrey Logvin <landrey@google.com> Cr-Commit-Position: refs/heads/main@{#37950}
This commit is contained in:
Andrey Logvin
committed by
WebRTC LUCI CQ
WebRTC LUCI CQ
parent
2d7db71fda
commit
bcc31826ab
@ -11,25 +11,17 @@
|
||||
#ifndef SDK_OBJC_NATIVE_SRC_OBJC_AUDIO_DEVICE_H_
|
||||
#define SDK_OBJC_NATIVE_SRC_OBJC_AUDIO_DEVICE_H_
|
||||
|
||||
#include <memory>
|
||||
|
||||
#import "components/audio/RTCAudioDevice.h"
|
||||
|
||||
#include "modules/audio_device/audio_device_buffer.h"
|
||||
#include "modules/audio_device/include/audio_device.h"
|
||||
#include "rtc_base/thread.h"
|
||||
|
||||
@class ObjCAudioDeviceDelegate;
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
class FineAudioBuffer;
|
||||
|
||||
namespace objc_adm {
|
||||
|
||||
class ObjCAudioDeviceModule : public AudioDeviceModule {
|
||||
public:
|
||||
explicit ObjCAudioDeviceModule(id<RTC_OBJC_TYPE(RTCAudioDevice)> audio_device);
|
||||
explicit ObjCAudioDeviceModule(
|
||||
id<RTC_OBJC_TYPE(RTCAudioDevice)> audio_device);
|
||||
~ObjCAudioDeviceModule() override;
|
||||
|
||||
// Retrieve the currently utilized audio layer
|
||||
@ -134,140 +126,8 @@ class ObjCAudioDeviceModule : public AudioDeviceModule {
|
||||
int GetRecordAudioParameters(AudioParameters* params) const override;
|
||||
#endif // WEBRTC_IOS
|
||||
|
||||
public:
|
||||
OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags,
|
||||
const AudioTimeStamp* time_stamp,
|
||||
NSInteger bus_number,
|
||||
UInt32 num_frames,
|
||||
const AudioBufferList* io_data,
|
||||
void* render_context,
|
||||
RTC_OBJC_TYPE(RTCAudioDeviceRenderRecordedDataBlock) render_block);
|
||||
|
||||
OSStatus OnGetPlayoutData(AudioUnitRenderActionFlags* flags,
|
||||
const AudioTimeStamp* time_stamp,
|
||||
NSInteger bus_number,
|
||||
UInt32 num_frames,
|
||||
AudioBufferList* io_data);
|
||||
|
||||
// Notifies `ObjCAudioDeviceModule` that at least one of the audio input
|
||||
// parameters or audio input latency of `RTCAudioDevice` has changed. It necessary to
|
||||
// update `record_parameters_` with current audio parameter of `RTCAudioDevice`
|
||||
// via `UpdateAudioParameters` and if parameters are actually change then
|
||||
// ADB parameters are updated with `UpdateInputAudioDeviceBuffer`. Audio input latency
|
||||
// stored in `cached_recording_delay_ms_` is also updated with current latency
|
||||
// of `RTCAudioDevice`.
|
||||
void HandleAudioInputParametersChange();
|
||||
|
||||
// Same as `HandleAudioInputParametersChange` but should be called when audio output
|
||||
// parameters of `RTCAudioDevice` has changed.
|
||||
void HandleAudioOutputParametersChange();
|
||||
|
||||
// Notifies `ObjCAudioDeviceModule` about audio input interruption happen due to
|
||||
// any reason so `ObjCAudioDeviceModule` is can prepare to restart of audio IO.
|
||||
void HandleAudioInputInterrupted();
|
||||
|
||||
// Same as `ObjCAudioDeviceModule` but should be called when audio output
|
||||
// is interrupted.
|
||||
void HandleAudioOutputInterrupted();
|
||||
|
||||
private:
|
||||
// Update our audio parameters if they are different from current device audio parameters
|
||||
// Returns true when our parameters are update, false - otherwise.
|
||||
// `ObjCAudioDeviceModule` has audio device buffer (ADB) which has audio parameters
|
||||
// of playout & recording. The ADB is configured to work with specific sample rate & channel
|
||||
// count. `ObjCAudioDeviceModule` stores audio parameters which were used to configure ADB in the
|
||||
// fields `playout_parameters_` and `recording_parameters_`.
|
||||
// `RTCAudioDevice` protocol has its own audio parameters exposed as individual properties.
|
||||
// `RTCAudioDevice` audio parameters might change when playout/recording is already in progress,
|
||||
// for example, when device is switched. `RTCAudioDevice` audio parameters must be kept in sync
|
||||
// with ADB audio parameters. This method is invoked when `RTCAudioDevice` reports that it's audio
|
||||
// parameters (`device_params`) are changed and it detects if there any difference with our
|
||||
// current audio parameters (`params`). Our parameters are updated in case of actual change and
|
||||
// method returns true. In case of actual change there is follow-up call to either
|
||||
// `UpdateOutputAudioDeviceBuffer` or `UpdateInputAudioDeviceBuffer` to apply updated
|
||||
// `playout_parameters_` or `recording_parameters_` to ADB.
|
||||
|
||||
bool UpdateAudioParameters(AudioParameters& params, const AudioParameters& device_params);
|
||||
|
||||
// Update our cached audio latency with device latency. Device latency is reported by
|
||||
// `RTCAudioDevice` object. Whenever latency is changed, `RTCAudioDevice` is obliged to notify ADM
|
||||
// about the change via `HandleAudioInputParametersChange` or `HandleAudioOutputParametersChange`.
|
||||
// Current device IO latency is cached in the atomic field and used from audio IO thread
|
||||
// to be reported to audio device buffer. It is highly recommended by Apple not to call any
|
||||
// ObjC methods from audio IO thread, that is why implementation relies on caching latency
|
||||
// into a field and being notified when latency is changed, which is the case when device
|
||||
// is switched.
|
||||
void UpdateAudioDelay(std::atomic<int>& delay_ms, const NSTimeInterval device_latency);
|
||||
|
||||
// Uses current `playout_parameters_` to inform the audio device buffer (ADB)
|
||||
// about our internal audio parameters.
|
||||
void UpdateOutputAudioDeviceBuffer();
|
||||
|
||||
// Uses current `record_parameters_` to inform the audio device buffer (ADB)
|
||||
// about our internal audio parameters.
|
||||
void UpdateInputAudioDeviceBuffer();
|
||||
|
||||
private:
|
||||
id<RTC_OBJC_TYPE(RTCAudioDevice)> audio_device_;
|
||||
|
||||
const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
|
||||
|
||||
// AudioDeviceBuffer is a buffer to consume audio recorded by `RTCAudioDevice`
|
||||
// and provide audio to be played via `RTCAudioDevice`.
|
||||
// Audio PCMs could have different sample rate and channels count, but expected
|
||||
// to be in 16-bit integer interleaved linear PCM format.
|
||||
// The current parameters ADB configured to work with is stored in field
|
||||
// `playout_parameters_` for playout and `record_parameters_` for recording.
|
||||
// These parameters and ADB must kept in sync with `RTCAudioDevice` audio parameters.
|
||||
std::unique_ptr<AudioDeviceBuffer> audio_device_buffer_;
|
||||
|
||||
// Set to 1 when recording is active and 0 otherwise.
|
||||
std::atomic<bool> recording_ = false;
|
||||
|
||||
// Set to 1 when playout is active and 0 otherwise.
|
||||
std::atomic<bool> playing_ = false;
|
||||
|
||||
// Stores cached value of `RTCAudioDevice outputLatency` to be used from
|
||||
// audio IO thread. Latency is updated on audio output parameters change.
|
||||
std::atomic<int> cached_playout_delay_ms_ = 0;
|
||||
|
||||
// Same as `cached_playout_delay_ms_` but for audio input
|
||||
std::atomic<int> cached_recording_delay_ms_ = 0;
|
||||
|
||||
// Thread that is initialized audio device module.
|
||||
rtc::Thread* thread_;
|
||||
|
||||
// Ensures that methods are called from the same thread as this object is
|
||||
// initialized on.
|
||||
SequenceChecker thread_checker_;
|
||||
|
||||
// I/O audio thread checker.
|
||||
SequenceChecker io_playout_thread_checker_;
|
||||
SequenceChecker io_record_thread_checker_;
|
||||
|
||||
bool is_initialized_ RTC_GUARDED_BY(thread_checker_) = false;
|
||||
bool is_playout_initialized_ RTC_GUARDED_BY(thread_checker_) = false;
|
||||
bool is_recording_initialized_ RTC_GUARDED_BY(thread_checker_) = false;
|
||||
|
||||
// Contains audio parameters (sample rate, #channels, buffer size etc.) for
|
||||
// the playout and recording sides.
|
||||
AudioParameters playout_parameters_;
|
||||
AudioParameters record_parameters_;
|
||||
|
||||
// `FineAudioBuffer` takes an `AudioDeviceBuffer` which delivers audio data
|
||||
// in chunks of 10ms. `RTCAudioDevice` might deliver recorded data in
|
||||
// chunks which are not 10ms long. `FineAudioBuffer` implements adaptation
|
||||
// from undetermined chunk size to 10ms chunks.
|
||||
std::unique_ptr<FineAudioBuffer> record_fine_audio_buffer_;
|
||||
|
||||
// Same as `record_fine_audio_buffer_` but for audio output.
|
||||
std::unique_ptr<FineAudioBuffer> playout_fine_audio_buffer_;
|
||||
|
||||
// Temporary storage for recorded data.
|
||||
rtc::BufferT<int16_t> record_audio_buffer_;
|
||||
|
||||
// Delegate object provided to RTCAudioDevice during initialization
|
||||
ObjCAudioDeviceDelegate* audio_device_delegate_;
|
||||
};
|
||||
|
||||
} // namespace objc_adm
|
||||
|
||||
Reference in New Issue
Block a user