From dea73ee8f9c0707aed35b9edb928216877f72091 Mon Sep 17 00:00:00 2001 From: Minyue Li Date: Tue, 18 Feb 2020 15:45:41 +0100 Subject: [PATCH] Pass absolute capture time from WebRtcVoiceEngine to ACM. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: webrtc:10739 Change-Id: I6f264cb89ce340db642db3ef7dfc2b5d459f749e Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/167211 Commit-Queue: Minyue Li Reviewed-by: Per Åhgren Reviewed-by: Chen Xing Reviewed-by: Danil Chapovalov Cr-Commit-Position: refs/heads/master@{#30547} --- api/audio/audio_frame.cc | 2 ++ api/audio/audio_frame.h | 19 +++++++++++++-- media/engine/webrtc_voice_engine.cc | 8 +++++-- .../audio_coding/acm2/audio_coding_module.cc | 24 ++++++++++++------- 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/api/audio/audio_frame.cc b/api/audio/audio_frame.cc index d9212a211d..47459ac333 100644 --- a/api/audio/audio_frame.cc +++ b/api/audio/audio_frame.cc @@ -41,6 +41,7 @@ void AudioFrame::ResetWithoutMuting() { vad_activity_ = kVadUnknown; profile_timestamp_ms_ = 0; packet_infos_ = RtpPacketInfos(); + absolute_capture_timestamp_ms_ = absl::nullopt; } void AudioFrame::UpdateFrame(uint32_t timestamp, @@ -86,6 +87,7 @@ void AudioFrame::CopyFrom(const AudioFrame& src) { vad_activity_ = src.vad_activity_; num_channels_ = src.num_channels_; channel_layout_ = src.channel_layout_; + absolute_capture_timestamp_ms_ = src.absolute_capture_timestamp_ms(); const size_t length = samples_per_channel_ * num_channels_; RTC_CHECK_LE(length, kMaxDataSizeSamples); diff --git a/api/audio/audio_frame.h b/api/audio/audio_frame.h index cda8c26780..06b0b28b38 100644 --- a/api/audio/audio_frame.h +++ b/api/audio/audio_frame.h @@ -104,6 +104,15 @@ class AudioFrame { ChannelLayout channel_layout() const { return channel_layout_; } int sample_rate_hz() const { return sample_rate_hz_; } + void set_absolute_capture_timestamp_ms( + int64_t absolute_capture_time_stamp_ms) { + absolute_capture_timestamp_ms_ = absolute_capture_time_stamp_ms; + } + + absl::optional absolute_capture_timestamp_ms() const { + return absolute_capture_timestamp_ms_; + } + // RTP timestamp of the first sample in the AudioFrame. uint32_t timestamp_ = 0; // Time since the first frame in milliseconds. @@ -121,8 +130,8 @@ class AudioFrame { // Monotonically increasing timestamp intended for profiling of audio frames. // Typically used for measuring elapsed time between two different points in // the audio path. No lock is used to save resources and we are thread safe - // by design. Also, absl::optional is not used since it will cause a "complex - // class/struct needs an explicit out-of-line destructor" build error. + // by design. + // TODO(nisse@webrtc.org): consider using absl::optional. int64_t profile_timestamp_ms_ = 0; // Information about packets used to assemble this audio frame. This is needed @@ -150,6 +159,12 @@ class AudioFrame { int16_t data_[kMaxDataSizeSamples]; bool muted_ = true; + // Absolute capture timestamp when this audio frame was originally captured. + // This is only valid for audio frames captured on this machine. The absolute + // capture timestamp of a received frame is found in |packet_infos_|. + // This timestamp MUST be based on the same clock as rtc::TimeMillis(). + absl::optional absolute_capture_timestamp_ms_; + RTC_DISALLOW_COPY_AND_ASSIGN(AudioFrame); }; diff --git a/media/engine/webrtc_voice_engine.cc b/media/engine/webrtc_voice_engine.cc index 2fe25630c4..b4b2b4a3ef 100644 --- a/media/engine/webrtc_voice_engine.cc +++ b/media/engine/webrtc_voice_engine.cc @@ -880,8 +880,12 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream audio_frame->timestamp_, static_cast(audio_data), number_of_frames, sample_rate, audio_frame->speech_type_, audio_frame->vad_activity_, number_of_channels); - // TODO(bugs.webrtc.org/10739): pass absolute_capture_timestamp_ms to - // stream_. + // TODO(bugs.webrtc.org/10739): add dcheck that + // |absolute_capture_timestamp_ms| always receives a value. + if (absolute_capture_timestamp_ms) { + audio_frame->set_absolute_capture_timestamp_ms( + *absolute_capture_timestamp_ms); + } stream_->SendAudioData(std::move(audio_frame)); } diff --git a/modules/audio_coding/acm2/audio_coding_module.cc b/modules/audio_coding/acm2/audio_coding_module.cc index f3dd5b1a1f..e28be186f5 100644 --- a/modules/audio_coding/acm2/audio_coding_module.cc +++ b/modules/audio_coding/acm2/audio_coding_module.cc @@ -109,7 +109,6 @@ class AudioCodingModuleImpl final : public AudioCodingModule { // If a re-mix is required (up or down), this buffer will store a re-mixed // version of the input. std::vector buffer; - int64_t absolute_capture_timestamp_ms; }; InputData input_data_ RTC_GUARDED_BY(acm_crit_sect_); @@ -132,7 +131,11 @@ class AudioCodingModuleImpl final : public AudioCodingModule { int Add10MsDataInternal(const AudioFrame& audio_frame, InputData* input_data) RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); - int Encode(const InputData& input_data) + + // TODO(bugs.webrtc.org/10739): change |absolute_capture_timestamp_ms| to + // int64_t when it always receives a valid value. + int Encode(const InputData& input_data, + absl::optional absolute_capture_timestamp_ms) RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); int InitializeReceiverSafe() RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); @@ -231,7 +234,11 @@ AudioCodingModuleImpl::AudioCodingModuleImpl( AudioCodingModuleImpl::~AudioCodingModuleImpl() = default; -int32_t AudioCodingModuleImpl::Encode(const InputData& input_data) { +int32_t AudioCodingModuleImpl::Encode( + const InputData& input_data, + absl::optional absolute_capture_timestamp_ms) { + // TODO(bugs.webrtc.org/10739): add dcheck that + // |audio_frame.absolute_capture_timestamp_ms()| always has a value. AudioEncoder::EncodedInfo encoded_info; uint8_t previous_pltype; @@ -304,7 +311,7 @@ int32_t AudioCodingModuleImpl::Encode(const InputData& input_data) { packetization_callback_->SendData( frame_type, encoded_info.payload_type, encoded_info.encoded_timestamp, encode_buffer_.data(), encode_buffer_.size(), - input_data.absolute_capture_timestamp_ms); + absolute_capture_timestamp_ms.value_or(-1)); } if (vad_callback_) { @@ -339,7 +346,11 @@ int AudioCodingModuleImpl::RegisterTransportCallback( int AudioCodingModuleImpl::Add10MsData(const AudioFrame& audio_frame) { rtc::CritScope lock(&acm_crit_sect_); int r = Add10MsDataInternal(audio_frame, &input_data_); - return r < 0 ? r : Encode(input_data_); + // TODO(bugs.webrtc.org/10739): add dcheck that + // |audio_frame.absolute_capture_timestamp_ms()| always has a value. + return r < 0 + ? r + : Encode(input_data_, audio_frame.absolute_capture_timestamp_ms()); } int AudioCodingModuleImpl::Add10MsDataInternal(const AudioFrame& audio_frame, @@ -394,9 +405,6 @@ int AudioCodingModuleImpl::Add10MsDataInternal(const AudioFrame& audio_frame, input_data->input_timestamp = ptr_frame->timestamp_; input_data->length_per_channel = ptr_frame->samples_per_channel_; input_data->audio_channel = current_num_channels; - // TODO(bugs.webrtc.org/10739): Assign from a corresponding field in - // audio_frame when it is added in AudioFrame. - input_data->absolute_capture_timestamp_ms = 0; if (!same_num_channels) { // Remixes the input frame to the output data and in the process resize the