zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add mute state field to AudioFrame and switch some callers to use it. Also make AudioFrame::data_ private and instead provide:

Browse Source 
const int16_t* data() const;
int16_t* mutable_data();

- data() returns a zeroed static buffer on muted frames (to avoid unnecessary zeroing of the member buffer) and directly returns AudioFrame::data_ on unmuted frames.
- mutable_data(), lazily zeroes AudioFrame::data_ if the frame is currently muted, sets muted=false, and returns AudioFrame::data_.

These accessors serve to "force" callers to be aware of the mute state field, i.e. lazy zeroing is not the primary motivation.

This change only optimizes handling of muted frames where it is somewhat trivial to do so. Other improvements requiring more significant structural changes will come later.

BUG=webrtc:7343
TBR=henrika

Review-Url: https://codereview.webrtc.org/2750783004
Cr-Commit-Position: refs/heads/master@{#18543}
This commit is contained in:
yujo
2017-06-12 12:45:32 -07:00
committed by Commit Bot
parent 0703856b53
commit 36b1a5fcec
55 changed files with 658 additions and 316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
webrtc/modules/audio_processing/aec_dump/aec_dump_impl.cc
View File

@ -130,7 +130,7 @@ void AecDumpImpl::WriteRenderStreamMessage(const AudioFrame& frame) {
audioproc::ReverseStream* msg = event->mutable_reverse_stream();
const size_t data_size =
sizeof(int16_t) * frame.samples_per_channel_ * frame.num_channels_;
msg->set_data(frame.data_, data_size);
msg->set_data(frame.data(), data_size);
worker_queue_->PostTask(std::unique_ptr<rtc::QueuedTask>(std::move(task)));
}

4
webrtc/modules/audio_processing/aec_dump/capture_stream_info.cc
View File

@ -46,7 +46,7 @@ void CaptureStreamInfo::AddInput(const AudioFrame& frame) {
auto* stream = task_->GetEvent()->mutable_stream();
const size_t data_size =
sizeof(int16_t) * frame.samples_per_channel_ * frame.num_channels_;
stream->set_input_data(frame.data_, data_size);
stream->set_input_data(frame.data(), data_size);
}
void CaptureStreamInfo::AddOutput(const AudioFrame& frame) {
@ -54,7 +54,7 @@ void CaptureStreamInfo::AddOutput(const AudioFrame& frame) {
auto* stream = task_->GetEvent()->mutable_stream();
const size_t data_size =
sizeof(int16_t) * frame.samples_per_channel_ * frame.num_channels_;
stream->set_output_data(frame.data_, data_size);
stream->set_output_data(frame.data(), data_size);
}
void CaptureStreamInfo::AddAudioProcessingState(

10
webrtc/modules/audio_processing/audio_buffer.cc
View File

@ -394,13 +394,14 @@ void AudioBuffer::DeinterleaveFrom(AudioFrame* frame) {
} else {
deinterleaved = input_buffer_->ibuf()->channels();
}
// TODO(yujo): handle muted frames more efficiently.
if (num_proc_channels_ == 1) {
// Downmix and deinterleave simultaneously.
DownmixInterleavedToMono(frame->data_, input_num_frames_,
DownmixInterleavedToMono(frame->data(), input_num_frames_,
num_input_channels_, deinterleaved[0]);
} else {
RTC_DCHECK_EQ(num_proc_channels_, num_input_channels_);
Deinterleave(frame->data_,
Deinterleave(frame->data(),
input_num_frames_,
num_proc_channels_,
deinterleaved);
@ -437,12 +438,13 @@ void AudioBuffer::InterleaveTo(AudioFrame* frame, bool data_changed) const {
data_ptr = output_buffer_.get();
}
// TODO(yujo): handle muted frames more efficiently.
if (frame->num_channels_ == num_channels_) {
Interleave(data_ptr->ibuf()->channels(), output_num_frames_, num_channels_,
frame->data_);
frame->mutable_data());
} else {
UpmixMonoToInterleaved(data_ptr->ibuf()->channels()[0], output_num_frames_,
frame->num_channels_, frame->data_);
frame->num_channels_, frame->mutable_data());
}
}

6
webrtc/modules/audio_processing/audio_processing_impl.cc
View File

@ -1160,7 +1160,7 @@ int AudioProcessingImpl::ProcessStream(AudioFrame* frame) {
audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream();
const size_t data_size =
sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_;
msg->set_input_data(frame->data_, data_size);
msg->set_input_data(frame->data(), data_size);
}
#endif
@ -1178,7 +1178,7 @@ int AudioProcessingImpl::ProcessStream(AudioFrame* frame) {
audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream();
const size_t data_size =
sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_;
msg->set_output_data(frame->data_, data_size);
msg->set_output_data(frame->data(), data_size);
RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
&debug_dump_.num_bytes_left_for_log_,
&crit_debug_, &debug_dump_.capture));
@ -1514,7 +1514,7 @@ int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) {
debug_dump_.render.event_msg->mutable_reverse_stream();
const size_t data_size =
sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_;
msg->set_data(frame->data_, data_size);
msg->set_data(frame->data(), data_size);
RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
&debug_dump_.num_bytes_left_for_log_,
&crit_debug_, &debug_dump_.render));

3
webrtc/modules/audio_processing/audio_processing_impl_locking_unittest.cc
View File

@ -479,11 +479,12 @@ void PopulateAudioFrame(AudioFrame* frame,
RandomGenerator* rand_gen) {
ASSERT_GT(amplitude, 0);
ASSERT_LE(amplitude, 32767);
int16_t* frame_data = frame->mutable_data();
for (size_t ch = 0; ch < frame->num_channels_; ch++) {
for (size_t k = 0; k < frame->samples_per_channel_; k++) {
// Store random 16 bit number between -(amplitude+1) and
// amplitude.
frame->data_[k * ch] =
frame_data[k * ch] =
rand_gen->RandInt(2 * amplitude + 1) - amplitude - 1;
}
}

43
webrtc/modules/audio_processing/audio_processing_unittest.cc
View File

@ -87,7 +87,7 @@ void ConvertToFloat(const int16_t* int_data, ChannelBuffer<float>* cb) {
}
void ConvertToFloat(const AudioFrame& frame, ChannelBuffer<float>* cb) {
ConvertToFloat(frame.data_, cb);
ConvertToFloat(frame.data(), cb);
}
// Number of channels including the keyboard channel.
@ -127,31 +127,34 @@ void CopyLeftToRightChannel(int16_t* stereo, size_t samples_per_channel) {
}
}
void VerifyChannelsAreEqual(int16_t* stereo, size_t samples_per_channel) {
void VerifyChannelsAreEqual(const int16_t* stereo, size_t samples_per_channel) {
for (size_t i = 0; i < samples_per_channel; i++) {
EXPECT_EQ(stereo[i * 2 + 1], stereo[i * 2]);
}
}
void SetFrameTo(AudioFrame* frame, int16_t value) {
int16_t* frame_data = frame->mutable_data();
for (size_t i = 0; i < frame->samples_per_channel_ * frame->num_channels_;
++i) {
frame->data_[i] = value;
frame_data[i] = value;
}
}
void SetFrameTo(AudioFrame* frame, int16_t left, int16_t right) {
ASSERT_EQ(2u, frame->num_channels_);
int16_t* frame_data = frame->mutable_data();
for (size_t i = 0; i < frame->samples_per_channel_ * 2; i += 2) {
frame->data_[i] = left;
frame->data_[i + 1] = right;
frame_data[i] = left;
frame_data[i + 1] = right;
}
}
void ScaleFrame(AudioFrame* frame, float scale) {
int16_t* frame_data = frame->mutable_data();
for (size_t i = 0; i < frame->samples_per_channel_ * frame->num_channels_;
++i) {
frame->data_[i] = FloatS16ToS16(frame->data_[i] * scale);
frame_data[i] = FloatS16ToS16(frame_data[i] * scale);
}
}
@ -162,7 +165,7 @@ bool FrameDataAreEqual(const AudioFrame& frame1, const AudioFrame& frame2) {
if (frame1.num_channels_ != frame2.num_channels_) {
return false;
}
if (memcmp(frame1.data_, frame2.data_,
if (memcmp(frame1.data(), frame2.data(),
frame1.samples_per_channel_ * frame1.num_channels_ *
sizeof(int16_t))) {
return false;
@ -205,9 +208,10 @@ T AbsValue(T a) {
int16_t MaxAudioFrame(const AudioFrame& frame) {
const size_t length = frame.samples_per_channel_ * frame.num_channels_;
int16_t max_data = AbsValue(frame.data_[0]);
const int16_t* frame_data = frame.data();
int16_t max_data = AbsValue(frame_data[0]);
for (size_t i = 1; i < length; i++) {
max_data = std::max(max_data, AbsValue(frame.data_[i]));
max_data = std::max(max_data, AbsValue(frame_data[i]));
}
return max_data;
@ -534,7 +538,7 @@ bool ApmTest::ReadFrame(FILE* file, AudioFrame* frame,
ChannelBuffer<float>* cb) {
// The files always contain stereo audio.
size_t frame_size = frame->samples_per_channel_ * 2;
size_t read_count = fread(frame->data_,
size_t read_count = fread(frame->mutable_data(),
sizeof(int16_t),
frame_size,
file);
@ -545,7 +549,7 @@ bool ApmTest::ReadFrame(FILE* file, AudioFrame* frame,
}
if (frame->num_channels_ == 1) {
MixStereoToMono(frame->data_, frame->data_,
MixStereoToMono(frame->data(), frame->mutable_data(),
frame->samples_per_channel_);
}
@ -1601,11 +1605,13 @@ TEST_F(ApmTest, IdenticalInputChannelsResultInIdenticalOutputChannels) {
ASSERT_EQ(0, feof(far_file_));
ASSERT_EQ(0, feof(near_file_));
while (ReadFrame(far_file_, revframe_) && ReadFrame(near_file_, frame_)) {
CopyLeftToRightChannel(revframe_->data_, revframe_->samples_per_channel_);
CopyLeftToRightChannel(revframe_->mutable_data(),
revframe_->samples_per_channel_);
ASSERT_EQ(kNoErr, apm_->ProcessReverseStream(revframe_));
CopyLeftToRightChannel(frame_->data_, frame_->samples_per_channel_);
CopyLeftToRightChannel(frame_->mutable_data(),
frame_->samples_per_channel_);
frame_->vad_activity_ = AudioFrame::kVadUnknown;
ASSERT_EQ(kNoErr, apm_->set_stream_delay_ms(0));
@ -1615,7 +1621,7 @@ TEST_F(ApmTest, IdenticalInputChannelsResultInIdenticalOutputChannels) {
ASSERT_EQ(kNoErr, apm_->ProcessStream(frame_));
analog_level = apm_->gain_control()->stream_analog_level();
VerifyChannelsAreEqual(frame_->data_, frame_->samples_per_channel_);
VerifyChannelsAreEqual(frame_->data(), frame_->samples_per_channel_);
}
rewind(far_file_);
rewind(near_file_);
@ -1747,7 +1753,7 @@ void ApmTest::ProcessDebugDump(const std::string& in_filename,
msg.channel(i).size());
}
} else {
memcpy(revframe_->data_, msg.data().data(), msg.data().size());
memcpy(revframe_->mutable_data(), msg.data().data(), msg.data().size());
if (format == kFloatFormat) {
// We're using an int16 input file; convert to float.
ConvertToFloat(*revframe_, revfloat_cb_.get());
@ -1778,7 +1784,8 @@ void ApmTest::ProcessDebugDump(const std::string& in_filename,
msg.input_channel(i).size());
}
} else {
memcpy(frame_->data_, msg.input_data().data(), msg.input_data().size());
memcpy(frame_->mutable_data(), msg.input_data().data(),
msg.input_data().size());
if (format == kFloatFormat) {
// We're using an int16 input file; convert to float.
ConvertToFloat(*frame_, float_cb_.get());
@ -1987,7 +1994,7 @@ TEST_F(ApmTest, FloatAndIntInterfacesGiveSimilarResults) {
EXPECT_NOERR(fapm->gain_control()->set_stream_analog_level(analog_level));
EXPECT_NOERR(apm_->ProcessStream(frame_));
Deinterleave(frame_->data_, samples_per_channel, num_output_channels,
Deinterleave(frame_->data(), samples_per_channel, num_output_channels,
output_int16.channels());
EXPECT_NOERR(fapm->ProcessStream(
@ -2151,7 +2158,7 @@ TEST_F(ApmTest, Process) {
ns_speech_prob_average += apm_->noise_suppression()->speech_probability();
size_t frame_size = frame_->samples_per_channel_ * frame_->num_channels_;
size_t write_count = fwrite(frame_->data_,
size_t write_count = fwrite(frame_->data(),
sizeof(int16_t),
frame_size,
out_file_);

15
webrtc/modules/audio_processing/test/aec_dump_based_simulator.cc
View File

@ -29,9 +29,10 @@ bool VerifyFixedBitExactness(const webrtc::audioproc::Stream& msg,
msg.output_data().size()) {
return false;
} else {
const int16_t* frame_data = frame.data();
for (size_t k = 0; k < frame.num_channels_ * frame.samples_per_channel_;
++k) {
if (msg.output_data().data()[k] != frame.data_[k]) {
if (msg.output_data().data()[k] != frame_data[k]) {
return false;
}
}
@ -78,10 +79,11 @@ void AecDumpBasedSimulator::PrepareProcessStreamCall(
interface_used_ = InterfaceType::kFixedInterface;
// Populate input buffer.
RTC_CHECK_EQ(sizeof(fwd_frame_.data_[0]) * fwd_frame_.samples_per_channel_ *
RTC_CHECK_EQ(sizeof(*fwd_frame_.data()) * fwd_frame_.samples_per_channel_ *
fwd_frame_.num_channels_,
msg.input_data().size());
memcpy(fwd_frame_.data_, msg.input_data().data(), msg.input_data().size());
memcpy(fwd_frame_.mutable_data(), msg.input_data().data(),
msg.input_data().size());
} else {
// Float interface processing.
// Verify interface invariance.
@ -105,9 +107,10 @@ void AecDumpBasedSimulator::PrepareProcessStreamCall(
if (artificial_nearend_buffer_reader_->Read(
artificial_nearend_buf_.get())) {
if (msg.has_input_data()) {
int16_t* fwd_frame_data = fwd_frame_.mutable_data();
for (size_t k = 0; k < in_buf_->num_frames(); ++k) {
fwd_frame_.data_[k] = rtc::saturated_cast<int16_t>(
fwd_frame_.data_[k] +
fwd_frame_data[k] = rtc::saturated_cast<int16_t>(
fwd_frame_data[k] +
static_cast<int16_t>(32767 *
artificial_nearend_buf_->channels()[0][k]));
}
@ -191,7 +194,7 @@ void AecDumpBasedSimulator::PrepareReverseProcessStreamCall(
RTC_CHECK_EQ(sizeof(int16_t) * rev_frame_.samples_per_channel_ *
rev_frame_.num_channels_,
msg.data().size());
memcpy(rev_frame_.data_, msg.data().data(), msg.data().size());
memcpy(rev_frame_.mutable_data(), msg.data().data(), msg.data().size());
} else {
// Float interface processing.
// Verify interface invariance.

5
webrtc/modules/audio_processing/test/audio_processing_simulator.cc
View File

@ -30,7 +30,7 @@ void CopyFromAudioFrame(const AudioFrame& src, ChannelBuffer<float>* dest) {
RTC_CHECK_EQ(src.samples_per_channel_, dest->num_frames());
// Copy the data from the input buffer.
std::vector<float> tmp(src.samples_per_channel_ * src.num_channels_);
S16ToFloat(src.data_, tmp.size(), tmp.data());
S16ToFloat(src.data(), tmp.size(), tmp.data());
Deinterleave(tmp.data(), src.samples_per_channel_, src.num_channels_,
dest->channels());
}
@ -68,9 +68,10 @@ SimulationSettings::~SimulationSettings() = default;
void CopyToAudioFrame(const ChannelBuffer<float>& src, AudioFrame* dest) {
RTC_CHECK_EQ(src.num_channels(), dest->num_channels_);
RTC_CHECK_EQ(src.num_frames(), dest->samples_per_channel_);
int16_t* dest_data = dest->mutable_data();
for (size_t ch = 0; ch < dest->num_channels_; ++ch) {
for (size_t sample = 0; sample < dest->samples_per_channel_; ++sample) {
dest->data_[sample * dest->num_channels_ + ch] =
dest_data[sample * dest->num_channels_ + ch] =
src.channels()[ch][sample] * 32767;
}
}