zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Encoder side of Multistream Opus.

Browse Source 
Follows https://webrtc-review.googlesource.com/c/src/+/129768 closely.
Adds an ENCODER and sets it up to parse SDP config for multistream
opus.

E.g. this is the new SDP syntax for 6.1 surround sound:
"multiopus/48000/6 channel_mapping=0,4,1,2,3,5 num_streams=4 coupled_streams=2"


Bug: webrtc:8649
Change-Id: I3fc341e76f5c41dab0243cf65f6461e4c3d9d67d
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/132001
Commit-Queue: Alex Loiko <aleloi@webrtc.org>
Reviewed-by: Oskar Sundbom <ossu@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27775}
This commit is contained in:
Alex Loiko
2019-04-25 15:09:32 +02:00
committed by Commit Bot
parent 9e79e6b9b3
commit 44c21f48ee
14 changed files with 966 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
api/audio_codecs/BUILD.gn
View File

@ -100,7 +100,10 @@ rtc_static_library("builtin_audio_encoder_factory") {
defines += [ "WEBRTC_USE_BUILTIN_ILBC=0" ]
}
if (rtc_include_opus) {
deps += [ "opus:audio_encoder_opus" ]
deps += [
"opus:audio_encoder_multiopus",
"opus:audio_encoder_opus",
]
defines += [ "WEBRTC_USE_BUILTIN_OPUS=1" ]
} else {
defines += [ "WEBRTC_USE_BUILTIN_OPUS=0" ]

3
api/audio_codecs/builtin_audio_encoder_factory.cc
View File

@ -22,6 +22,7 @@
#endif
#include "api/audio_codecs/isac/audio_encoder_isac.h"
#if WEBRTC_USE_BUILTIN_OPUS
#include "api/audio_codecs/opus/audio_encoder_multi_channel_opus.h"
#include "api/audio_codecs/opus/audio_encoder_opus.h" // nogncheck
#endif
@ -57,7 +58,7 @@ rtc::scoped_refptr<AudioEncoderFactory> CreateBuiltinAudioEncoderFactory() {
return CreateAudioEncoderFactory<
#if WEBRTC_USE_BUILTIN_OPUS
AudioEncoderOpus,
AudioEncoderOpus, NotAdvertised<AudioEncoderMultiChannelOpus>,
#endif
AudioEncoderIsac, AudioEncoderG722,

31
api/audio_codecs/opus/BUILD.gn
View File

@ -15,7 +15,8 @@ if (is_android) {
rtc_static_library("audio_encoder_opus_config") {
visibility = [ "*" ]
sources = [
"audio_decoder_multi_channel_opus_config.h",
"audio_encoder_multi_channel_opus_config.cc",
"audio_encoder_multi_channel_opus_config.h",
"audio_encoder_opus_config.cc",
"audio_encoder_opus_config.h",
]
@ -32,6 +33,13 @@ rtc_static_library("audio_encoder_opus_config") {
}
}
rtc_source_set("audio_decoder_opus_config") {
visibility = [ "*" ]
sources = [
"audio_decoder_multi_channel_opus_config.h",
]
}
rtc_source_set("audio_encoder_opus") {
visibility = [ "*" ]
poisonous = [ "audio_codecs" ]
@ -70,6 +78,25 @@ rtc_static_library("audio_decoder_opus") {
]
}
rtc_source_set("audio_encoder_multiopus") {
visibility = [ "*" ]
poisonous = [ "audio_codecs" ]
public = [
"audio_encoder_multi_channel_opus.h",
]
sources = [
"audio_encoder_multi_channel_opus.cc",
]
deps = [
"..:audio_codecs_api",
"../../../modules/audio_coding:webrtc_multiopus",
"../../../rtc_base:rtc_base_approved",
"../../../rtc_base/system:rtc_export",
"../opus:audio_encoder_opus_config",
"//third_party/abseil-cpp/absl/types:optional",
]
}
rtc_static_library("audio_decoder_multiopus") {
visibility = [ "*" ]
poisonous = [ "audio_codecs" ]
@ -78,7 +105,7 @@ rtc_static_library("audio_decoder_multiopus") {
"audio_decoder_multi_channel_opus.h",
]
deps = [
":audio_encoder_opus_config",
":audio_decoder_opus_config",
"..:audio_codecs_api",
"../../../modules/audio_coding:webrtc_multiopus",
"../../../rtc_base:rtc_base_approved",

74
api/audio_codecs/opus/audio_encoder_multi_channel_opus.cc Normal file
View File

@ -0,0 +1,74 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "api/audio_codecs/opus/audio_encoder_multi_channel_opus.h"
#include <utility>
#include "modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_impl.h"
namespace webrtc {
absl::optional<AudioEncoderMultiChannelOpusConfig>
AudioEncoderMultiChannelOpus::SdpToConfig(const SdpAudioFormat& format) {
return AudioEncoderMultiChannelOpusImpl::SdpToConfig(format);
}
void AudioEncoderMultiChannelOpus::AppendSupportedEncoders(
std::vector<AudioCodecSpec>* specs) {
// To get full utilization of the surround support of the Opus lib, we can
// mark which channel is the low frequency effects (LFE). But that is not done
// ATM.
{
AudioCodecInfo surround_5_1_opus_info{48000, 6,
/* default_bitrate_bps= */ 128000};
surround_5_1_opus_info.allow_comfort_noise = false;
surround_5_1_opus_info.supports_network_adaption = false;
SdpAudioFormat opus_format({"multiopus",
48000,
6,
{{"minptime", "10"},
{"useinbandfec", "1"},
{"channel_mapping", "0,4,1,2,3,5"},
{"num_streams", "4"},
{"coupled_streams", "2"}}});
specs->push_back({std::move(opus_format), surround_5_1_opus_info});
}
{
AudioCodecInfo surround_7_1_opus_info{48000, 8,
/* default_bitrate_bps= */ 200000};
surround_7_1_opus_info.allow_comfort_noise = false;
surround_7_1_opus_info.supports_network_adaption = false;
SdpAudioFormat opus_format({"multiopus",
48000,
8,
{{"minptime", "10"},
{"useinbandfec", "1"},
{"channel_mapping", "0,6,1,2,3,4,5,7"},
{"num_streams", "5"},
{"coupled_streams", "3"}}});
specs->push_back({std::move(opus_format), surround_7_1_opus_info});
}
}
AudioCodecInfo AudioEncoderMultiChannelOpus::QueryAudioEncoder(
const AudioEncoderMultiChannelOpusConfig& config) {
return AudioEncoderMultiChannelOpusImpl::QueryAudioEncoder(config);
}
std::unique_ptr<AudioEncoder> AudioEncoderMultiChannelOpus::MakeAudioEncoder(
const AudioEncoderMultiChannelOpusConfig& config,
int payload_type,
absl::optional<AudioCodecPairId> /*codec_pair_id*/) {
return AudioEncoderMultiChannelOpusImpl::MakeAudioEncoder(config,
payload_type);
}
} // namespace webrtc

41
api/audio_codecs/opus/audio_encoder_multi_channel_opus.h Normal file
View File

@ -0,0 +1,41 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef API_AUDIO_CODECS_OPUS_AUDIO_ENCODER_MULTI_CHANNEL_OPUS_H_
#define API_AUDIO_CODECS_OPUS_AUDIO_ENCODER_MULTI_CHANNEL_OPUS_H_
#include <memory>
#include <vector>
#include "absl/types/optional.h"
#include "api/audio_codecs/audio_codec_pair_id.h"
#include "api/audio_codecs/audio_encoder.h"
#include "api/audio_codecs/audio_format.h"
#include "api/audio_codecs/opus/audio_encoder_multi_channel_opus_config.h"
#include "rtc_base/system/rtc_export.h"
namespace webrtc {
// Opus encoder API for use as a template parameter to
// CreateAudioEncoderFactory<...>().
struct RTC_EXPORT AudioEncoderMultiChannelOpus {
using Config = AudioEncoderMultiChannelOpusConfig;
static absl::optional<Config> SdpToConfig(const SdpAudioFormat& audio_format);
static void AppendSupportedEncoders(std::vector<AudioCodecSpec>* specs);
static AudioCodecInfo QueryAudioEncoder(const Config& config);
static std::unique_ptr<AudioEncoder> MakeAudioEncoder(
const Config& config,
int payload_type,
absl::optional<AudioCodecPairId> codec_pair_id = absl::nullopt);
};
} // namespace webrtc
#endif // API_AUDIO_CODECS_OPUS_AUDIO_ENCODER_MULTI_CHANNEL_OPUS_H_

106
api/audio_codecs/opus/audio_encoder_multi_channel_opus_config.cc Normal file
View File

@ -0,0 +1,106 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "api/audio_codecs/opus/audio_encoder_multi_channel_opus_config.h"
namespace webrtc {
namespace {
constexpr int kDefaultComplexity = 9;
} // namespace
AudioEncoderMultiChannelOpusConfig::AudioEncoderMultiChannelOpusConfig()
: frame_size_ms(kDefaultFrameSizeMs),
num_channels(1),
application(ApplicationMode::kVoip),
bitrate_bps(32000),
fec_enabled(false),
cbr_enabled(false),
dtx_enabled(false),
max_playback_rate_hz(48000),
complexity(kDefaultComplexity),
num_streams(-1),
coupled_streams(-1) {}
AudioEncoderMultiChannelOpusConfig::AudioEncoderMultiChannelOpusConfig(
const AudioEncoderMultiChannelOpusConfig&) = default;
AudioEncoderMultiChannelOpusConfig::~AudioEncoderMultiChannelOpusConfig() =
default;
AudioEncoderMultiChannelOpusConfig& AudioEncoderMultiChannelOpusConfig::
operator=(const AudioEncoderMultiChannelOpusConfig&) = default;
bool AudioEncoderMultiChannelOpusConfig::IsOk() const {
if (frame_size_ms <= 0 || frame_size_ms % 10 != 0)
return false;
if (num_channels < 0 || num_channels >= 255) {
return false;
}
if (bitrate_bps < kMinBitrateBps || bitrate_bps > kMaxBitrateBps)
return false;
if (complexity < 0 || complexity > 10)
return false;
// Check the lengths:
if (num_channels < 0 || num_streams < 0 || coupled_streams < 0) {
return false;
}
if (num_streams < coupled_streams) {
return false;
}
if (channel_mapping.size() != static_cast<size_t>(num_channels)) {
return false;
}
// Every mono stream codes one channel, every coupled stream codes two. This
// is the total coded channel count:
const int max_coded_channel = num_streams + coupled_streams;
for (const auto& x : channel_mapping) {
// Coded channels >= max_coded_channel don't exist. Except for 255, which
// tells Opus to ignore input channel x.
if (x >= max_coded_channel && x != 255) {
return false;
}
}
// Inverse mapping.
constexpr int kNotSet = -1;
std::vector<int> coded_channels_to_input_channels(max_coded_channel, kNotSet);
for (size_t i = 0; i < num_channels; ++i) {
if (channel_mapping[i] == 255) {
continue;
}
// If it's not ignored, put it in the inverted mapping. But first check if
// we've told Opus to use another input channel for this coded channel:
const int coded_channel = channel_mapping[i];
if (coded_channels_to_input_channels[coded_channel] != kNotSet) {
// Coded channel `coded_channel` comes from both input channels
// `coded_channels_to_input_channels[coded_channel]` and `i`.
return false;
}
coded_channels_to_input_channels[coded_channel] = i;
}
// Check that we specified what input the encoder should use to produce
// every coded channel.
for (int i = 0; i < max_coded_channel; ++i) {
if (coded_channels_to_input_channels[i] == kNotSet) {
// Coded channel `i` has unspecified input channel.
return false;
}
}
if (num_channels > 255 || max_coded_channel >= 255) {
return false;
}
return true;
}
} // namespace webrtc

66
api/audio_codecs/opus/audio_encoder_multi_channel_opus_config.h Normal file
View File

@ -0,0 +1,66 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef API_AUDIO_CODECS_OPUS_AUDIO_ENCODER_MULTI_CHANNEL_OPUS_CONFIG_H_
#define API_AUDIO_CODECS_OPUS_AUDIO_ENCODER_MULTI_CHANNEL_OPUS_CONFIG_H_
#include <stddef.h>
#include <vector>
#include "absl/types/optional.h"
#include "api/audio_codecs/opus/audio_encoder_opus_config.h"
#include "rtc_base/system/rtc_export.h"
namespace webrtc {
struct RTC_EXPORT AudioEncoderMultiChannelOpusConfig {
static constexpr int kDefaultFrameSizeMs = 20;
// Opus API allows a min bitrate of 500bps, but Opus documentation suggests
// bitrate should be in the range of 6000 to 510000, inclusive.
static constexpr int kMinBitrateBps = 6000;
static constexpr int kMaxBitrateBps = 510000;
AudioEncoderMultiChannelOpusConfig();
AudioEncoderMultiChannelOpusConfig(const AudioEncoderMultiChannelOpusConfig&);
~AudioEncoderMultiChannelOpusConfig();
AudioEncoderMultiChannelOpusConfig& operator=(
const AudioEncoderMultiChannelOpusConfig&);
int frame_size_ms;
size_t num_channels;
enum class ApplicationMode { kVoip, kAudio };
ApplicationMode application;
int bitrate_bps;
bool fec_enabled;
bool cbr_enabled;
bool dtx_enabled;
int max_playback_rate_hz;
std::vector<int> supported_frame_lengths_ms;
int complexity;
// Number of mono/stereo Opus streams.
int num_streams;
// Number of channel pairs coupled together, see RFC 7845 section
// 5.1.1. Has to be less than the number of streams
int coupled_streams;
// Channel mapping table, defines the mapping from encoded streams to input
// channels. See RFC 7845 section 5.1.1.
std::vector<unsigned char> channel_mapping;
bool IsOk() const;
};
} // namespace webrtc
#endif // API_AUDIO_CODECS_OPUS_AUDIO_ENCODER_MULTI_CHANNEL_OPUS_CONFIG_H_

4
api/audio_codecs/opus/audio_encoder_opus_config.cc
View File

@ -55,9 +55,9 @@ AudioEncoderOpusConfig& AudioEncoderOpusConfig::operator=(
bool AudioEncoderOpusConfig::IsOk() const {
if (frame_size_ms <= 0 || frame_size_ms % 10 != 0)
return false;
if (num_channels != 1 && num_channels != 2 && num_channels != 4 &&
num_channels != 6 && num_channels != 8)
if (num_channels < 0 || num_channels >= 255) {
return false;
}
if (!bitrate_bps)
return false;
if (*bitrate_bps < kMinBitrateBps || *bitrate_bps > kMaxBitrateBps)

6
modules/audio_coding/BUILD.gn
View File

@ -797,11 +797,14 @@ rtc_static_library("webrtc_multiopus") {
sources = [
"codecs/opus/audio_decoder_multi_channel_opus_impl.cc",
"codecs/opus/audio_decoder_multi_channel_opus_impl.h",
"codecs/opus/audio_encoder_multi_channel_opus_impl.cc",
"codecs/opus/audio_encoder_multi_channel_opus_impl.h",
]
deps = [
":audio_coding_opus_common",
"../../api/audio_codecs:audio_codecs_api",
"../../api/audio_codecs/opus:audio_decoder_opus_config",
"../../api/audio_codecs/opus:audio_encoder_opus_config",
"../../rtc_base:checks",
"../../rtc_base:logging",
@ -810,6 +813,7 @@ rtc_static_library("webrtc_multiopus") {
"../../rtc_base:safe_minmax",
"../../rtc_base:stringutils",
"//third_party/abseil-cpp/absl/memory",
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
]
public_deps = [ # no-presubmit-check TODO(webrtc:8603)
@ -1928,6 +1932,7 @@ if (rtc_include_tests) {
"codecs/isac/unittest.cc",
"codecs/legacy_encoded_audio_frame_unittest.cc",
"codecs/opus/audio_decoder_multi_channel_opus_unittest.cc",
"codecs/opus/audio_encoder_multi_channel_opus_unittest.cc",
"codecs/opus/audio_encoder_opus_unittest.cc",
"codecs/opus/opus_bandwidth_unittest.cc",
"codecs/opus/opus_unittest.cc",
@ -2008,6 +2013,7 @@ if (rtc_include_tests) {
"../../api/audio_codecs:builtin_audio_encoder_factory",
"../../api/audio_codecs/opus:audio_decoder_multiopus",
"../../api/audio_codecs/opus:audio_decoder_opus",
"../../api/audio_codecs/opus:audio_encoder_multiopus",
"../../api/audio_codecs/opus:audio_encoder_opus",
"../../common_audio",
"../../common_audio:common_audio_c",

49
modules/audio_coding/acm2/audio_coding_module_unittest.cc
View File

@ -19,6 +19,7 @@
#include "api/audio_codecs/builtin_audio_encoder_factory.h"
#include "api/audio_codecs/opus/audio_decoder_multi_channel_opus.h"
#include "api/audio_codecs/opus/audio_decoder_opus.h"
#include "api/audio_codecs/opus/audio_encoder_multi_channel_opus.h"
#include "api/audio_codecs/opus/audio_encoder_opus.h"
#include "modules/audio_coding/acm2/acm_receive_test.h"
#include "modules/audio_coding/acm2/acm_send_test.h"
@ -26,8 +27,6 @@
#include "modules/audio_coding/codecs/g711/audio_decoder_pcm.h"
#include "modules/audio_coding/codecs/g711/audio_encoder_pcm.h"
#include "modules/audio_coding/codecs/isac/main/include/audio_encoder_isac.h"
#include "modules/audio_coding/codecs/opus/audio_decoder_opus.h"
#include "modules/audio_coding/codecs/opus/audio_encoder_opus.h"
#include "modules/audio_coding/include/audio_coding_module.h"
#include "modules/audio_coding/include/audio_coding_module_typedefs.h"
#include "modules/audio_coding/neteq/tools/audio_checksum.h"
@ -1516,51 +1515,45 @@ TEST_F(AcmSenderBitExactnessNewApi, MAYBE_OpusFromFormat_stereo_20ms) {
TEST_F(AcmSenderBitExactnessNewApi, DISABLED_OpusManyChannels) {
constexpr int kNumChannels = 4;
constexpr int kOpusPayloadType = 120;
constexpr int kBitrateBps = 128000;
// Read a 4 channel file at 48kHz.
ASSERT_TRUE(SetUpSender(kTestFileQuad48kHz, 48000));
// TODO(webrtc:8649): change to higher level
// AudioEncoderOpus::MakeAudioEncoder once a multistream encoder can be set up
// from SDP. - This is now done for the Decoder.
const auto sdp_format = SdpAudioFormat("multiopus", 48000, kNumChannels,
{{"channel_mapping", "0,1,2,3"},
{"coupled_streams", "2"},
{"num_streams", "2"}});
const auto encoder_config =
AudioEncoderMultiChannelOpus::SdpToConfig(sdp_format);
// The Encoder and Decoder are set up differently (and the test is disabled)
// until the changes from
// https://webrtc-review.googlesource.com/c/src/+/121764 land.
AudioEncoderOpusConfig config = *AudioEncoderOpus::SdpToConfig(
SdpAudioFormat("opus", 48000, 2, {{"stereo", "1"}}));
config.num_channels = kNumChannels;
config.bitrate_bps = kBitrateBps;
ASSERT_TRUE(encoder_config.has_value());
ASSERT_NO_FATAL_FAILURE(
SetUpTestExternalEncoder(AudioEncoderMultiChannelOpus::MakeAudioEncoder(
*encoder_config, kOpusPayloadType),
kOpusPayloadType));
const auto sdp_format = SdpAudioFormat(
"multiopus", 48000, kNumChannels,
{{"channel_mapping", "0,1,2,3"}, {"coupled_streams", "2"}});
const auto decoder_config =
AudioDecoderMultiChannelOpus::SdpToConfig(sdp_format);
const auto opus_decoder =
AudioDecoderMultiChannelOpus::MakeAudioDecoder(*decoder_config);
ASSERT_NO_FATAL_FAILURE(SetUpTestExternalEncoder(
absl::make_unique<AudioEncoderOpusImpl>(config, kOpusPayloadType),
kOpusPayloadType));
rtc::scoped_refptr<AudioDecoderFactory> decoder_factory =
new rtc::RefCountedObject<test::AudioDecoderProxyFactory>(
opus_decoder.get());
// Set up an EXTERNAL DECODER to parse 4 channels.
Run(AcmReceiverBitExactnessOldApi::PlatformChecksum( // audio checksum
"b70470884d9a8613eff019b0d1c8876e|d0a73d377e0ca1be6b06e989e0ad2c35",
"d0a73d377e0ca1be6b06e989e0ad2c35",
"b45d2ce5fc4723e9eb41350af9c68f56", "android arm64 audio checksum",
"1c9a3c9dacdd4b8fc9ff608227e531f2"),
"audio checksum check downstream|8051617907766bec5f4e4a4f7c6d5291",
"8051617907766bec5f4e4a4f7c6d5291",
"6183752a62dc1368f959eb3a8c93b846", "android arm64 audio checksum",
"48bf1f3ca0b72f3c9cdfbe79956122b1"),
// payload_checksum,
AcmReceiverBitExactnessOldApi::PlatformChecksum( // payload checksum
"c2e7d40f8269ef754bd86d6be9623fa7|76de0f4992e3937ca60d35bbb0d308d6",
"76de0f4992e3937ca60d35bbb0d308d6",
"2a310aca965c16c2dfd61a9f9fc0c877", "android arm64 payload checksum",
"2294f4b61fb8f174f5196776a0a49be7"),
"payload checksum check downstream|b09c52e44b2bdd9a0809e3a5b1623a76",
"b09c52e44b2bdd9a0809e3a5b1623a76",
"2ea535ef60f7d0c9d89e3002d4c2124f", "android arm64 payload checksum",
"e87995a80f50a0a735a230ca8b04a67d"),
50, test::AcmReceiveTestOldApi::kQuadOutput, decoder_factory);
}

357
modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_impl.cc Normal file
View File

@ -0,0 +1,357 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
/*
* LEFT TO DO:
* - WRITE TESTS for the stuff in this file.
* - Check the creation, maybe make it safer by returning an empty optional or
* unique_ptr. --- It looks OK, but RecreateEncoderInstance can perhaps crash
* on a valid config. Can run it in the fuzzer for some time. Should prbl also
* fuzz the config.
*/
#include "modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_impl.h"
#include <algorithm>
#include <memory>
#include <string>
#include <vector>
#include "absl/memory/memory.h"
#include "absl/strings/match.h"
#include "modules/audio_coding/codecs/opus/audio_coder_opus_common.h"
#include "rtc_base/arraysize.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/string_to_number.h"
namespace webrtc {
namespace {
// Recommended bitrates for one channel:
// 8-12 kb/s for NB speech,
// 16-20 kb/s for WB speech,
// 28-40 kb/s for FB speech,
// 48-64 kb/s for FB mono music, and
// 64-128 kb/s for FB stereo music.
// The current implementation multiplies these values by the number of channels.
constexpr int kOpusBitrateNbBps = 12000;
constexpr int kOpusBitrateWbBps = 20000;
constexpr int kOpusBitrateFbBps = 32000;
constexpr int kDefaultMaxPlaybackRate = 48000;
// These two lists must be sorted from low to high
#if WEBRTC_OPUS_SUPPORT_120MS_PTIME
constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60, 120};
#else
constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60};
#endif
int GetBitrateBps(const AudioEncoderMultiChannelOpusConfig& config) {
RTC_DCHECK(config.IsOk());
return config.bitrate_bps;
}
int GetMaxPlaybackRate(const SdpAudioFormat& format) {
const auto param = GetFormatParameter<int>(format, "maxplaybackrate");
if (param && *param >= 8000) {
return std::min(*param, kDefaultMaxPlaybackRate);
}
return kDefaultMaxPlaybackRate;
}
int GetFrameSizeMs(const SdpAudioFormat& format) {
const auto ptime = GetFormatParameter<int>(format, "ptime");
if (ptime.has_value()) {
// Pick the next highest supported frame length from
// kOpusSupportedFrameLengths.
for (const int supported_frame_length : kOpusSupportedFrameLengths) {
if (supported_frame_length >= *ptime) {
return supported_frame_length;
}
}
// If none was found, return the largest supported frame length.
return *(std::end(kOpusSupportedFrameLengths) - 1);
}
return AudioEncoderOpusConfig::kDefaultFrameSizeMs;
}
int CalculateDefaultBitrate(int max_playback_rate, size_t num_channels) {
const int bitrate = [&] {
if (max_playback_rate <= 8000) {
return kOpusBitrateNbBps * rtc::dchecked_cast<int>(num_channels);
} else if (max_playback_rate <= 16000) {
return kOpusBitrateWbBps * rtc::dchecked_cast<int>(num_channels);
} else {
return kOpusBitrateFbBps * rtc::dchecked_cast<int>(num_channels);
}
}();
RTC_DCHECK_GE(bitrate, AudioEncoderMultiChannelOpusConfig::kMinBitrateBps);
return bitrate;
}
// Get the maxaveragebitrate parameter in string-form, so we can properly figure
// out how invalid it is and accurately log invalid values.
int CalculateBitrate(int max_playback_rate_hz,
size_t num_channels,
absl::optional<std::string> bitrate_param) {
const int default_bitrate =
CalculateDefaultBitrate(max_playback_rate_hz, num_channels);
if (bitrate_param) {
const auto bitrate = rtc::StringToNumber<int>(*bitrate_param);
if (bitrate) {
const int chosen_bitrate =
std::max(AudioEncoderOpusConfig::kMinBitrateBps,
std::min(*bitrate, AudioEncoderOpusConfig::kMaxBitrateBps));
if (bitrate != chosen_bitrate) {
RTC_LOG(LS_WARNING) << "Invalid maxaveragebitrate " << *bitrate
<< " clamped to " << chosen_bitrate;
}
return chosen_bitrate;
}
RTC_LOG(LS_WARNING) << "Invalid maxaveragebitrate \"" << *bitrate_param
<< "\" replaced by default bitrate " << default_bitrate;
}
return default_bitrate;
}
} // namespace
std::unique_ptr<AudioEncoder>
AudioEncoderMultiChannelOpusImpl::MakeAudioEncoder(
const AudioEncoderMultiChannelOpusConfig& config,
int payload_type) {
if (!config.IsOk()) {
return nullptr;
}
return absl::make_unique<AudioEncoderMultiChannelOpusImpl>(config,
payload_type);
}
AudioEncoderMultiChannelOpusImpl::AudioEncoderMultiChannelOpusImpl(
const AudioEncoderMultiChannelOpusConfig& config,
int payload_type)
: payload_type_(payload_type), inst_(nullptr) {
RTC_DCHECK(0 <= payload_type && payload_type <= 127);
RTC_CHECK(RecreateEncoderInstance(config));
}
AudioEncoderMultiChannelOpusImpl::~AudioEncoderMultiChannelOpusImpl() {
RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_));
}
size_t AudioEncoderMultiChannelOpusImpl::SufficientOutputBufferSize() const {
// Calculate the number of bytes we expect the encoder to produce,
// then multiply by two to give a wide margin for error.
const size_t bytes_per_millisecond =
static_cast<size_t>(GetBitrateBps(config_) / (1000 * 8) + 1);
const size_t approx_encoded_bytes =
Num10msFramesPerPacket() * 10 * bytes_per_millisecond;
return 2 * approx_encoded_bytes;
}
void AudioEncoderMultiChannelOpusImpl::Reset() {
RTC_CHECK(RecreateEncoderInstance(config_));
}
// If the given config is OK, recreate the Opus encoder instance with those
// settings, save the config, and return true. Otherwise, do nothing and return
// false.
bool AudioEncoderMultiChannelOpusImpl::RecreateEncoderInstance(
const AudioEncoderMultiChannelOpusConfig& config) {
if (!config.IsOk())
return false;
config_ = config;
if (inst_)
RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_));
input_buffer_.clear();
input_buffer_.reserve(Num10msFramesPerPacket() * SamplesPer10msFrame());
RTC_CHECK_EQ(
0, WebRtcOpus_MultistreamEncoderCreate(
&inst_, config.num_channels,
config.application ==
AudioEncoderMultiChannelOpusConfig::ApplicationMode::kVoip
? 0
: 1,
config.num_streams, config.coupled_streams,
config.channel_mapping.data()));
const int bitrate = GetBitrateBps(config);
RTC_CHECK_EQ(0, WebRtcOpus_SetBitRate(inst_, bitrate));
RTC_LOG(LS_VERBOSE) << "Set Opus bitrate to " << bitrate << " bps.";
if (config.fec_enabled) {
RTC_CHECK_EQ(0, WebRtcOpus_EnableFec(inst_));
RTC_LOG(LS_VERBOSE) << "Opus enable FEC";
} else {
RTC_CHECK_EQ(0, WebRtcOpus_DisableFec(inst_));
RTC_LOG(LS_VERBOSE) << "Opus disable FEC";
}
RTC_CHECK_EQ(
0, WebRtcOpus_SetMaxPlaybackRate(inst_, config.max_playback_rate_hz));
RTC_LOG(LS_VERBOSE) << "Set Opus playback rate to "
<< config.max_playback_rate_hz << " hz.";
// Use the DEFAULT complexity.
RTC_CHECK_EQ(
0, WebRtcOpus_SetComplexity(inst_, AudioEncoderOpusConfig().complexity));
RTC_LOG(LS_VERBOSE) << "Set Opus coding complexity to "
<< AudioEncoderOpusConfig().complexity;
if (config.dtx_enabled) {
RTC_CHECK_EQ(0, WebRtcOpus_EnableDtx(inst_));
RTC_LOG(LS_VERBOSE) << "Opus enable DTX";
} else {
RTC_CHECK_EQ(0, WebRtcOpus_DisableDtx(inst_));
RTC_LOG(LS_VERBOSE) << "Opus disable DTX";
}
if (config.cbr_enabled) {
RTC_CHECK_EQ(0, WebRtcOpus_EnableCbr(inst_));
RTC_LOG(LS_VERBOSE) << "Opus enable CBR";
} else {
RTC_CHECK_EQ(0, WebRtcOpus_DisableCbr(inst_));
RTC_LOG(LS_VERBOSE) << "Opus disable CBR";
}
num_channels_to_encode_ = NumChannels();
next_frame_length_ms_ = config_.frame_size_ms;
RTC_LOG(LS_VERBOSE) << "Set Opus frame length to " << config_.frame_size_ms
<< " ms";
return true;
}
absl::optional<AudioEncoderMultiChannelOpusConfig>
AudioEncoderMultiChannelOpusImpl::SdpToConfig(const SdpAudioFormat& format) {
if (!absl::EqualsIgnoreCase(format.name, "multiopus") ||
format.clockrate_hz != 48000) {
return absl::nullopt;
}
AudioEncoderMultiChannelOpusConfig config;
config.num_channels = format.num_channels;
config.frame_size_ms = GetFrameSizeMs(format);
config.max_playback_rate_hz = GetMaxPlaybackRate(format);
config.fec_enabled = (GetFormatParameter(format, "useinbandfec") == "1");
config.dtx_enabled = (GetFormatParameter(format, "usedtx") == "1");
config.cbr_enabled = (GetFormatParameter(format, "cbr") == "1");
config.bitrate_bps =
CalculateBitrate(config.max_playback_rate_hz, config.num_channels,
GetFormatParameter(format, "maxaveragebitrate"));
config.application =
config.num_channels == 1
? AudioEncoderMultiChannelOpusConfig::ApplicationMode::kVoip
: AudioEncoderMultiChannelOpusConfig::ApplicationMode::kAudio;
config.supported_frame_lengths_ms.clear();
std::copy(std::begin(kOpusSupportedFrameLengths),
std::end(kOpusSupportedFrameLengths),
std::back_inserter(config.supported_frame_lengths_ms));
auto num_streams = GetFormatParameter<int>(format, "num_streams");
if (!num_streams.has_value()) {
return absl::nullopt;
}
config.num_streams = *num_streams;
auto coupled_streams = GetFormatParameter<int>(format, "coupled_streams");
if (!coupled_streams.has_value()) {
return absl::nullopt;
}
config.coupled_streams = *coupled_streams;
auto channel_mapping =
GetFormatParameter<std::vector<unsigned char>>(format, "channel_mapping");
if (!channel_mapping.has_value()) {
return absl::nullopt;
}
config.channel_mapping = *channel_mapping;
return config;
}
AudioCodecInfo AudioEncoderMultiChannelOpusImpl::QueryAudioEncoder(
const AudioEncoderMultiChannelOpusConfig& config) {
RTC_DCHECK(config.IsOk());
AudioCodecInfo info(48000, config.num_channels, config.bitrate_bps,
AudioEncoderOpusConfig::kMinBitrateBps,
AudioEncoderOpusConfig::kMaxBitrateBps);
info.allow_comfort_noise = false;
info.supports_network_adaption = false;
return info;
}
size_t AudioEncoderMultiChannelOpusImpl::Num10msFramesPerPacket() const {
return static_cast<size_t>(rtc::CheckedDivExact(config_.frame_size_ms, 10));
}
size_t AudioEncoderMultiChannelOpusImpl::SamplesPer10msFrame() const {
return rtc::CheckedDivExact(48000, 100) * config_.num_channels;
}
int AudioEncoderMultiChannelOpusImpl::SampleRateHz() const {
return 48000;
}
size_t AudioEncoderMultiChannelOpusImpl::NumChannels() const {
return config_.num_channels;
}
size_t AudioEncoderMultiChannelOpusImpl::Num10MsFramesInNextPacket() const {
return Num10msFramesPerPacket();
}
size_t AudioEncoderMultiChannelOpusImpl::Max10MsFramesInAPacket() const {
return Num10msFramesPerPacket();
}
int AudioEncoderMultiChannelOpusImpl::GetTargetBitrate() const {
return GetBitrateBps(config_);
}
AudioEncoder::EncodedInfo AudioEncoderMultiChannelOpusImpl::EncodeImpl(
uint32_t rtp_timestamp,
rtc::ArrayView<const int16_t> audio,
rtc::Buffer* encoded) {
if (input_buffer_.empty())
first_timestamp_in_buffer_ = rtp_timestamp;
input_buffer_.insert(input_buffer_.end(), audio.cbegin(), audio.cend());
if (input_buffer_.size() <
(Num10msFramesPerPacket() * SamplesPer10msFrame())) {
return EncodedInfo();
}
RTC_CHECK_EQ(input_buffer_.size(),
Num10msFramesPerPacket() * SamplesPer10msFrame());
const size_t max_encoded_bytes = SufficientOutputBufferSize();
EncodedInfo info;
info.encoded_bytes = encoded->AppendData(
max_encoded_bytes, [&](rtc::ArrayView<uint8_t> encoded) {
int status = WebRtcOpus_Encode(
inst_, &input_buffer_[0],
rtc::CheckedDivExact(input_buffer_.size(), config_.num_channels),
rtc::saturated_cast<int16_t>(max_encoded_bytes), encoded.data());
RTC_CHECK_GE(status, 0); // Fails only if fed invalid data.
return static_cast<size_t>(status);
});
input_buffer_.clear();
// Will use new packet size for next encoding.
config_.frame_size_ms = next_frame_length_ms_;
info.encoded_timestamp = first_timestamp_in_buffer_;
info.payload_type = payload_type_;
info.send_even_if_empty = true; // Allows Opus to send empty packets.
info.speech = true;
info.encoder_type = CodecType::kOther;
return info;
}
} // namespace webrtc

85
modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_impl.h Normal file
View File

@ -0,0 +1,85 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_ENCODER_MULTI_CHANNEL_OPUS_IMPL_H_
#define MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_ENCODER_MULTI_CHANNEL_OPUS_IMPL_H_
#include <memory>
#include <vector>
#include "absl/types/optional.h"
#include "api/audio_codecs/audio_encoder.h"
#include "api/audio_codecs/audio_format.h"
#include "api/audio_codecs/opus/audio_encoder_multi_channel_opus_config.h"
#include "modules/audio_coding/codecs/opus/opus_interface.h"
#include "rtc_base/constructor_magic.h"
namespace webrtc {
class RtcEventLog;
class AudioEncoderMultiChannelOpusImpl final : public AudioEncoder {
public:
AudioEncoderMultiChannelOpusImpl(
const AudioEncoderMultiChannelOpusConfig& config,
int payload_type);
~AudioEncoderMultiChannelOpusImpl() override;
// Static interface for use by BuiltinAudioEncoderFactory.
static constexpr const char* GetPayloadName() { return "multiopus"; }
static absl::optional<AudioCodecInfo> QueryAudioEncoder(
const SdpAudioFormat& format);
int SampleRateHz() const override;
size_t NumChannels() const override;
size_t Num10MsFramesInNextPacket() const override;
size_t Max10MsFramesInAPacket() const override;
int GetTargetBitrate() const override;
void Reset() override;
protected:
EncodedInfo EncodeImpl(uint32_t rtp_timestamp,
rtc::ArrayView<const int16_t> audio,
rtc::Buffer* encoded) override;
private:
static absl::optional<AudioEncoderMultiChannelOpusConfig> SdpToConfig(
const SdpAudioFormat& format);
static AudioCodecInfo QueryAudioEncoder(
const AudioEncoderMultiChannelOpusConfig& config);
static std::unique_ptr<AudioEncoder> MakeAudioEncoder(
const AudioEncoderMultiChannelOpusConfig&,
int payload_type);
size_t Num10msFramesPerPacket() const;
size_t SamplesPer10msFrame() const;
size_t SufficientOutputBufferSize() const;
bool RecreateEncoderInstance(
const AudioEncoderMultiChannelOpusConfig& config);
void SetFrameLength(int frame_length_ms);
void SetNumChannelsToEncode(size_t num_channels_to_encode);
void SetProjectedPacketLossRate(float fraction);
AudioEncoderMultiChannelOpusConfig config_;
const int payload_type_;
std::vector<int16_t> input_buffer_;
OpusEncInst* inst_;
uint32_t first_timestamp_in_buffer_;
size_t num_channels_to_encode_;
int next_frame_length_ms_;
friend struct AudioEncoderMultiChannelOpus;
RTC_DISALLOW_COPY_AND_ASSIGN(AudioEncoderMultiChannelOpusImpl);
};
} // namespace webrtc
#endif // MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_ENCODER_MULTI_CHANNEL_OPUS_IMPL_H_

172
modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_unittest.cc Normal file
View File

@ -0,0 +1,172 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "api/audio_codecs/opus/audio_encoder_multi_channel_opus.h"
#include "test/gmock.h"
namespace webrtc {
using ::testing::NiceMock;
using ::testing::Return;
namespace {
constexpr int kOpusPayloadType = 120;
} // namespace
TEST(AudioEncoderMultiOpusTest, CheckConfigValidity) {
{
const SdpAudioFormat sdp_format("multiopus", 48000, 2,
{{"channel_mapping", "3,0"},
{"coupled_streams", "1"},
{"num_streams", "2"}});
const absl::optional<AudioEncoderMultiChannelOpus::Config> encoder_config =
AudioEncoderMultiChannelOpus::SdpToConfig(sdp_format);
ASSERT_TRUE(encoder_config.has_value());
// Maps input channel 0 to coded channel 3, which doesn't exist.
EXPECT_FALSE(encoder_config->IsOk());
}
{
const SdpAudioFormat sdp_format("multiopus", 48000, 2,
{{"channel_mapping", "0"},
{"coupled_streams", "1"},
{"num_streams", "2"}});
const absl::optional<AudioEncoderMultiChannelOpus::Config> encoder_config =
AudioEncoderMultiChannelOpus::SdpToConfig(sdp_format);
ASSERT_TRUE(encoder_config.has_value());
// The mapping is too short.
EXPECT_FALSE(encoder_config->IsOk());
}
{
const SdpAudioFormat sdp_format("multiopus", 48000, 3,
{{"channel_mapping", "0,0,0"},
{"coupled_streams", "0"},
{"num_streams", "1"}});
const absl::optional<AudioEncoderMultiChannelOpus::Config> encoder_config =
AudioEncoderMultiChannelOpus::SdpToConfig(sdp_format);
ASSERT_TRUE(encoder_config.has_value());
// Coded channel 0 comes from both input channels 0, 1 and 2.
EXPECT_FALSE(encoder_config->IsOk());
}
{
const SdpAudioFormat sdp_format("multiopus", 48000, 3,
{{"channel_mapping", "0,255,255"},
{"coupled_streams", "0"},
{"num_streams", "1"}});
const absl::optional<AudioEncoderMultiChannelOpus::Config> encoder_config =
AudioEncoderMultiChannelOpus::SdpToConfig(sdp_format);
ASSERT_TRUE(encoder_config.has_value());
// This is fine, because channels 1, 2 are set to be ignored.
EXPECT_TRUE(encoder_config->IsOk());
}
{
const SdpAudioFormat sdp_format("multiopus", 48000, 3,
{{"channel_mapping", "0,255,255"},
{"coupled_streams", "0"},
{"num_streams", "2"}});
const absl::optional<AudioEncoderMultiChannelOpus::Config> encoder_config =
AudioEncoderMultiChannelOpus::SdpToConfig(sdp_format);
ASSERT_TRUE(encoder_config.has_value());
// This is NOT fine, because channels nothing says how coded channel 1
// should be coded.
EXPECT_FALSE(encoder_config->IsOk());
}
}
TEST(AudioEncoderMultiOpusTest, ConfigValuesAreParsedCorrectly) {
SdpAudioFormat sdp_format({"multiopus",
48000,
6,
{{"minptime", "10"},
{"useinbandfec", "1"},
{"channel_mapping", "0,4,1,2,3,5"},
{"num_streams", "4"},
{"coupled_streams", "2"}}});
const absl::optional<AudioEncoderMultiChannelOpus::Config> encoder_config =
AudioEncoderMultiChannelOpus::SdpToConfig(sdp_format);
ASSERT_TRUE(encoder_config.has_value());
EXPECT_EQ(encoder_config->coupled_streams, 2);
EXPECT_EQ(encoder_config->num_streams, 4);
EXPECT_THAT(
encoder_config->channel_mapping,
testing::ContainerEq(std::vector<unsigned char>({0, 4, 1, 2, 3, 5})));
}
TEST(AudioEncoderMultiOpusTest, CreateFromValidOrInvalidConfig) {
{
const SdpAudioFormat sdp_format("multiopus", 48000, 3,
{{"channel_mapping", "0,255,255"},
{"coupled_streams", "0"},
{"num_streams", "2"}});
const absl::optional<AudioEncoderMultiChannelOpus::Config> encoder_config =
AudioEncoderMultiChannelOpus::SdpToConfig(sdp_format);
ASSERT_TRUE(encoder_config.has_value());
// Invalid config from the ConfigValidity test. It's not allowed by our
// checks, but Opus is more forgiving.
EXPECT_FALSE(encoder_config->IsOk());
const std::unique_ptr<AudioEncoder> opus_encoder =
AudioEncoderMultiChannelOpus::MakeAudioEncoder(*encoder_config,
kOpusPayloadType);
// Shouldn't be possible (but shouldn't result in a crash) to create an
// Encoder from an invalid config.
EXPECT_FALSE(opus_encoder);
}
{
const SdpAudioFormat sdp_format("multiopus", 48000, 3,
{{"channel_mapping", "1,255,0"},
{"coupled_streams", "1"},
{"num_streams", "1"}});
const absl::optional<AudioEncoderMultiChannelOpus::Config> encoder_config =
AudioEncoderMultiChannelOpus::SdpToConfig(sdp_format);
ASSERT_TRUE(encoder_config.has_value());
EXPECT_THAT(encoder_config->channel_mapping,
testing::ContainerEq(std::vector<unsigned char>({1, 255, 0})));
EXPECT_TRUE(encoder_config->IsOk());
const std::unique_ptr<AudioEncoder> opus_encoder =
AudioEncoderMultiChannelOpus::MakeAudioEncoder(*encoder_config,
kOpusPayloadType);
// Creating an encoder from a valid config should work.
EXPECT_TRUE(opus_encoder);
}
}
TEST(AudioEncoderMultiOpusTest, AdvertisedCodecsCanBeCreated) {
std::vector<AudioCodecSpec> specs;
AudioEncoderMultiChannelOpus::AppendSupportedEncoders(&specs);
EXPECT_FALSE(specs.empty());
for (const AudioCodecSpec& spec : specs) {
const absl::optional<AudioEncoderMultiChannelOpus::Config> encoder_config =
AudioEncoderMultiChannelOpus::SdpToConfig(spec.format);
ASSERT_TRUE(encoder_config.has_value());
const std::unique_ptr<AudioEncoder> opus_encoder =
AudioEncoderMultiChannelOpus::MakeAudioEncoder(*encoder_config,
kOpusPayloadType);
EXPECT_TRUE(opus_encoder);
}
}
} // namespace webrtc

2
test/fuzzers/BUILD.gn
View File

@ -344,7 +344,7 @@ webrtc_fuzzer_test("audio_decoder_multiopus_fuzzer") {
deps = [
":audio_decoder_fuzzer",
"../../api/audio_codecs/opus:audio_decoder_multiopus",
"../../api/audio_codecs/opus:audio_encoder_opus_config",
"../../api/audio_codecs/opus:audio_decoder_opus_config",
]
}