zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support external audio mixer in webrtc.

Browse Source 
An external audio mixer will be passed from PeerConnectionFactory to
AudioTransportProxy.

BUG=webrtc:6457

Review-Url: https://codereview.webrtc.org/2539213003
Cr-Commit-Position: refs/heads/master@{#15556}
This commit is contained in:
gyzhou
2016-12-12 16:25:06 -08:00
committed by Commit bot
parent 1354901663
commit f6bcac59e8
13 changed files with 121 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
webrtc/api/peerconnectionfactory.cc
View File

@ -57,27 +57,9 @@ rtc::scoped_refptr<PeerConnectionFactoryInterface> CreatePeerConnectionFactory(
AudioDeviceModule* default_adm,
cricket::WebRtcVideoEncoderFactory* encoder_factory,
cricket::WebRtcVideoDecoderFactory* decoder_factory) {
rtc::scoped_refptr<PeerConnectionFactory> pc_factory(
new rtc::RefCountedObject<PeerConnectionFactory>(
network_thread,
worker_thread,
signaling_thread,
default_adm,
CreateBuiltinAudioDecoderFactory(),
encoder_factory,
decoder_factory));
// Call Initialize synchronously but make sure its executed on
// |signaling_thread|.
MethodCall0<PeerConnectionFactory, bool> call(
pc_factory.get(),
&PeerConnectionFactory::Initialize);
bool result = call.Marshal(RTC_FROM_HERE, signaling_thread);
if (!result) {
return nullptr;
}
return PeerConnectionFactoryProxy::Create(signaling_thread, pc_factory);
return CreatePeerConnectionFactoryWithAudioMixer(
network_thread, worker_thread, signaling_thread, default_adm,
encoder_factory, decoder_factory, nullptr);
}
PeerConnectionFactory::PeerConnectionFactory()
@ -95,6 +77,33 @@ PeerConnectionFactory::PeerConnectionFactory()
worker_thread_->Start();
}
rtc::scoped_refptr<PeerConnectionFactoryInterface>
CreatePeerConnectionFactoryWithAudioMixer(
rtc::Thread* network_thread,
rtc::Thread* worker_thread,
rtc::Thread* signaling_thread,
AudioDeviceModule* default_adm,
cricket::WebRtcVideoEncoderFactory* encoder_factory,
cricket::WebRtcVideoDecoderFactory* decoder_factory,
rtc::scoped_refptr<AudioMixer> audio_mixer) {
rtc::scoped_refptr<PeerConnectionFactory> pc_factory(
new rtc::RefCountedObject<PeerConnectionFactory>(
network_thread, worker_thread, signaling_thread, default_adm,
CreateBuiltinAudioDecoderFactory(), encoder_factory, decoder_factory,
audio_mixer));
// Call Initialize synchronously but make sure it is executed on
// |signaling_thread|.
MethodCall0<PeerConnectionFactory, bool> call(
pc_factory.get(), &PeerConnectionFactory::Initialize);
bool result = call.Marshal(RTC_FROM_HERE, signaling_thread);
if (!result) {
return nullptr;
}
return PeerConnectionFactoryProxy::Create(signaling_thread, pc_factory);
}
PeerConnectionFactory::PeerConnectionFactory(
rtc::Thread* network_thread,
rtc::Thread* worker_thread,
@ -103,7 +112,8 @@ PeerConnectionFactory::PeerConnectionFactory(
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>&
audio_decoder_factory,
cricket::WebRtcVideoEncoderFactory* video_encoder_factory,
cricket::WebRtcVideoDecoderFactory* video_decoder_factory)
cricket::WebRtcVideoDecoderFactory* video_decoder_factory,
rtc::scoped_refptr<AudioMixer> audio_mixer)
: owns_ptrs_(false),
wraps_current_thread_(false),
network_thread_(network_thread),
@ -112,7 +122,8 @@ PeerConnectionFactory::PeerConnectionFactory(
default_adm_(default_adm),
audio_decoder_factory_(audio_decoder_factory),
video_encoder_factory_(video_encoder_factory),
video_decoder_factory_(video_decoder_factory) {
video_decoder_factory_(video_decoder_factory),
external_audio_mixer_(audio_mixer) {
RTC_DCHECK(network_thread);
RTC_DCHECK(worker_thread);
RTC_DCHECK(signaling_thread);
@ -336,10 +347,8 @@ rtc::Thread* PeerConnectionFactory::network_thread() {
cricket::MediaEngineInterface* PeerConnectionFactory::CreateMediaEngine_w() {
ASSERT(worker_thread_ == rtc::Thread::Current());
return cricket::WebRtcMediaEngineFactory::Create(
default_adm_.get(),
audio_decoder_factory_,
video_encoder_factory_.get(),
video_decoder_factory_.get());
default_adm_.get(), audio_decoder_factory_, video_encoder_factory_.get(),
video_decoder_factory_.get(), external_audio_mixer_);
}
} // namespace webrtc

6
webrtc/api/peerconnectionfactory.h
View File

@ -111,7 +111,8 @@ class PeerConnectionFactory : public PeerConnectionFactoryInterface {
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>&
audio_decoder_factory,
cricket::WebRtcVideoEncoderFactory* video_encoder_factory,
cricket::WebRtcVideoDecoderFactory* video_decoder_factory);
cricket::WebRtcVideoDecoderFactory* video_decoder_factory,
rtc::scoped_refptr<AudioMixer> audio_mixer);
virtual ~PeerConnectionFactory();
private:
@ -135,6 +136,9 @@ class PeerConnectionFactory : public PeerConnectionFactoryInterface {
std::unique_ptr<cricket::WebRtcVideoDecoderFactory> video_decoder_factory_;
std::unique_ptr<rtc::BasicNetworkManager> default_network_manager_;
std::unique_ptr<rtc::BasicPacketSocketFactory> default_socket_factory_;
// External audio mixer. This can be NULL. In that case, internal audio mixer
// will be created and used.
rtc::scoped_refptr<AudioMixer> external_audio_mixer_;
};
} // namespace webrtc

15
webrtc/api/peerconnectioninterface.h
View File

@ -87,6 +87,7 @@ class WebRtcVideoEncoderFactory;
namespace webrtc {
class AudioDeviceModule;
class AudioMixer;
class MediaConstraintsInterface;
// MediaStream container interface.
@ -803,6 +804,20 @@ rtc::scoped_refptr<PeerConnectionFactoryInterface> CreatePeerConnectionFactory(
cricket::WebRtcVideoEncoderFactory* encoder_factory,
cricket::WebRtcVideoDecoderFactory* decoder_factory);
// Create a new instance of PeerConnectionFactoryInterface with external audio
// mixer.
//
// If |audio_mixer| is null, an internal audio mixer will be created and used.
rtc::scoped_refptr<PeerConnectionFactoryInterface>
CreatePeerConnectionFactoryWithAudioMixer(
rtc::Thread* network_thread,
rtc::Thread* worker_thread,
rtc::Thread* signaling_thread,
AudioDeviceModule* default_adm,
cricket::WebRtcVideoEncoderFactory* encoder_factory,
cricket::WebRtcVideoDecoderFactory* decoder_factory,
rtc::scoped_refptr<AudioMixer> audio_mixer);
// Create a new instance of PeerConnectionFactoryInterface.
// Same thread is used as worker and network thread.
inline rtc::scoped_refptr<PeerConnectionFactoryInterface>

2
webrtc/audio/audio_receive_stream.cc
View File

@ -293,7 +293,7 @@ int AudioReceiveStream::PreferredSampleRate() const {
}
int AudioReceiveStream::Ssrc() const {
return config_.rtp.local_ssrc;
return config_.rtp.remote_ssrc;
}
internal::AudioState* AudioReceiveStream::audio_state() const {

9
webrtc/media/base/fakemediaengine.h
View File

@ -753,10 +753,10 @@ class FakeBaseEngine {
class FakeVoiceEngine : public FakeBaseEngine {
public:
FakeVoiceEngine(
webrtc::AudioDeviceModule* adm,
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>&
audio_decoder_factory) {
FakeVoiceEngine(webrtc::AudioDeviceModule* adm,
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>&
audio_decoder_factory,
rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer) {
// Add a fake audio codec. Note that the name must not be "" as there are
// sanity checks against that.
codecs_.push_back(AudioCodec(101, "fake_audio_codec", 0, 0, 1));
@ -864,6 +864,7 @@ class FakeMediaEngine :
public:
FakeMediaEngine()
: CompositeMediaEngine<FakeVoiceEngine, FakeVideoEngine>(nullptr,
nullptr,
nullptr) {}
virtual ~FakeMediaEngine() {}

11
webrtc/media/base/mediaengine.h
View File

@ -33,6 +33,7 @@
namespace webrtc {
class AudioDeviceModule;
class AudioMixer;
class Call;
}
@ -110,11 +111,11 @@ class MediaEngineFactory {
template<class VOICE, class VIDEO>
class CompositeMediaEngine : public MediaEngineInterface {
public:
CompositeMediaEngine(
webrtc::AudioDeviceModule* adm,
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>&
audio_decoder_factory)
: voice_(adm, audio_decoder_factory) {}
CompositeMediaEngine(webrtc::AudioDeviceModule* adm,
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>&
audio_decoder_factory,
rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer)
: voice_(adm, audio_decoder_factory, audio_mixer) {}
virtual ~CompositeMediaEngine() {}
virtual bool Init() {
video_.Init();

4
webrtc/media/engine/nullwebrtcvideoengine_unittest.cc
View File

@ -25,7 +25,9 @@ class WebRtcMediaEngineNullVideo
WebRtcVideoEncoderFactory* video_encoder_factory,
WebRtcVideoDecoderFactory* video_decoder_factory)
: CompositeMediaEngine<WebRtcVoiceEngine, NullWebRtcVideoEngine>(
adm, audio_decoder_factory) {
adm,
audio_decoder_factory,
nullptr) {
video_.SetExternalDecoderFactory(video_decoder_factory);
video_.SetExternalEncoderFactory(video_encoder_factory);
}

31
webrtc/media/engine/webrtcmediaengine.cc
View File

@ -33,13 +33,18 @@ class WebRtcMediaEngine2
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>&
audio_decoder_factory,
WebRtcVideoEncoderFactory* video_encoder_factory,
WebRtcVideoDecoderFactory* video_decoder_factory)
WebRtcVideoDecoderFactory* video_decoder_factory,
rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer)
#ifdef HAVE_WEBRTC_VIDEO
: CompositeMediaEngine<WebRtcVoiceEngine, WebRtcVideoEngine2>(
adm, audio_decoder_factory){
adm,
audio_decoder_factory,
audio_mixer){
#else
: CompositeMediaEngine<WebRtcVoiceEngine, NullWebRtcVideoEngine>(
adm, audio_decoder_factory) {
adm,
audio_decoder_factory,
audio_mixer) {
#endif
video_.SetExternalDecoderFactory(video_decoder_factory);
video_.SetExternalEncoderFactory(video_encoder_factory);
@ -53,9 +58,11 @@ cricket::MediaEngineInterface* CreateWebRtcMediaEngine(
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>&
audio_decoder_factory,
cricket::WebRtcVideoEncoderFactory* video_encoder_factory,
cricket::WebRtcVideoDecoderFactory* video_decoder_factory) {
return new cricket::WebRtcMediaEngine2(
adm, audio_decoder_factory, video_encoder_factory, video_decoder_factory);
cricket::WebRtcVideoDecoderFactory* video_decoder_factory,
rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer) {
return new cricket::WebRtcMediaEngine2(adm, audio_decoder_factory,
video_encoder_factory,
video_decoder_factory, audio_mixer);
}
void DestroyWebRtcMediaEngine(cricket::MediaEngineInterface* media_engine) {
@ -71,9 +78,9 @@ MediaEngineInterface* WebRtcMediaEngineFactory::Create(
webrtc::AudioDeviceModule* adm,
WebRtcVideoEncoderFactory* video_encoder_factory,
WebRtcVideoDecoderFactory* video_decoder_factory) {
return CreateWebRtcMediaEngine(adm,
webrtc::CreateBuiltinAudioDecoderFactory(),
video_encoder_factory, video_decoder_factory);
return CreateWebRtcMediaEngine(
adm, webrtc::CreateBuiltinAudioDecoderFactory(), video_encoder_factory,
video_decoder_factory, nullptr);
}
// Used by PeerConnectionFactory to create a media engine passed into
@ -83,9 +90,11 @@ MediaEngineInterface* WebRtcMediaEngineFactory::Create(
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>&
audio_decoder_factory,
WebRtcVideoEncoderFactory* video_encoder_factory,
WebRtcVideoDecoderFactory* video_decoder_factory) {
WebRtcVideoDecoderFactory* video_decoder_factory,
rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer) {
return CreateWebRtcMediaEngine(adm, audio_decoder_factory,
video_encoder_factory, video_decoder_factory);
video_encoder_factory, video_decoder_factory,
audio_mixer);
}
namespace {

4
webrtc/media/engine/webrtcmediaengine.h
View File

@ -21,6 +21,7 @@
namespace webrtc {
class AudioDecoderFactory;
class AudioDeviceModule;
class AudioMixer;
}
namespace cricket {
class WebRtcVideoDecoderFactory;
@ -44,7 +45,8 @@ class WebRtcMediaEngineFactory {
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>&
audio_decoder_factory,
WebRtcVideoEncoderFactory* video_encoder_factory,
WebRtcVideoDecoderFactory* video_decoder_factory);
WebRtcVideoDecoderFactory* video_decoder_factory,
rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer);
};
// Verify that extension IDs are within 1-byte extension range and are not

3
webrtc/media/engine/webrtcmediaengine_unittest.cc
View File

@ -188,7 +188,8 @@ TEST(WebRtcMediaEngineFactoryTest, CreateOldApi) {
TEST(WebRtcMediaEngineFactoryTest, CreateWithBuiltinDecoders) {
std::unique_ptr<MediaEngineInterface> engine(WebRtcMediaEngineFactory::Create(
nullptr, webrtc::CreateBuiltinAudioDecoderFactory(), nullptr, nullptr));
nullptr, webrtc::CreateBuiltinAudioDecoderFactory(), nullptr, nullptr,
nullptr));
EXPECT_TRUE(engine);
}

19
webrtc/media/engine/webrtcvoiceengine.cc
View File

@ -277,10 +277,16 @@ void GetOpusConfig(const AudioCodec& codec,
voe_codec->rate = GetOpusBitrate(codec, *max_playback_rate);
}
webrtc::AudioState::Config MakeAudioStateConfig(VoEWrapper* voe_wrapper) {
webrtc::AudioState::Config MakeAudioStateConfig(
VoEWrapper* voe_wrapper,
rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer) {
webrtc::AudioState::Config config;
config.voice_engine = voe_wrapper->engine();
config.audio_mixer = webrtc::AudioMixerImpl::Create();
if (audio_mixer) {
config.audio_mixer = audio_mixer;
} else {
config.audio_mixer = webrtc::AudioMixerImpl::Create();
}
return config;
}
@ -540,14 +546,17 @@ bool WebRtcVoiceEngine::ToCodecInst(const AudioCodec& in,
WebRtcVoiceEngine::WebRtcVoiceEngine(
webrtc::AudioDeviceModule* adm,
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory)
: WebRtcVoiceEngine(adm, decoder_factory, new VoEWrapper()) {
audio_state_ = webrtc::AudioState::Create(MakeAudioStateConfig(voe()));
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory,
rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer)
: WebRtcVoiceEngine(adm, decoder_factory, audio_mixer, new VoEWrapper()) {
audio_state_ =
webrtc::AudioState::Create(MakeAudioStateConfig(voe(), audio_mixer));
}
WebRtcVoiceEngine::WebRtcVoiceEngine(
webrtc::AudioDeviceModule* adm,
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory,
rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer,
VoEWrapper* voe_wrapper)
: adm_(adm), decoder_factory_(decoder_factory), voe_wrapper_(voe_wrapper) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());

5
webrtc/media/engine/webrtcvoiceengine.h
View File

@ -34,6 +34,7 @@
namespace cricket {
class AudioDeviceModule;
class AudioMixer;
class AudioSource;
class VoEWrapper;
class WebRtcVoiceMediaChannel;
@ -48,11 +49,13 @@ class WebRtcVoiceEngine final : public webrtc::TraceCallback {
WebRtcVoiceEngine(
webrtc::AudioDeviceModule* adm,
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory);
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory,
rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer);
// Dependency injection for testing.
WebRtcVoiceEngine(
webrtc::AudioDeviceModule* adm,
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory,
rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer,
VoEWrapper* voe_wrapper);
~WebRtcVoiceEngine() override;

14
webrtc/media/engine/webrtcvoiceengine_unittest.cc
View File

@ -84,7 +84,7 @@ TEST(WebRtcVoiceEngineTestStubLibrary, StartupShutdown) {
EXPECT_FALSE(voe.IsInited());
{
cricket::WebRtcVoiceEngine engine(
&adm, webrtc::MockAudioDecoderFactory::CreateUnusedFactory(),
&adm, webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr,
new FakeVoEWrapper(&voe));
EXPECT_TRUE(voe.IsInited());
}
@ -116,7 +116,7 @@ class WebRtcVoiceEngineTestFake : public testing::Test {
EXPECT_CALL(apm_, ApplyConfig(testing::_));
EXPECT_CALL(apm_, SetExtraOptions(testing::_));
EXPECT_CALL(apm_, Initialize()).WillOnce(Return(0));
engine_.reset(new cricket::WebRtcVoiceEngine(&adm_, factory,
engine_.reset(new cricket::WebRtcVoiceEngine(&adm_, factory, nullptr,
new FakeVoEWrapper(&voe_)));
send_parameters_.codecs.push_back(kPcmuCodec);
recv_parameters_.codecs.push_back(kPcmuCodec);
@ -3466,7 +3466,7 @@ TEST(WebRtcVoiceEngineTest, StartupShutdown) {
// If the VoiceEngine wants to gather available codecs early, that's fine but
// we never want it to create a decoder at this stage.
cricket::WebRtcVoiceEngine engine(
nullptr, webrtc::MockAudioDecoderFactory::CreateUnusedFactory());
nullptr, webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr);
webrtc::RtcEventLogNullImpl event_log;
std::unique_ptr<webrtc::Call> call(
webrtc::Call::Create(webrtc::Call::Config(&event_log)));
@ -3483,7 +3483,7 @@ TEST(WebRtcVoiceEngineTest, StartupShutdownWithExternalADM) {
EXPECT_CALL(adm, Release()).Times(3).WillRepeatedly(Return(0));
{
cricket::WebRtcVoiceEngine engine(
&adm, webrtc::MockAudioDecoderFactory::CreateUnusedFactory());
&adm, webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr);
webrtc::RtcEventLogNullImpl event_log;
std::unique_ptr<webrtc::Call> call(
webrtc::Call::Create(webrtc::Call::Config(&event_log)));
@ -3551,7 +3551,7 @@ TEST(WebRtcVoiceEngineTest, HasCorrectCodecs) {
// TODO(ossu): Why are the payload types of codecs with non-static payload
// type assignments checked here? It shouldn't really matter.
cricket::WebRtcVoiceEngine engine(
nullptr, webrtc::MockAudioDecoderFactory::CreateUnusedFactory());
nullptr, webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr);
for (const cricket::AudioCodec& codec : engine.send_codecs()) {
if (codec.name == "CN" && codec.clockrate == 16000) {
EXPECT_EQ(105, codec.id);
@ -3587,7 +3587,7 @@ TEST(WebRtcVoiceEngineTest, HasCorrectCodecs) {
// Tests that VoE supports at least 32 channels
TEST(WebRtcVoiceEngineTest, Has32Channels) {
cricket::WebRtcVoiceEngine engine(
nullptr, webrtc::MockAudioDecoderFactory::CreateUnusedFactory());
nullptr, webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr);
webrtc::RtcEventLogNullImpl event_log;
std::unique_ptr<webrtc::Call> call(
webrtc::Call::Create(webrtc::Call::Config(&event_log)));
@ -3620,7 +3620,7 @@ TEST(WebRtcVoiceEngineTest, SetRecvCodecs) {
// SetRecvParameters returns true.
// I think it will become clear once audio decoder injection is completed.
cricket::WebRtcVoiceEngine engine(
nullptr, webrtc::CreateBuiltinAudioDecoderFactory());
nullptr, webrtc::CreateBuiltinAudioDecoderFactory(), nullptr);
webrtc::RtcEventLogNullImpl event_log;
std::unique_ptr<webrtc::Call> call(
webrtc::Call::Create(webrtc::Call::Config(&event_log)));