diff --git a/test/pc/e2e/peer_connection_e2e_smoke_test.cc b/test/pc/e2e/peer_connection_e2e_smoke_test.cc index b09b093c25..39dbbf6269 100644 --- a/test/pc/e2e/peer_connection_e2e_smoke_test.cc +++ b/test/pc/e2e/peer_connection_e2e_smoke_test.cc @@ -46,74 +46,58 @@ class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test { using EchoEmulationConfig = PeerConnectionE2EQualityTestFixture::EchoEmulationConfig; - void RunTest(const std::string& test_case_name, - const RunParams& run_params, - rtc::FunctionView alice_configurer, - rtc::FunctionView bob_configurer) { - // Setup emulated network - std::unique_ptr network_emulation_manager = - CreateNetworkEmulationManager(); - - auto alice_network_behavior = - std::make_unique(BuiltInNetworkBehaviorConfig()); - SimulatedNetwork* alice_network_behavior_ptr = alice_network_behavior.get(); - EmulatedNetworkNode* alice_node = - network_emulation_manager->CreateEmulatedNode( - std::move(alice_network_behavior)); - EmulatedNetworkNode* bob_node = - network_emulation_manager->CreateEmulatedNode( - std::make_unique(BuiltInNetworkBehaviorConfig())); - auto* alice_endpoint = - network_emulation_manager->CreateEndpoint(EmulatedEndpointConfig()); - EmulatedEndpoint* bob_endpoint = - network_emulation_manager->CreateEndpoint(EmulatedEndpointConfig()); - network_emulation_manager->CreateRoute(alice_endpoint, {alice_node}, - bob_endpoint); - network_emulation_manager->CreateRoute(bob_endpoint, {bob_node}, - alice_endpoint); - - // Create analyzers. - std::unique_ptr video_quality_analyzer = - std::make_unique( - network_emulation_manager->time_controller()->GetClock()); - // This is only done for the sake of smoke testing. In general there should - // be no need to explicitly pull data from analyzers after the run. - auto* video_analyzer_ptr = - static_cast(video_quality_analyzer.get()); - - auto fixture = CreatePeerConnectionE2EQualityTestFixture( - test_case_name, *network_emulation_manager->time_controller(), + void SetUp() override { + network_emulation_ = CreateNetworkEmulationManager(); + auto video_quality_analyzer = std::make_unique( + network_emulation_->time_controller()->GetClock()); + video_quality_analyzer_ = video_quality_analyzer.get(); + fixture_ = CreatePeerConnectionE2EQualityTestFixture( + testing::UnitTest::GetInstance()->current_test_info()->name(), + *network_emulation_->time_controller(), /*audio_quality_analyzer=*/nullptr, std::move(video_quality_analyzer)); - fixture->ExecuteAt(TimeDelta::Seconds(1), - [alice_network_behavior_ptr](TimeDelta) { - BuiltInNetworkBehaviorConfig config; - config.loss_percent = 5; - alice_network_behavior_ptr->SetConfig(config); - }); + test::ScopedFieldTrials field_trials( + std::string(field_trial::GetFieldTrialString()) + + "WebRTC-UseStandardBytesStats/Enabled/"); + } + + std::pair + CreateNetwork() { + EmulatedNetworkNode* alice_node = network_emulation_->CreateEmulatedNode( + std::make_unique(BuiltInNetworkBehaviorConfig())); + EmulatedNetworkNode* bob_node = network_emulation_->CreateEmulatedNode( + std::make_unique(BuiltInNetworkBehaviorConfig())); + + EmulatedEndpoint* alice_endpoint = + network_emulation_->CreateEndpoint(EmulatedEndpointConfig()); + EmulatedEndpoint* bob_endpoint = + network_emulation_->CreateEndpoint(EmulatedEndpointConfig()); + + network_emulation_->CreateRoute(alice_endpoint, {alice_node}, bob_endpoint); + network_emulation_->CreateRoute(bob_endpoint, {bob_node}, alice_endpoint); - // Setup components. We need to provide rtc::NetworkManager compatible with - // emulated network layer. EmulatedNetworkManagerInterface* alice_network = - network_emulation_manager->CreateEmulatedNetworkManagerInterface( + network_emulation_->CreateEmulatedNetworkManagerInterface( {alice_endpoint}); EmulatedNetworkManagerInterface* bob_network = - network_emulation_manager->CreateEmulatedNetworkManagerInterface( + network_emulation_->CreateEmulatedNetworkManagerInterface( {bob_endpoint}); - fixture->AddPeer(alice_network->network_thread(), - alice_network->network_manager(), alice_configurer); - fixture->AddPeer(bob_network->network_thread(), - bob_network->network_manager(), bob_configurer); - fixture->AddQualityMetricsReporter( - std::make_unique(alice_network, - bob_network)); + return std::make_pair(alice_network, bob_network); + } - fixture->Run(run_params); + void AddPeer(EmulatedNetworkManagerInterface* network, + rtc::FunctionView configurer) { + fixture_->AddPeer(network->network_thread(), network->network_manager(), + configurer); + } - EXPECT_GE(fixture->GetRealTestDuration(), run_params.run_duration); - for (auto stream_key : video_analyzer_ptr->GetKnownVideoStreams()) { + void RunAndCheckEachVideoStreamReceivedFrames(const RunParams& run_params) { + fixture_->Run(run_params); + + EXPECT_GE(fixture_->GetRealTestDuration(), run_params.run_duration); + for (auto stream_key : video_quality_analyzer_->GetKnownVideoStreams()) { FrameCounters stream_conters = - video_analyzer_ptr->GetPerStreamCounters().at(stream_key); + video_quality_analyzer_->GetPerStreamCounters().at(stream_key); // On some devices the pipeline can be too slow, so we actually can't // force real constraints here. Lets just check, that at least 1 // frame passed whole pipeline. @@ -127,9 +111,18 @@ class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test { EXPECT_GE(stream_conters.rendered, 1) << stream_key.ToString(); } } -}; -} // namespace + NetworkEmulationManager* network_emulation() { + return network_emulation_.get(); + } + + PeerConnectionE2EQualityTestFixture* fixture() { return fixture_.get(); } + + private: + std::unique_ptr network_emulation_; + DefaultVideoQualityAnalyzer* video_quality_analyzer_; + std::unique_ptr fixture_; +}; // IOS debug builds can be quite slow, disabling to avoid issues with timeouts. #if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) @@ -138,46 +131,108 @@ class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test { #define MAYBE_Smoke Smoke #endif TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Smoke) { + std::pair + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + VideoConfig video(160, 120, 15); + video.stream_label = "alice-video"; + video.sync_group = "alice-media"; + alice->AddVideoConfig(std::move(video)); + + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + audio.sampling_frequency_in_hz = 48000; + audio.sync_group = "alice-media"; + alice->SetAudioConfig(std::move(audio)); + }); + AddPeer(network_links.second, [](PeerConfigurer* charlie) { + charlie->SetName("charlie"); + VideoConfig video(160, 120, 15); + video.stream_label = "charlie-video"; + video.temporal_layers_count = 2; + charlie->AddVideoConfig(std::move(video)); + + AudioConfig audio; + audio.stream_label = "charlie-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_bob_source", "wav"); + charlie->SetAudioConfig(std::move(audio)); + }); + fixture()->AddQualityMetricsReporter( + std::make_unique(network_links.first, + network_links.second)); RunParams run_params(TimeDelta::Seconds(2)); run_params.video_codecs = { VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}; run_params.use_flex_fec = true; run_params.use_ulp_fec = true; run_params.video_encoder_bitrate_multiplier = 1.1; - test::ScopedFieldTrials field_trials( - std::string(field_trial::GetFieldTrialString()) + - "WebRTC-UseStandardBytesStats/Enabled/"); - RunTest( - "smoke", run_params, - [](PeerConfigurer* alice) { - VideoConfig video(160, 120, 15); - video.stream_label = "alice-video"; - video.sync_group = "alice-media"; - alice->AddVideoConfig(std::move(video)); + RunAndCheckEachVideoStreamReceivedFrames(run_params); +} - AudioConfig audio; - audio.stream_label = "alice-audio"; - audio.mode = AudioConfig::Mode::kFile; - audio.input_file_name = - test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); - audio.sampling_frequency_in_hz = 48000; - audio.sync_group = "alice-media"; - alice->SetAudioConfig(std::move(audio)); - }, - [](PeerConfigurer* charlie) { - charlie->SetName("charlie"); - VideoConfig video(160, 120, 15); - video.stream_label = "charlie-video"; - video.temporal_layers_count = 2; - charlie->AddVideoConfig(std::move(video)); +// IOS debug builds can be quite slow, disabling to avoid issues with timeouts. +#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) +#define MAYBE_ChangeNetworkConditions DISABLED_ChangeNetworkConditions +#else +#define MAYBE_ChangeNetworkConditions ChangeNetworkConditions +#endif +TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_ChangeNetworkConditions) { + NetworkEmulationManager::SimulatedNetworkNode alice_node = + network_emulation() + ->NodeBuilder() + .config(BuiltInNetworkBehaviorConfig()) + .Build(); + NetworkEmulationManager::SimulatedNetworkNode bob_node = + network_emulation() + ->NodeBuilder() + .config(BuiltInNetworkBehaviorConfig()) + .Build(); - AudioConfig audio; - audio.stream_label = "charlie-audio"; - audio.mode = AudioConfig::Mode::kFile; - audio.input_file_name = - test::ResourcePath("pc_quality_smoke_test_bob_source", "wav"); - charlie->SetAudioConfig(std::move(audio)); - }); + EmulatedEndpoint* alice_endpoint = + network_emulation()->CreateEndpoint(EmulatedEndpointConfig()); + EmulatedEndpoint* bob_endpoint = + network_emulation()->CreateEndpoint(EmulatedEndpointConfig()); + + network_emulation()->CreateRoute(alice_endpoint, {alice_node.node}, + bob_endpoint); + network_emulation()->CreateRoute(bob_endpoint, {bob_node.node}, + alice_endpoint); + + EmulatedNetworkManagerInterface* alice_network = + network_emulation()->CreateEmulatedNetworkManagerInterface( + {alice_endpoint}); + EmulatedNetworkManagerInterface* bob_network = + network_emulation()->CreateEmulatedNetworkManagerInterface( + {bob_endpoint}); + + AddPeer(alice_network, [](PeerConfigurer* alice) { + VideoConfig video(160, 120, 15); + video.stream_label = "alice-video"; + video.sync_group = "alice-media"; + alice->AddVideoConfig(std::move(video)); + }); + AddPeer(bob_network, [](PeerConfigurer* bob) {}); + fixture()->AddQualityMetricsReporter( + std::make_unique(alice_network, + bob_network)); + + fixture()->ExecuteAt(TimeDelta::Seconds(1), [alice_node](TimeDelta) { + BuiltInNetworkBehaviorConfig config; + config.loss_percent = 5; + alice_node.simulation->SetConfig(config); + }); + + RunParams run_params(TimeDelta::Seconds(2)); + run_params.video_codecs = { + VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}; + run_params.use_flex_fec = true; + run_params.use_ulp_fec = true; + run_params.video_encoder_bitrate_multiplier = 1.1; + RunAndCheckEachVideoStreamReceivedFrames(run_params); } // IOS debug builds can be quite slow, disabling to avoid issues with timeouts. @@ -187,13 +242,10 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Smoke) { #define MAYBE_Screenshare Screenshare #endif TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Screenshare) { - RunParams run_params(TimeDelta::Seconds(2)); - test::ScopedFieldTrials field_trials( - std::string(field_trial::GetFieldTrialString()) + - "WebRTC-UseStandardBytesStats/Enabled/"); - RunTest( - "screenshare", run_params, - [](PeerConfigurer* alice) { + std::pair + network_links = CreateNetwork(); + AddPeer( + network_links.first, [](PeerConfigurer* alice) { VideoConfig screenshare(320, 180, 30); screenshare.stream_label = "alice-screenshare"; screenshare.content_hint = VideoTrackInterface::ContentHint::kText; @@ -205,8 +257,9 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Screenshare) { CreateScreenShareFrameGenerator(screenshare, screen_share_config); alice->AddVideoConfig(std::move(screenshare), std::move(screen_share_frame_generator)); - }, - [](PeerConfigurer* charlie) {}); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) {}); + RunAndCheckEachVideoStreamReceivedFrames(RunParams(TimeDelta::Seconds(2))); } // IOS debug builds can be quite slow, disabling to avoid issues with timeouts. @@ -216,27 +269,28 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Screenshare) { #define MAYBE_Echo Echo #endif TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Echo) { + std::pair + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + audio.sampling_frequency_in_hz = 48000; + alice->SetAudioConfig(std::move(audio)); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) { + AudioConfig audio; + audio.stream_label = "bob-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_bob_source", "wav"); + bob->SetAudioConfig(std::move(audio)); + }); RunParams run_params(TimeDelta::Seconds(2)); run_params.echo_emulation_config = EchoEmulationConfig(); - RunTest( - "smoke", run_params, - [](PeerConfigurer* alice) { - AudioConfig audio; - audio.stream_label = "alice-audio"; - audio.mode = AudioConfig::Mode::kFile; - audio.input_file_name = - test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); - audio.sampling_frequency_in_hz = 48000; - alice->SetAudioConfig(std::move(audio)); - }, - [](PeerConfigurer* bob) { - AudioConfig audio; - audio.stream_label = "bob-audio"; - audio.mode = AudioConfig::Mode::kFile; - audio.input_file_name = - test::ResourcePath("pc_quality_smoke_test_bob_source", "wav"); - bob->SetAudioConfig(std::move(audio)); - }); + RunAndCheckEachVideoStreamReceivedFrames(run_params); } // IOS debug builds can be quite slow, disabling to avoid issues with timeouts. @@ -246,24 +300,25 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Echo) { #define MAYBE_Simulcast Simulcast #endif TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Simulcast) { + std::pair + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + VideoConfig simulcast(1280, 720, 15); + simulcast.stream_label = "alice-simulcast"; + simulcast.simulcast_config = VideoSimulcastConfig(2, 0); + alice->AddVideoConfig(std::move(simulcast)); + + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + alice->SetAudioConfig(std::move(audio)); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) {}); RunParams run_params(TimeDelta::Seconds(2)); run_params.video_codecs = {VideoCodecConfig(cricket::kVp8CodecName)}; - RunTest( - "simulcast", run_params, - [](PeerConfigurer* alice) { - VideoConfig simulcast(1280, 720, 15); - simulcast.stream_label = "alice-simulcast"; - simulcast.simulcast_config = VideoSimulcastConfig(2, 0); - alice->AddVideoConfig(std::move(simulcast)); - - AudioConfig audio; - audio.stream_label = "alice-audio"; - audio.mode = AudioConfig::Mode::kFile; - audio.input_file_name = - test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); - alice->SetAudioConfig(std::move(audio)); - }, - [](PeerConfigurer* bob) {}); + RunAndCheckEachVideoStreamReceivedFrames(run_params); } // IOS debug builds can be quite slow, disabling to avoid issues with timeouts. @@ -273,26 +328,27 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Simulcast) { #define MAYBE_Svc Svc #endif TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Svc) { + std::pair + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + VideoConfig simulcast(1280, 720, 15); + simulcast.stream_label = "alice-svc"; + // Because we have network with packets loss we can analyze only the + // highest spatial layer in SVC mode. + simulcast.simulcast_config = VideoSimulcastConfig(2, 1); + alice->AddVideoConfig(std::move(simulcast)); + + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + alice->SetAudioConfig(std::move(audio)); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) {}); RunParams run_params(TimeDelta::Seconds(2)); run_params.video_codecs = {VideoCodecConfig(cricket::kVp9CodecName)}; - RunTest( - "simulcast", run_params, - [](PeerConfigurer* alice) { - VideoConfig simulcast(1280, 720, 15); - simulcast.stream_label = "alice-svc"; - // Because we have network with packets loss we can analyze only the - // highest spatial layer in SVC mode. - simulcast.simulcast_config = VideoSimulcastConfig(2, 1); - alice->AddVideoConfig(std::move(simulcast)); - - AudioConfig audio; - audio.stream_label = "alice-audio"; - audio.mode = AudioConfig::Mode::kFile; - audio.input_file_name = - test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); - alice->SetAudioConfig(std::move(audio)); - }, - [](PeerConfigurer* bob) {}); + RunAndCheckEachVideoStreamReceivedFrames(run_params); } // IOS debug builds can be quite slow, disabling to avoid issues with timeouts. @@ -302,33 +358,34 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Svc) { #define MAYBE_HighBitrate HighBitrate #endif TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_HighBitrate) { + std::pair + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + BitrateSettings bitrate_settings; + bitrate_settings.start_bitrate_bps = 3'000'000; + bitrate_settings.max_bitrate_bps = 3'000'000; + alice->SetBitrateSettings(bitrate_settings); + VideoConfig video(800, 600, 15); + video.stream_label = "alice-video"; + video.min_encode_bitrate_bps = 500'000; + video.max_encode_bitrate_bps = 3'000'000; + alice->AddVideoConfig(std::move(video)); + + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + audio.sampling_frequency_in_hz = 48000; + alice->SetAudioConfig(std::move(audio)); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) {}); RunParams run_params(TimeDelta::Seconds(2)); run_params.video_codecs = { VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}; - - RunTest( - "smoke", run_params, - [](PeerConfigurer* alice) { - BitrateSettings bitrate_settings; - bitrate_settings.start_bitrate_bps = 3'000'000; - bitrate_settings.max_bitrate_bps = 3'000'000; - alice->SetBitrateSettings(bitrate_settings); - VideoConfig video(800, 600, 15); - video.stream_label = "alice-video"; - video.min_encode_bitrate_bps = 500'000; - video.max_encode_bitrate_bps = 3'000'000; - alice->AddVideoConfig(std::move(video)); - - AudioConfig audio; - audio.stream_label = "alice-audio"; - audio.mode = AudioConfig::Mode::kFile; - audio.input_file_name = - test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); - audio.sampling_frequency_in_hz = 48000; - alice->SetAudioConfig(std::move(audio)); - }, - [](PeerConfigurer* bob) {}); + RunAndCheckEachVideoStreamReceivedFrames(run_params); } +} // namespace } // namespace webrtc_pc_e2e } // namespace webrtc