From 048a1885b4eb0d4554d76af434e426657652c605 Mon Sep 17 00:00:00 2001 From: Artem Titov Date: Mon, 3 Aug 2020 21:18:11 +0200 Subject: [PATCH] Refactor PC level smoke test for adding new network stats reporter New network stats reporter will require peer name to be passed into constructor. Because peer name can be set inside PeerConfigurer adding of the reporter have to be moved to PeerConfigurer creation. Bug: webrtc:11756 Change-Id: I4840566a3cd2c73fbb1d186aa1c40ebf889fd830 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/180805 Commit-Queue: Artem Titov Reviewed-by: Andrey Logvin Cr-Commit-Position: refs/heads/master@{#31841} --- test/pc/e2e/peer_connection_e2e_smoke_test.cc | 411 ++++++++++-------- 1 file changed, 234 insertions(+), 177 deletions(-) diff --git a/test/pc/e2e/peer_connection_e2e_smoke_test.cc b/test/pc/e2e/peer_connection_e2e_smoke_test.cc index b09b093c25..39dbbf6269 100644 --- a/test/pc/e2e/peer_connection_e2e_smoke_test.cc +++ b/test/pc/e2e/peer_connection_e2e_smoke_test.cc @@ -46,74 +46,58 @@ class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test { using EchoEmulationConfig = PeerConnectionE2EQualityTestFixture::EchoEmulationConfig; - void RunTest(const std::string& test_case_name, - const RunParams& run_params, - rtc::FunctionView alice_configurer, - rtc::FunctionView bob_configurer) { - // Setup emulated network - std::unique_ptr network_emulation_manager = - CreateNetworkEmulationManager(); - - auto alice_network_behavior = - std::make_unique(BuiltInNetworkBehaviorConfig()); - SimulatedNetwork* alice_network_behavior_ptr = alice_network_behavior.get(); - EmulatedNetworkNode* alice_node = - network_emulation_manager->CreateEmulatedNode( - std::move(alice_network_behavior)); - EmulatedNetworkNode* bob_node = - network_emulation_manager->CreateEmulatedNode( - std::make_unique(BuiltInNetworkBehaviorConfig())); - auto* alice_endpoint = - network_emulation_manager->CreateEndpoint(EmulatedEndpointConfig()); - EmulatedEndpoint* bob_endpoint = - network_emulation_manager->CreateEndpoint(EmulatedEndpointConfig()); - network_emulation_manager->CreateRoute(alice_endpoint, {alice_node}, - bob_endpoint); - network_emulation_manager->CreateRoute(bob_endpoint, {bob_node}, - alice_endpoint); - - // Create analyzers. - std::unique_ptr video_quality_analyzer = - std::make_unique( - network_emulation_manager->time_controller()->GetClock()); - // This is only done for the sake of smoke testing. In general there should - // be no need to explicitly pull data from analyzers after the run. - auto* video_analyzer_ptr = - static_cast(video_quality_analyzer.get()); - - auto fixture = CreatePeerConnectionE2EQualityTestFixture( - test_case_name, *network_emulation_manager->time_controller(), + void SetUp() override { + network_emulation_ = CreateNetworkEmulationManager(); + auto video_quality_analyzer = std::make_unique( + network_emulation_->time_controller()->GetClock()); + video_quality_analyzer_ = video_quality_analyzer.get(); + fixture_ = CreatePeerConnectionE2EQualityTestFixture( + testing::UnitTest::GetInstance()->current_test_info()->name(), + *network_emulation_->time_controller(), /*audio_quality_analyzer=*/nullptr, std::move(video_quality_analyzer)); - fixture->ExecuteAt(TimeDelta::Seconds(1), - [alice_network_behavior_ptr](TimeDelta) { - BuiltInNetworkBehaviorConfig config; - config.loss_percent = 5; - alice_network_behavior_ptr->SetConfig(config); - }); + test::ScopedFieldTrials field_trials( + std::string(field_trial::GetFieldTrialString()) + + "WebRTC-UseStandardBytesStats/Enabled/"); + } + + std::pair + CreateNetwork() { + EmulatedNetworkNode* alice_node = network_emulation_->CreateEmulatedNode( + std::make_unique(BuiltInNetworkBehaviorConfig())); + EmulatedNetworkNode* bob_node = network_emulation_->CreateEmulatedNode( + std::make_unique(BuiltInNetworkBehaviorConfig())); + + EmulatedEndpoint* alice_endpoint = + network_emulation_->CreateEndpoint(EmulatedEndpointConfig()); + EmulatedEndpoint* bob_endpoint = + network_emulation_->CreateEndpoint(EmulatedEndpointConfig()); + + network_emulation_->CreateRoute(alice_endpoint, {alice_node}, bob_endpoint); + network_emulation_->CreateRoute(bob_endpoint, {bob_node}, alice_endpoint); - // Setup components. We need to provide rtc::NetworkManager compatible with - // emulated network layer. EmulatedNetworkManagerInterface* alice_network = - network_emulation_manager->CreateEmulatedNetworkManagerInterface( + network_emulation_->CreateEmulatedNetworkManagerInterface( {alice_endpoint}); EmulatedNetworkManagerInterface* bob_network = - network_emulation_manager->CreateEmulatedNetworkManagerInterface( + network_emulation_->CreateEmulatedNetworkManagerInterface( {bob_endpoint}); - fixture->AddPeer(alice_network->network_thread(), - alice_network->network_manager(), alice_configurer); - fixture->AddPeer(bob_network->network_thread(), - bob_network->network_manager(), bob_configurer); - fixture->AddQualityMetricsReporter( - std::make_unique(alice_network, - bob_network)); + return std::make_pair(alice_network, bob_network); + } - fixture->Run(run_params); + void AddPeer(EmulatedNetworkManagerInterface* network, + rtc::FunctionView configurer) { + fixture_->AddPeer(network->network_thread(), network->network_manager(), + configurer); + } - EXPECT_GE(fixture->GetRealTestDuration(), run_params.run_duration); - for (auto stream_key : video_analyzer_ptr->GetKnownVideoStreams()) { + void RunAndCheckEachVideoStreamReceivedFrames(const RunParams& run_params) { + fixture_->Run(run_params); + + EXPECT_GE(fixture_->GetRealTestDuration(), run_params.run_duration); + for (auto stream_key : video_quality_analyzer_->GetKnownVideoStreams()) { FrameCounters stream_conters = - video_analyzer_ptr->GetPerStreamCounters().at(stream_key); + video_quality_analyzer_->GetPerStreamCounters().at(stream_key); // On some devices the pipeline can be too slow, so we actually can't // force real constraints here. Lets just check, that at least 1 // frame passed whole pipeline. @@ -127,9 +111,18 @@ class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test { EXPECT_GE(stream_conters.rendered, 1) << stream_key.ToString(); } } -}; -} // namespace + NetworkEmulationManager* network_emulation() { + return network_emulation_.get(); + } + + PeerConnectionE2EQualityTestFixture* fixture() { return fixture_.get(); } + + private: + std::unique_ptr network_emulation_; + DefaultVideoQualityAnalyzer* video_quality_analyzer_; + std::unique_ptr fixture_; +}; // IOS debug builds can be quite slow, disabling to avoid issues with timeouts. #if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) @@ -138,46 +131,108 @@ class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test { #define MAYBE_Smoke Smoke #endif TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Smoke) { + std::pair + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + VideoConfig video(160, 120, 15); + video.stream_label = "alice-video"; + video.sync_group = "alice-media"; + alice->AddVideoConfig(std::move(video)); + + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + audio.sampling_frequency_in_hz = 48000; + audio.sync_group = "alice-media"; + alice->SetAudioConfig(std::move(audio)); + }); + AddPeer(network_links.second, [](PeerConfigurer* charlie) { + charlie->SetName("charlie"); + VideoConfig video(160, 120, 15); + video.stream_label = "charlie-video"; + video.temporal_layers_count = 2; + charlie->AddVideoConfig(std::move(video)); + + AudioConfig audio; + audio.stream_label = "charlie-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_bob_source", "wav"); + charlie->SetAudioConfig(std::move(audio)); + }); + fixture()->AddQualityMetricsReporter( + std::make_unique(network_links.first, + network_links.second)); RunParams run_params(TimeDelta::Seconds(2)); run_params.video_codecs = { VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}; run_params.use_flex_fec = true; run_params.use_ulp_fec = true; run_params.video_encoder_bitrate_multiplier = 1.1; - test::ScopedFieldTrials field_trials( - std::string(field_trial::GetFieldTrialString()) + - "WebRTC-UseStandardBytesStats/Enabled/"); - RunTest( - "smoke", run_params, - [](PeerConfigurer* alice) { - VideoConfig video(160, 120, 15); - video.stream_label = "alice-video"; - video.sync_group = "alice-media"; - alice->AddVideoConfig(std::move(video)); + RunAndCheckEachVideoStreamReceivedFrames(run_params); +} - AudioConfig audio; - audio.stream_label = "alice-audio"; - audio.mode = AudioConfig::Mode::kFile; - audio.input_file_name = - test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); - audio.sampling_frequency_in_hz = 48000; - audio.sync_group = "alice-media"; - alice->SetAudioConfig(std::move(audio)); - }, - [](PeerConfigurer* charlie) { - charlie->SetName("charlie"); - VideoConfig video(160, 120, 15); - video.stream_label = "charlie-video"; - video.temporal_layers_count = 2; - charlie->AddVideoConfig(std::move(video)); +// IOS debug builds can be quite slow, disabling to avoid issues with timeouts. +#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) +#define MAYBE_ChangeNetworkConditions DISABLED_ChangeNetworkConditions +#else +#define MAYBE_ChangeNetworkConditions ChangeNetworkConditions +#endif +TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_ChangeNetworkConditions) { + NetworkEmulationManager::SimulatedNetworkNode alice_node = + network_emulation() + ->NodeBuilder() + .config(BuiltInNetworkBehaviorConfig()) + .Build(); + NetworkEmulationManager::SimulatedNetworkNode bob_node = + network_emulation() + ->NodeBuilder() + .config(BuiltInNetworkBehaviorConfig()) + .Build(); - AudioConfig audio; - audio.stream_label = "charlie-audio"; - audio.mode = AudioConfig::Mode::kFile; - audio.input_file_name = - test::ResourcePath("pc_quality_smoke_test_bob_source", "wav"); - charlie->SetAudioConfig(std::move(audio)); - }); + EmulatedEndpoint* alice_endpoint = + network_emulation()->CreateEndpoint(EmulatedEndpointConfig()); + EmulatedEndpoint* bob_endpoint = + network_emulation()->CreateEndpoint(EmulatedEndpointConfig()); + + network_emulation()->CreateRoute(alice_endpoint, {alice_node.node}, + bob_endpoint); + network_emulation()->CreateRoute(bob_endpoint, {bob_node.node}, + alice_endpoint); + + EmulatedNetworkManagerInterface* alice_network = + network_emulation()->CreateEmulatedNetworkManagerInterface( + {alice_endpoint}); + EmulatedNetworkManagerInterface* bob_network = + network_emulation()->CreateEmulatedNetworkManagerInterface( + {bob_endpoint}); + + AddPeer(alice_network, [](PeerConfigurer* alice) { + VideoConfig video(160, 120, 15); + video.stream_label = "alice-video"; + video.sync_group = "alice-media"; + alice->AddVideoConfig(std::move(video)); + }); + AddPeer(bob_network, [](PeerConfigurer* bob) {}); + fixture()->AddQualityMetricsReporter( + std::make_unique(alice_network, + bob_network)); + + fixture()->ExecuteAt(TimeDelta::Seconds(1), [alice_node](TimeDelta) { + BuiltInNetworkBehaviorConfig config; + config.loss_percent = 5; + alice_node.simulation->SetConfig(config); + }); + + RunParams run_params(TimeDelta::Seconds(2)); + run_params.video_codecs = { + VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}; + run_params.use_flex_fec = true; + run_params.use_ulp_fec = true; + run_params.video_encoder_bitrate_multiplier = 1.1; + RunAndCheckEachVideoStreamReceivedFrames(run_params); } // IOS debug builds can be quite slow, disabling to avoid issues with timeouts. @@ -187,13 +242,10 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Smoke) { #define MAYBE_Screenshare Screenshare #endif TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Screenshare) { - RunParams run_params(TimeDelta::Seconds(2)); - test::ScopedFieldTrials field_trials( - std::string(field_trial::GetFieldTrialString()) + - "WebRTC-UseStandardBytesStats/Enabled/"); - RunTest( - "screenshare", run_params, - [](PeerConfigurer* alice) { + std::pair + network_links = CreateNetwork(); + AddPeer( + network_links.first, [](PeerConfigurer* alice) { VideoConfig screenshare(320, 180, 30); screenshare.stream_label = "alice-screenshare"; screenshare.content_hint = VideoTrackInterface::ContentHint::kText; @@ -205,8 +257,9 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Screenshare) { CreateScreenShareFrameGenerator(screenshare, screen_share_config); alice->AddVideoConfig(std::move(screenshare), std::move(screen_share_frame_generator)); - }, - [](PeerConfigurer* charlie) {}); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) {}); + RunAndCheckEachVideoStreamReceivedFrames(RunParams(TimeDelta::Seconds(2))); } // IOS debug builds can be quite slow, disabling to avoid issues with timeouts. @@ -216,27 +269,28 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Screenshare) { #define MAYBE_Echo Echo #endif TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Echo) { + std::pair + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + audio.sampling_frequency_in_hz = 48000; + alice->SetAudioConfig(std::move(audio)); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) { + AudioConfig audio; + audio.stream_label = "bob-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_bob_source", "wav"); + bob->SetAudioConfig(std::move(audio)); + }); RunParams run_params(TimeDelta::Seconds(2)); run_params.echo_emulation_config = EchoEmulationConfig(); - RunTest( - "smoke", run_params, - [](PeerConfigurer* alice) { - AudioConfig audio; - audio.stream_label = "alice-audio"; - audio.mode = AudioConfig::Mode::kFile; - audio.input_file_name = - test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); - audio.sampling_frequency_in_hz = 48000; - alice->SetAudioConfig(std::move(audio)); - }, - [](PeerConfigurer* bob) { - AudioConfig audio; - audio.stream_label = "bob-audio"; - audio.mode = AudioConfig::Mode::kFile; - audio.input_file_name = - test::ResourcePath("pc_quality_smoke_test_bob_source", "wav"); - bob->SetAudioConfig(std::move(audio)); - }); + RunAndCheckEachVideoStreamReceivedFrames(run_params); } // IOS debug builds can be quite slow, disabling to avoid issues with timeouts. @@ -246,24 +300,25 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Echo) { #define MAYBE_Simulcast Simulcast #endif TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Simulcast) { + std::pair + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + VideoConfig simulcast(1280, 720, 15); + simulcast.stream_label = "alice-simulcast"; + simulcast.simulcast_config = VideoSimulcastConfig(2, 0); + alice->AddVideoConfig(std::move(simulcast)); + + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + alice->SetAudioConfig(std::move(audio)); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) {}); RunParams run_params(TimeDelta::Seconds(2)); run_params.video_codecs = {VideoCodecConfig(cricket::kVp8CodecName)}; - RunTest( - "simulcast", run_params, - [](PeerConfigurer* alice) { - VideoConfig simulcast(1280, 720, 15); - simulcast.stream_label = "alice-simulcast"; - simulcast.simulcast_config = VideoSimulcastConfig(2, 0); - alice->AddVideoConfig(std::move(simulcast)); - - AudioConfig audio; - audio.stream_label = "alice-audio"; - audio.mode = AudioConfig::Mode::kFile; - audio.input_file_name = - test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); - alice->SetAudioConfig(std::move(audio)); - }, - [](PeerConfigurer* bob) {}); + RunAndCheckEachVideoStreamReceivedFrames(run_params); } // IOS debug builds can be quite slow, disabling to avoid issues with timeouts. @@ -273,26 +328,27 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Simulcast) { #define MAYBE_Svc Svc #endif TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Svc) { + std::pair + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + VideoConfig simulcast(1280, 720, 15); + simulcast.stream_label = "alice-svc"; + // Because we have network with packets loss we can analyze only the + // highest spatial layer in SVC mode. + simulcast.simulcast_config = VideoSimulcastConfig(2, 1); + alice->AddVideoConfig(std::move(simulcast)); + + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + alice->SetAudioConfig(std::move(audio)); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) {}); RunParams run_params(TimeDelta::Seconds(2)); run_params.video_codecs = {VideoCodecConfig(cricket::kVp9CodecName)}; - RunTest( - "simulcast", run_params, - [](PeerConfigurer* alice) { - VideoConfig simulcast(1280, 720, 15); - simulcast.stream_label = "alice-svc"; - // Because we have network with packets loss we can analyze only the - // highest spatial layer in SVC mode. - simulcast.simulcast_config = VideoSimulcastConfig(2, 1); - alice->AddVideoConfig(std::move(simulcast)); - - AudioConfig audio; - audio.stream_label = "alice-audio"; - audio.mode = AudioConfig::Mode::kFile; - audio.input_file_name = - test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); - alice->SetAudioConfig(std::move(audio)); - }, - [](PeerConfigurer* bob) {}); + RunAndCheckEachVideoStreamReceivedFrames(run_params); } // IOS debug builds can be quite slow, disabling to avoid issues with timeouts. @@ -302,33 +358,34 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Svc) { #define MAYBE_HighBitrate HighBitrate #endif TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_HighBitrate) { + std::pair + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + BitrateSettings bitrate_settings; + bitrate_settings.start_bitrate_bps = 3'000'000; + bitrate_settings.max_bitrate_bps = 3'000'000; + alice->SetBitrateSettings(bitrate_settings); + VideoConfig video(800, 600, 15); + video.stream_label = "alice-video"; + video.min_encode_bitrate_bps = 500'000; + video.max_encode_bitrate_bps = 3'000'000; + alice->AddVideoConfig(std::move(video)); + + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + audio.sampling_frequency_in_hz = 48000; + alice->SetAudioConfig(std::move(audio)); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) {}); RunParams run_params(TimeDelta::Seconds(2)); run_params.video_codecs = { VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}; - - RunTest( - "smoke", run_params, - [](PeerConfigurer* alice) { - BitrateSettings bitrate_settings; - bitrate_settings.start_bitrate_bps = 3'000'000; - bitrate_settings.max_bitrate_bps = 3'000'000; - alice->SetBitrateSettings(bitrate_settings); - VideoConfig video(800, 600, 15); - video.stream_label = "alice-video"; - video.min_encode_bitrate_bps = 500'000; - video.max_encode_bitrate_bps = 3'000'000; - alice->AddVideoConfig(std::move(video)); - - AudioConfig audio; - audio.stream_label = "alice-audio"; - audio.mode = AudioConfig::Mode::kFile; - audio.input_file_name = - test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); - audio.sampling_frequency_in_hz = 48000; - alice->SetAudioConfig(std::move(audio)); - }, - [](PeerConfigurer* bob) {}); + RunAndCheckEachVideoStreamReceivedFrames(run_params); } +} // namespace } // namespace webrtc_pc_e2e } // namespace webrtc