diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index 83a1b59090..0240957578 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -51,6 +51,8 @@ if (!build_with_chromium) { testonly = true deps = [ + ":analyzing_video_sink_test", + ":analyzing_video_sinks_helper_test", ":default_video_quality_analyzer_frames_comparator_test", ":default_video_quality_analyzer_metric_names_test", ":default_video_quality_analyzer_stream_state_test", @@ -59,6 +61,7 @@ if (!build_with_chromium) { ":names_collection_test", ":peer_connection_e2e_smoke_test", ":peer_connection_quality_test_metric_names_test", + ":peer_connection_quality_test_test", ":simulcast_dummy_buffer_helper_test", ":single_process_encoded_image_data_injector_unittest", ":stats_poller_test", @@ -222,6 +225,53 @@ if (!build_with_chromium) { absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } + rtc_library("analyzing_video_sinks_helper") { + testonly = true + sources = [ + "analyzer/video/analyzing_video_sinks_helper.cc", + "analyzer/video/analyzing_video_sinks_helper.h", + ] + deps = [ + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api/test/video:video_frame_writer", + "../../../rtc_base:macromagic", + "../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + + rtc_library("analyzing_video_sink") { + testonly = true + sources = [ + "analyzer/video/analyzing_video_sink.cc", + "analyzer/video/analyzing_video_sink.h", + ] + deps = [ + ":analyzing_video_sinks_helper", + ":simulcast_dummy_buffer_helper", + ":video_dumping", + "../..:fixed_fps_video_frame_writer_adapter", + "../..:test_renderer", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:video_quality_analyzer_api", + "../../../api/test/video:video_frame_writer", + "../../../api/video:video_frame", + "../../../rtc_base:checks", + "../../../rtc_base:logging", + "../../../rtc_base:macromagic", + "../../../rtc_base/synchronization:mutex", + "../../../system_wrappers", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory:memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + rtc_library("video_quality_analyzer_injection_helper") { visibility = [ "*" ] testonly = true @@ -230,6 +280,8 @@ if (!build_with_chromium) { "analyzer/video/video_quality_analyzer_injection_helper.h", ] deps = [ + ":analyzing_video_sink", + ":analyzing_video_sinks_helper", ":encoded_image_data_injector_api", ":quality_analyzing_video_decoder", ":quality_analyzing_video_encoder", @@ -506,6 +558,44 @@ if (!build_with_chromium) { ] } + rtc_library("analyzing_video_sinks_helper_test") { + testonly = true + sources = [ "analyzer/video/analyzing_video_sinks_helper_test.cc" ] + deps = [ + ":analyzing_video_sinks_helper", + "../..:test_support", + "../../../api:peer_connection_quality_test_fixture_api", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_library("analyzing_video_sink_test") { + testonly = true + sources = [ "analyzer/video/analyzing_video_sink_test.cc" ] + deps = [ + ":analyzing_video_sink", + ":example_video_quality_analyzer", + "../..:fileutils", + "../..:test_support", + "../..:video_test_support", + "../../../api:create_frame_generator", + "../../../api:frame_generator_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:scoped_refptr", + "../../../api/units:time_delta", + "../../../api/units:timestamp", + "../../../api/video:video_frame", + "../../../common_video", + "../../../rtc_base:timeutils", + "../../../system_wrappers", + "../../time_controller", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + peer_connection_e2e_smoke_test_resources = [ "../../../resources/pc_quality_smoke_test_alice_source.wav", "../../../resources/pc_quality_smoke_test_bob_source.wav", @@ -586,6 +676,23 @@ if (!build_with_chromium) { ] } + rtc_library("peer_connection_quality_test_test") { + testonly = true + sources = [ "peer_connection_quality_test_test.cc" ] + deps = [ + ":peerconnection_quality_test", + "../..:fileutils", + "../..:test_support", + "../..:video_test_support", + "../../../api:create_network_emulation_manager", + "../../../api:network_emulation_manager_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api/test/metrics:global_metrics_logger_and_exporter", + "../../../api/units:time_delta", + "../../../rtc_base:timeutils", + ] + } + rtc_library("stats_provider") { visibility = [ "*" ] testonly = true diff --git a/test/pc/e2e/analyzer/video/analyzing_video_sink.cc b/test/pc/e2e/analyzer/video/analyzing_video_sink.cc new file mode 100644 index 0000000000..97afda692d --- /dev/null +++ b/test/pc/e2e/analyzer/video/analyzing_video_sink.cc @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/analyzing_video_sink.h" + +#include +#include +#include + +#include "absl/memory/memory.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/test/video/video_frame_writer.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/synchronization/mutex.h" +#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h" +#include "test/pc/e2e/analyzer/video/video_dumping.h" +#include "test/testsupport/fixed_fps_video_frame_writer_adapter.h" +#include "test/video_renderer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +using VideoSubscription = ::webrtc::webrtc_pc_e2e:: + PeerConnectionE2EQualityTestFixture::VideoSubscription; +using VideoResolution = ::webrtc::webrtc_pc_e2e:: + PeerConnectionE2EQualityTestFixture::VideoResolution; +using VideoConfig = + ::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::VideoConfig; + +AnalyzingVideoSink::AnalyzingVideoSink(absl::string_view peer_name, + Clock* clock, + VideoQualityAnalyzerInterface& analyzer, + AnalyzingVideoSinksHelper& sinks_helper, + const VideoSubscription& subscription) + : peer_name_(peer_name), + clock_(clock), + analyzer_(&analyzer), + sinks_helper_(&sinks_helper), + subscription_(subscription) {} + +void AnalyzingVideoSink::UpdateSubscription( + const VideoSubscription& subscription) { + // For peers with changed resolutions we need to close current writers and + // open new ones. This is done by removing existing sinks, which will force + // creation of the new sinks when next frame will be received. + std::set writers_to_close; + { + MutexLock lock(&mutex_); + subscription_ = subscription; + for (auto it = stream_sinks_.cbegin(); it != stream_sinks_.cend();) { + absl::optional new_requested_resolution = + subscription_.GetResolutionForPeer(it->second.sender_peer_name); + if (!new_requested_resolution.has_value() || + (*new_requested_resolution != it->second.resolution)) { + writers_to_close.insert(it->second.video_frame_writer); + it = stream_sinks_.erase(it); + } else { + ++it; + } + } + } + sinks_helper_->CloseAndRemoveVideoWriters(writers_to_close); +} + +void AnalyzingVideoSink::OnFrame(const VideoFrame& frame) { + if (IsDummyFrame(frame)) { + // This is dummy frame, so we don't need to process it further. + return; + } + // Copy entire video frame including video buffer to ensure that analyzer + // won't hold any WebRTC internal buffers. + VideoFrame frame_copy = frame; + frame_copy.set_video_frame_buffer( + I420Buffer::Copy(*frame.video_frame_buffer()->ToI420())); + analyzer_->OnFrameRendered(peer_name_, frame_copy); + + if (frame.id() != VideoFrame::kNotSetId) { + std::string stream_label = analyzer_->GetStreamLabel(frame.id()); + SinksDescriptor* sinks_descriptor = PopulateSinks(stream_label); + if (sinks_descriptor == nullptr) { + return; + } + for (auto& sink : sinks_descriptor->sinks) { + sink->OnFrame(frame); + } + } +} + +AnalyzingVideoSink::SinksDescriptor* AnalyzingVideoSink::PopulateSinks( + absl::string_view stream_label) { + // Fast pass: sinks already exists. + MutexLock lock(&mutex_); + auto sinks_it = stream_sinks_.find(std::string(stream_label)); + if (sinks_it != stream_sinks_.end()) { + return &sinks_it->second; + } + + // Slow pass: we need to create and save sinks + absl::optional> peer_and_config = + sinks_helper_->GetPeerAndConfig(stream_label); + RTC_CHECK(peer_and_config.has_value()) + << "No video config for stream " << stream_label; + const std::string& sender_peer_name = peer_and_config->first; + const VideoConfig& config = peer_and_config->second; + + absl::optional resolution = + subscription_.GetResolutionForPeer(sender_peer_name); + if (!resolution.has_value()) { + RTC_LOG(LS_ERROR) << peer_name_ << " received stream " << stream_label + << " from " << sender_peer_name + << " for which they were not subscribed"; + resolution = config.GetResolution(); + } + + RTC_CHECK(resolution.has_value()); + + SinksDescriptor sinks_descriptor(sender_peer_name, *resolution); + if (config.output_dump_options.has_value()) { + std::unique_ptr writer = + config.output_dump_options->CreateOutputDumpVideoFrameWriter( + stream_label, peer_name_, *resolution); + if (config.output_dump_use_fixed_framerate) { + writer = std::make_unique( + resolution->fps(), clock_, std::move(writer)); + } + sinks_descriptor.sinks.push_back(std::make_unique( + writer.get(), config.output_dump_options->sampling_modulo())); + sinks_descriptor.video_frame_writer = + sinks_helper_->AddVideoWriter(std::move(writer)); + } + if (config.show_on_screen) { + sinks_descriptor.sinks.push_back( + absl::WrapUnique(test::VideoRenderer::Create( + (*config.stream_label + "-render").c_str(), resolution->width(), + resolution->height()))); + } + return &stream_sinks_.emplace(stream_label, std::move(sinks_descriptor)) + .first->second; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/analyzing_video_sink.h b/test/pc/e2e/analyzer/video/analyzing_video_sink.h new file mode 100644 index 0000000000..17b58fe2ec --- /dev/null +++ b/test/pc/e2e/analyzer/video/analyzing_video_sink.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINK_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINK_H_ + +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/test/video/video_frame_writer.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// A sink to inject video quality analyzer as a sink into WebRTC. +class AnalyzingVideoSink : public rtc::VideoSinkInterface { + public: + AnalyzingVideoSink( + absl::string_view peer_name, + Clock* clock, + VideoQualityAnalyzerInterface& analyzer, + AnalyzingVideoSinksHelper& sinks_helper, + const PeerConnectionE2EQualityTestFixture::VideoSubscription& + subscription); + + // Updates subscription used by this peer to render received video. + void UpdateSubscription( + const PeerConnectionE2EQualityTestFixture::VideoSubscription& + subscription); + + void OnFrame(const VideoFrame& frame) override; + + private: + struct SinksDescriptor { + SinksDescriptor( + absl::string_view sender_peer_name, + const PeerConnectionE2EQualityTestFixture::VideoResolution& resolution) + : sender_peer_name(sender_peer_name), resolution(resolution) {} + + // Required to be able to resolve resolutions on new subscription and + // understand if we need to recreate `video_frame_writer` and `sinks`. + std::string sender_peer_name; + // Resolution which was used to create `video_frame_writer` and `sinks`. + PeerConnectionE2EQualityTestFixture::VideoResolution resolution; + + // Is set if dumping of output video was requested; + test::VideoFrameWriter* video_frame_writer = nullptr; + std::vector>> sinks; + }; + + // Populates sink for specified stream and caches them in `stream_sinks_`. + SinksDescriptor* PopulateSinks(absl::string_view stream_label); + + const std::string peer_name_; + Clock* const clock_; + VideoQualityAnalyzerInterface* const analyzer_; + AnalyzingVideoSinksHelper* const sinks_helper_; + + Mutex mutex_; + PeerConnectionE2EQualityTestFixture::VideoSubscription subscription_ + RTC_GUARDED_BY(mutex_); + std::map stream_sinks_ RTC_GUARDED_BY(mutex_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINK_H_ diff --git a/test/pc/e2e/analyzer/video/analyzing_video_sink_test.cc b/test/pc/e2e/analyzer/video/analyzing_video_sink_test.cc new file mode 100644 index 0000000000..8cc7aee1b5 --- /dev/null +++ b/test/pc/e2e/analyzer/video/analyzing_video_sink_test.cc @@ -0,0 +1,419 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/analyzing_video_sink.h" + +#include + +#include +#include + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "api/test/create_frame_generator.h" +#include "api/test/frame_generator_interface.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/clock.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/pc/e2e/analyzer/video/example_video_quality_analyzer.h" +#include "test/testsupport/file_utils.h" +#include "test/testsupport/frame_reader.h" +#include "test/time_controller/simulated_time_controller.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::testing::ElementsAreArray; +using ::testing::Eq; +using ::testing::Test; + +using VideoConfig = + ::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::VideoConfig; +using VideoSubscription = ::webrtc::webrtc_pc_e2e:: + PeerConnectionE2EQualityTestFixture::VideoSubscription; +using VideoResolution = ::webrtc::webrtc_pc_e2e:: + PeerConnectionE2EQualityTestFixture::VideoResolution; +using VideoDumpOptions = ::webrtc::webrtc_pc_e2e:: + PeerConnectionE2EQualityTestFixture::VideoDumpOptions; + +// Remove files and directories in a directory non-recursively. +void CleanDir(absl::string_view dir, size_t expected_output_files_count) { + absl::optional> dir_content = + test::ReadDirectory(dir); + if (expected_output_files_count == 0) { + ASSERT_FALSE(dir_content.has_value()) << "Empty directory is expected"; + } else { + ASSERT_TRUE(dir_content.has_value()) << "Test directory is empty!"; + EXPECT_EQ(dir_content->size(), expected_output_files_count); + for (const auto& entry : *dir_content) { + if (test::DirExists(entry)) { + EXPECT_TRUE(test::RemoveDir(entry)) + << "Failed to remove sub directory: " << entry; + } else if (test::FileExists(entry)) { + EXPECT_TRUE(test::RemoveFile(entry)) + << "Failed to remove file: " << entry; + } else { + FAIL() << "Can't remove unknown file type: " << entry; + } + } + } + EXPECT_TRUE(test::RemoveDir(dir)) << "Failed to remove directory: " << dir; +} + +VideoFrame CreateFrame(test::FrameGeneratorInterface& frame_generator) { + test::FrameGeneratorInterface::VideoFrameData frame_data = + frame_generator.NextFrame(); + return VideoFrame::Builder() + .set_video_frame_buffer(frame_data.buffer) + .set_update_rect(frame_data.update_rect) + .build(); +} + +std::unique_ptr CreateFrameGenerator( + size_t width, + size_t height) { + return test::CreateSquareFrameGenerator(width, height, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); +} + +void AssertFrameIdsAre(const std::string& filename, + std::vector expected_ids) { + FILE* file = fopen(filename.c_str(), "r"); + ASSERT_TRUE(file != nullptr) << "Failed to open frame ids file: " << filename; + std::vector actual_ids; + char buffer[8]; + while (fgets(buffer, sizeof buffer, file) != nullptr) { + std::string current_id(buffer); + EXPECT_GE(current_id.size(), 2lu) + << "Found invalid frame id: [" << current_id << "]"; + if (current_id.size() < 2) { + continue; + } + // Trim "\n" at the end. + actual_ids.push_back(current_id.substr(0, current_id.size() - 1)); + } + fclose(file); + EXPECT_THAT(actual_ids, ElementsAreArray(expected_ids)); +} + +class AnalyzingVideoSinkTest : public Test { + protected: + ~AnalyzingVideoSinkTest() override = default; + + void SetUp() override { + // Create an empty temporary directory for this test. + test_directory_ = test::JoinFilename( + test::OutputPath(), + "TestDir_AnalyzingVideoSinkTest_" + + std::string( + testing::UnitTest::GetInstance()->current_test_info()->name())); + test::CreateDir(test_directory_); + } + + void TearDown() override { + CleanDir(test_directory_, expected_output_files_count_); + } + + void ExpectOutputFilesCount(size_t count) { + expected_output_files_count_ = count; + } + + std::string test_directory_; + size_t expected_output_files_count_ = 0; +}; + +TEST_F(AnalyzingVideoSinkTest, VideoFramesAreDumpedCorrectly) { + VideoSubscription subscription; + subscription.SubscribeToPeer( + "alice", VideoResolution(/*width=*/640, /*height=*/360, /*fps=*/30)); + VideoConfig video_config("alice_video", /*width=*/1280, /*height=*/720, + /*fps=*/30); + video_config.output_dump_options = VideoDumpOptions(test_directory_); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr frame_generator = + CreateFrameGenerator(/*width=*/1280, /*height=*/720); + VideoFrame frame = CreateFrame(*frame_generator); + frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame)); + + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription); + sink.OnFrame(frame); + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast(1))); + + test::Y4mFrameReaderImpl frame_reader( + test::JoinFilename(test_directory_, "alice_video_bob_640x360_30.y4m"), + /*width=*/640, + /*height=*/360); + ASSERT_TRUE(frame_reader.Init()); + EXPECT_THAT(frame_reader.NumberOfFrames(), Eq(1)); + rtc::scoped_refptr actual_frame = frame_reader.ReadFrame(); + rtc::scoped_refptr expected_frame = + frame.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Actual should be downscaled version of expected. + EXPECT_GT(ssim, 0.98); + EXPECT_GT(psnr, 38); + + ExpectOutputFilesCount(1); +} + +TEST_F(AnalyzingVideoSinkTest, + FallbackOnConfigResolutionIfNoSucscriptionProvided) { + VideoSubscription subscription; + VideoConfig video_config("alice_video", /*width=*/320, /*height=*/240, + /*fps=*/30); + video_config.output_dump_options = VideoDumpOptions(test_directory_); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr frame_generator = + CreateFrameGenerator(/*width=*/320, /*height=*/240); + VideoFrame frame = CreateFrame(*frame_generator); + frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame)); + + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription); + sink.OnFrame(frame); + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast(1))); + + test::Y4mFrameReaderImpl frame_reader( + test::JoinFilename(test_directory_, "alice_video_bob_320x240_30.y4m"), + /*width=*/320, + /*height=*/240); + ASSERT_TRUE(frame_reader.Init()); + EXPECT_THAT(frame_reader.NumberOfFrames(), Eq(1)); + rtc::scoped_refptr actual_frame = frame_reader.ReadFrame(); + rtc::scoped_refptr expected_frame = + frame.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Frames should be equal. + EXPECT_DOUBLE_EQ(ssim, 1.00); + EXPECT_DOUBLE_EQ(psnr, 48); + + ExpectOutputFilesCount(1); +} + +TEST_F(AnalyzingVideoSinkTest, + VideoFramesAreDumpedCorrectlyWhenSubscriptionChanged) { + VideoSubscription subscription_before; + subscription_before.SubscribeToPeer( + "alice", VideoResolution(/*width=*/1280, /*height=*/720, /*fps=*/30)); + VideoSubscription subscription_after; + subscription_after.SubscribeToPeer( + "alice", VideoResolution(/*width=*/640, /*height=*/360, /*fps=*/30)); + VideoConfig video_config("alice_video", /*width=*/1280, /*height=*/720, + /*fps=*/30); + video_config.output_dump_options = VideoDumpOptions(test_directory_); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr frame_generator = + CreateFrameGenerator(/*width=*/1280, /*height=*/720); + VideoFrame frame_before = CreateFrame(*frame_generator); + frame_before.set_id( + analyzer.OnFrameCaptured("alice", "alice_video", frame_before)); + VideoFrame frame_after = CreateFrame(*frame_generator); + frame_after.set_id( + analyzer.OnFrameCaptured("alice", "alice_video", frame_after)); + + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription_before); + sink.OnFrame(frame_before); + + sink.UpdateSubscription(subscription_after); + sink.OnFrame(frame_after); + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast(2))); + + { + test::Y4mFrameReaderImpl frame_reader( + test::JoinFilename(test_directory_, "alice_video_bob_1280x720_30.y4m"), + /*width=*/1280, + /*height=*/720); + ASSERT_TRUE(frame_reader.Init()); + EXPECT_THAT(frame_reader.NumberOfFrames(), Eq(1)); + rtc::scoped_refptr actual_frame = frame_reader.ReadFrame(); + rtc::scoped_refptr expected_frame = + frame_before.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Frames should be equal. + EXPECT_DOUBLE_EQ(ssim, 1.00); + EXPECT_DOUBLE_EQ(psnr, 48); + } + { + test::Y4mFrameReaderImpl frame_reader( + test::JoinFilename(test_directory_, "alice_video_bob_640x360_30.y4m"), + /*width=*/640, + /*height=*/360); + ASSERT_TRUE(frame_reader.Init()); + EXPECT_THAT(frame_reader.NumberOfFrames(), Eq(1)); + rtc::scoped_refptr actual_frame = frame_reader.ReadFrame(); + rtc::scoped_refptr expected_frame = + frame_after.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Actual should be downscaled version of expected. + EXPECT_GT(ssim, 0.98); + EXPECT_GT(psnr, 38); + } + + ExpectOutputFilesCount(2); +} + +TEST_F(AnalyzingVideoSinkTest, VideoFramesIdsAreDumpedWhenRequested) { + VideoSubscription subscription; + subscription.SubscribeToPeer( + "alice", VideoResolution(/*width=*/320, /*height=*/240, /*fps=*/30)); + VideoConfig video_config("alice_video", /*width=*/320, /*height=*/240, + /*fps=*/30); + video_config.output_dump_options = + VideoDumpOptions(test_directory_, /*export_frame_ids=*/true); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr frame_generator = + CreateFrameGenerator(/*width=*/320, /*height=*/240); + + std::vector expected_frame_ids; + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription); + for (int i = 0; i < 10; ++i) { + VideoFrame frame = CreateFrame(*frame_generator); + frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame)); + expected_frame_ids.push_back(std::to_string(frame.id())); + sink.OnFrame(frame); + } + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast(10))); + + AssertFrameIdsAre( + test::JoinFilename(test_directory_, + "alice_video_bob_320x240_30.frame_ids.txt"), + expected_frame_ids); + + ExpectOutputFilesCount(2); +} + +TEST_F(AnalyzingVideoSinkTest, + VideoFramesAndIdsAreDumpedWithFixedFpsWhenRequested) { + GlobalSimulatedTimeController simulated_time(Timestamp::Seconds(100000)); + + VideoSubscription subscription; + subscription.SubscribeToPeer( + "alice", VideoResolution(/*width=*/320, /*height=*/240, /*fps=*/10)); + VideoConfig video_config("alice_video", /*width=*/320, /*height=*/240, + /*fps=*/10); + video_config.output_dump_options = + VideoDumpOptions(test_directory_, /*export_frame_ids=*/true); + video_config.output_dump_use_fixed_framerate = true; + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr frame_generator = + CreateFrameGenerator(/*width=*/320, /*height=*/240); + + VideoFrame frame1 = CreateFrame(*frame_generator); + frame1.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame1)); + VideoFrame frame2 = CreateFrame(*frame_generator); + frame2.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame2)); + + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", simulated_time.GetClock(), analyzer, helper, + subscription); + sink.OnFrame(frame1); + // Advance almost 1 second, so the first frame has to be repeated 9 time + // more. + simulated_time.AdvanceTime(TimeDelta::Millis(990)); + sink.OnFrame(frame2); + simulated_time.AdvanceTime(TimeDelta::Millis(100)); + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast(2))); + + test::Y4mFrameReaderImpl frame_reader( + test::JoinFilename(test_directory_, "alice_video_bob_320x240_10.y4m"), + /*width=*/320, + /*height=*/240); + ASSERT_TRUE(frame_reader.Init()); + EXPECT_THAT(frame_reader.NumberOfFrames(), Eq(11)); + for (int i = 0; i < 10; ++i) { + rtc::scoped_refptr actual_frame = frame_reader.ReadFrame(); + rtc::scoped_refptr expected_frame = + frame1.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Frames should be equal. + EXPECT_DOUBLE_EQ(ssim, 1.00); + EXPECT_DOUBLE_EQ(psnr, 48); + } + rtc::scoped_refptr actual_frame = frame_reader.ReadFrame(); + rtc::scoped_refptr expected_frame = + frame2.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Frames should be equal. + EXPECT_DOUBLE_EQ(ssim, 1.00); + EXPECT_DOUBLE_EQ(psnr, 48); + + AssertFrameIdsAre( + test::JoinFilename(test_directory_, + "alice_video_bob_320x240_10.frame_ids.txt"), + {std::to_string(frame1.id()), std::to_string(frame1.id()), + std::to_string(frame1.id()), std::to_string(frame1.id()), + std::to_string(frame1.id()), std::to_string(frame1.id()), + std::to_string(frame1.id()), std::to_string(frame1.id()), + std::to_string(frame1.id()), std::to_string(frame1.id()), + std::to_string(frame2.id())}); + + ExpectOutputFilesCount(2); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.cc b/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.cc new file mode 100644 index 0000000000..64c37e576f --- /dev/null +++ b/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.cc @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h" + +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/test/video/video_frame_writer.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +using VideoConfig = + ::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::VideoConfig; + +void AnalyzingVideoSinksHelper::AddConfig(absl::string_view sender_peer_name, + VideoConfig config) { + MutexLock lock(&mutex_); + auto it = video_configs_.find(*config.stream_label); + if (it == video_configs_.end()) { + std::string stream_label = *config.stream_label; + video_configs_.emplace( + std::move(stream_label), + std::pair{std::string(sender_peer_name), std::move(config)}); + } else { + it->second = std::pair{std::string(sender_peer_name), std::move(config)}; + } +} + +absl::optional> +AnalyzingVideoSinksHelper::GetPeerAndConfig(absl::string_view stream_label) { + MutexLock lock(&mutex_); + auto it = video_configs_.find(std::string(stream_label)); + if (it == video_configs_.end()) { + return absl::nullopt; + } + return it->second; +} + +void AnalyzingVideoSinksHelper::RemoveConfig(absl::string_view stream_label) { + MutexLock lock(&mutex_); + video_configs_.erase(std::string(stream_label)); +} + +test::VideoFrameWriter* AnalyzingVideoSinksHelper::AddVideoWriter( + std::unique_ptr video_writer) { + MutexLock lock(&mutex_); + test::VideoFrameWriter* out = video_writer.get(); + video_writers_.push_back(std::move(video_writer)); + return out; +} + +void AnalyzingVideoSinksHelper::CloseAndRemoveVideoWriters( + std::set writers_to_close) { + MutexLock lock(&mutex_); + for (auto it = video_writers_.cbegin(); it != video_writers_.cend();) { + if (writers_to_close.find(it->get()) != writers_to_close.end()) { + (*it)->Close(); + it = video_writers_.erase(it); + } else { + ++it; + } + } +} + +void AnalyzingVideoSinksHelper::Clear() { + MutexLock lock(&mutex_); + video_configs_.clear(); + for (const auto& video_writer : video_writers_) { + video_writer->Close(); + } + video_writers_.clear(); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h b/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h new file mode 100644 index 0000000000..38e8ccc350 --- /dev/null +++ b/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINKS_HELPER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINKS_HELPER_H_ + +#include +#include +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/test/video/video_frame_writer.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Registry of known video configs and video writers. +// This class is thread safe. +class AnalyzingVideoSinksHelper { + public: + // Adds config in the registry. If config with such stream label was + // registered before, the new value will override the old one. + void AddConfig(absl::string_view sender_peer_name, + PeerConnectionE2EQualityTestFixture::VideoConfig config); + absl::optional< + std::pair> + GetPeerAndConfig(absl::string_view stream_label); + // Removes video config for specified stream label. If there are no know video + // config for such stream label - does nothing. + void RemoveConfig(absl::string_view stream_label); + + // Takes ownership of the provided video writer. All video writers owned by + // this class will be closed during `AnalyzingVideoSinksHelper` destruction + // and guaranteed to be alive either until explicitly removed by + // `CloseAndRemoveVideoWriters` or until `AnalyzingVideoSinksHelper` is + // destroyed. + // + // Returns pointer to the added writer. Ownership is maintained by + // `AnalyzingVideoSinksHelper`. + test::VideoFrameWriter* AddVideoWriter( + std::unique_ptr video_writer); + // For each provided `writers_to_close`, if it is known, will close and + // destroy it, otherwise does nothing with it. + void CloseAndRemoveVideoWriters( + std::set writers_to_close); + + // Removes all added configs and close and removes all added writers. + void Clear(); + + private: + Mutex mutex_; + std::map< + std::string, + std::pair> + video_configs_ RTC_GUARDED_BY(mutex_); + std::list> video_writers_ + RTC_GUARDED_BY(mutex_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINKS_HELPER_H_ diff --git a/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper_test.cc b/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper_test.cc new file mode 100644 index 0000000000..67f8f75bf4 --- /dev/null +++ b/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper_test.cc @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h" + +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::testing::Eq; + +using VideoConfig = + ::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::VideoConfig; + +// Asserts equality of the main fields of the video config. We don't compare +// the full config due to the lack of equality definition for a lot of subtypes. +void AssertConfigsAreEquals(const VideoConfig& actual, + const VideoConfig& expected) { + EXPECT_THAT(actual.stream_label, Eq(expected.stream_label)); + EXPECT_THAT(actual.width, Eq(expected.width)); + EXPECT_THAT(actual.height, Eq(expected.height)); + EXPECT_THAT(actual.fps, Eq(expected.fps)); +} + +TEST(AnalyzingVideoSinksHelperTest, ConfigsCanBeAdded) { + VideoConfig config("alice_video", /*width=*/1280, /*height=*/720, /*fps=*/30); + + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", config); + + absl::optional> registred_config = + helper.GetPeerAndConfig("alice_video"); + ASSERT_TRUE(registred_config.has_value()); + EXPECT_THAT(registred_config->first, Eq("alice")); + AssertConfigsAreEquals(registred_config->second, config); +} + +TEST(AnalyzingVideoSinksHelperTest, AddingForExistingLabelWillOverwriteValue) { + VideoConfig config_before("alice_video", /*width=*/1280, /*height=*/720, + /*fps=*/30); + VideoConfig config_after("alice_video", /*width=*/640, /*height=*/360, + /*fps=*/15); + + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", config_before); + + absl::optional> registred_config = + helper.GetPeerAndConfig("alice_video"); + ASSERT_TRUE(registred_config.has_value()); + EXPECT_THAT(registred_config->first, Eq("alice")); + AssertConfigsAreEquals(registred_config->second, config_before); + + helper.AddConfig("alice", config_after); + + registred_config = helper.GetPeerAndConfig("alice_video"); + ASSERT_TRUE(registred_config.has_value()); + EXPECT_THAT(registred_config->first, Eq("alice")); + AssertConfigsAreEquals(registred_config->second, config_after); +} + +TEST(AnalyzingVideoSinksHelperTest, ConfigsCanBeRemoved) { + VideoConfig config("alice_video", /*width=*/1280, /*height=*/720, /*fps=*/30); + + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", config); + + ASSERT_TRUE(helper.GetPeerAndConfig("alice_video").has_value()); + + helper.RemoveConfig("alice_video"); + ASSERT_FALSE(helper.GetPeerAndConfig("alice_video").has_value()); +} + +TEST(AnalyzingVideoSinksHelperTest, RemoveOfNonExistingConfigDontCrash) { + AnalyzingVideoSinksHelper helper; + helper.RemoveConfig("alice_video"); +} + +TEST(AnalyzingVideoSinksHelperTest, ClearRemovesAllConfigs) { + VideoConfig config1("alice_video", /*width=*/640, /*height=*/360, /*fps=*/30); + VideoConfig config2("bob_video", /*width=*/640, /*height=*/360, /*fps=*/30); + + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", config1); + helper.AddConfig("bob", config2); + + ASSERT_TRUE(helper.GetPeerAndConfig("alice_video").has_value()); + ASSERT_TRUE(helper.GetPeerAndConfig("bob_video").has_value()); + + helper.Clear(); + ASSERT_FALSE(helper.GetPeerAndConfig("alice_video").has_value()); + ASSERT_FALSE(helper.GetPeerAndConfig("bob_video").has_value()); +} + +struct TestVideoFrameWriterFactory { + int closed_writers_count = 0; + int deleted_writers_count = 0; + + std::unique_ptr CreateWriter() { + return std::make_unique(this); + } + + private: + class TestVideoFrameWriter : public test::VideoFrameWriter { + public: + explicit TestVideoFrameWriter(TestVideoFrameWriterFactory* factory) + : factory_(factory) {} + ~TestVideoFrameWriter() override { factory_->deleted_writers_count++; } + + bool WriteFrame(const VideoFrame& frame) override { return true; } + + void Close() override { factory_->closed_writers_count++; } + + private: + TestVideoFrameWriterFactory* factory_; + }; +}; + +TEST(AnalyzingVideoSinksHelperTest, RemovingWritersCloseAndDestroyAllOfThem) { + TestVideoFrameWriterFactory factory; + + AnalyzingVideoSinksHelper helper; + test::VideoFrameWriter* writer1 = + helper.AddVideoWriter(factory.CreateWriter()); + test::VideoFrameWriter* writer2 = + helper.AddVideoWriter(factory.CreateWriter()); + + helper.CloseAndRemoveVideoWriters({writer1, writer2}); + + EXPECT_THAT(factory.closed_writers_count, Eq(2)); + EXPECT_THAT(factory.deleted_writers_count, Eq(2)); +} + +TEST(AnalyzingVideoSinksHelperTest, ClearCloseAndDestroyAllWriters) { + TestVideoFrameWriterFactory factory; + + AnalyzingVideoSinksHelper helper; + helper.AddVideoWriter(factory.CreateWriter()); + helper.AddVideoWriter(factory.CreateWriter()); + + helper.Clear(); + + EXPECT_THAT(factory.closed_writers_count, Eq(2)); + EXPECT_THAT(factory.deleted_writers_count, Eq(2)); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/video_dumping.cc b/test/pc/e2e/analyzer/video/video_dumping.cc index 2e10e9008d..4fec0a8f9e 100644 --- a/test/pc/e2e/analyzer/video/video_dumping.cc +++ b/test/pc/e2e/analyzer/video/video_dumping.cc @@ -32,6 +32,7 @@ class VideoFrameIdsWriter final : public test::VideoFrameWriter { explicit VideoFrameIdsWriter(absl::string_view file_name) : file_name_(file_name) { output_file_ = fopen(file_name_.c_str(), "wb"); + RTC_LOG(LS_INFO) << "Writing VideoFrame IDs into " << file_name_; RTC_CHECK(output_file_ != nullptr) << "Failed to open file to dump frame ids for writing: " << file_name_; } @@ -50,6 +51,7 @@ class VideoFrameIdsWriter final : public test::VideoFrameWriter { void Close() override { if (output_file_ != nullptr) { + RTC_LOG(LS_INFO) << "Closing file for VideoFrame IDs: " << file_name_; fclose(output_file_); output_file_ = nullptr; } diff --git a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc index 5c6c9e3aa7..dfe9ff8491 100644 --- a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc +++ b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc @@ -24,6 +24,7 @@ #include "rtc_base/logging.h" #include "rtc_base/strings/string_builder.h" #include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/analyzing_video_sink.h" #include "test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h" #include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h" #include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h" @@ -37,6 +38,8 @@ namespace { using EmulatedSFUConfigMap = ::webrtc::webrtc_pc_e2e::QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap; +using VideoSubscription = ::webrtc::webrtc_pc_e2e:: + PeerConnectionE2EQualityTestFixture::VideoSubscription; class AnalyzingFramePreprocessor : public test::TestVideoCapturer::FramePreprocessor { @@ -128,6 +131,7 @@ VideoQualityAnalyzerInjectionHelper::CreateFramePreprocessor( test::VideoRenderer::Create((*config.stream_label + "-capture").c_str(), config.width, config.height))); } + sinks_helper_.AddConfig(peer_name, config); { MutexLock lock(&mutex_); known_video_configs_.insert({*config.stream_label, config}); @@ -140,7 +144,16 @@ VideoQualityAnalyzerInjectionHelper::CreateFramePreprocessor( std::unique_ptr> VideoQualityAnalyzerInjectionHelper::CreateVideoSink( absl::string_view peer_name) { - return std::make_unique(peer_name, this); + return std::make_unique(peer_name, this); +} + +std::unique_ptr +VideoQualityAnalyzerInjectionHelper::CreateVideoSink( + absl::string_view peer_name, + const PeerConnectionE2EQualityTestFixture::VideoSubscription& + subscription) { + return std::make_unique(peer_name, clock_, *analyzer_, + sinks_helper_, subscription); } void VideoQualityAnalyzerInjectionHelper::Start( @@ -181,6 +194,7 @@ void VideoQualityAnalyzerInjectionHelper::Stop() { video_writer->Close(); } video_writers_.clear(); + sinks_helper_.Clear(); } void VideoQualityAnalyzerInjectionHelper::OnFrame(absl::string_view peer_name, diff --git a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h index 3eee5a0566..5b800c6aa9 100644 --- a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h +++ b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h @@ -29,6 +29,8 @@ #include "api/video_codecs/video_encoder_factory.h" #include "rtc_base/synchronization/mutex.h" #include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/analyzing_video_sink.h" +#include "test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h" #include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" #include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h" #include "test/test_video_capturer.h" @@ -77,6 +79,10 @@ class VideoQualityAnalyzerInjectionHelper : public StatsObserverInterface { // into that file. std::unique_ptr> CreateVideoSink( absl::string_view peer_name); + std::unique_ptr CreateVideoSink( + absl::string_view peer_name, + const PeerConnectionE2EQualityTestFixture::VideoSubscription& + subscription); void Start(std::string test_case_name, rtc::ArrayView peer_names, @@ -101,12 +107,13 @@ class VideoQualityAnalyzerInjectionHelper : public StatsObserverInterface { void Stop(); private: - class AnalyzingVideoSink final : public rtc::VideoSinkInterface { + // Deprecated, to be removed when old API isn't used anymore. + class AnalyzingVideoSink2 final : public rtc::VideoSinkInterface { public: - explicit AnalyzingVideoSink(absl::string_view peer_name, - VideoQualityAnalyzerInjectionHelper* helper) + explicit AnalyzingVideoSink2(absl::string_view peer_name, + VideoQualityAnalyzerInjectionHelper* helper) : peer_name_(peer_name), helper_(helper) {} - ~AnalyzingVideoSink() override = default; + ~AnalyzingVideoSink2() override = default; void OnFrame(const VideoFrame& frame) override { helper_->OnFrame(peer_name_, frame); @@ -147,6 +154,7 @@ class VideoQualityAnalyzerInjectionHelper : public StatsObserverInterface { std::vector> video_writers_; + AnalyzingVideoSinksHelper sinks_helper_; Mutex mutex_; int peers_count_ RTC_GUARDED_BY(mutex_); // Map from stream label to the video config. diff --git a/test/pc/e2e/peer_connection_quality_test.cc b/test/pc/e2e/peer_connection_quality_test.cc index 6cf7449a14..b714bdd720 100644 --- a/test/pc/e2e/peer_connection_quality_test.cc +++ b/test/pc/e2e/peer_connection_quality_test.cc @@ -48,8 +48,8 @@ namespace { using ::webrtc::test::ImprovementDirection; using ::webrtc::test::Unit; -using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig; -using VideoCodecConfig = PeerConnectionE2EQualityTestFixture::VideoCodecConfig; +using VideoConfig = + ::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::VideoConfig; constexpr TimeDelta kDefaultTimeout = TimeDelta::Seconds(10); constexpr char kSignalThreadName[] = "signaling_thread"; @@ -262,11 +262,15 @@ void PeerConnectionE2EQualityTest::Run(RunParams run_params) { RemotePeerAudioConfig::Create(bob_configurer->params()->audio_config); absl::optional bob_remote_audio_config = RemotePeerAudioConfig::Create(alice_configurer->params()->audio_config); - // Copy Alice and Bob video configs and names to correctly pass them into - // lambdas. + // Copy Alice and Bob video configs, subscriptions and names to correctly pass + // them into lambdas. + VideoSubscription alice_subscription = + alice_configurer->configurable_params()->video_subscription; std::vector alice_video_configs = alice_configurer->configurable_params()->video_configs; std::string alice_name = alice_configurer->params()->name.value(); + VideoSubscription bob_subscription = + alice_configurer->configurable_params()->video_subscription; std::vector bob_video_configs = bob_configurer->configurable_params()->video_configs; std::string bob_name = bob_configurer->params()->name.value(); @@ -277,18 +281,20 @@ void PeerConnectionE2EQualityTest::Run(RunParams run_params) { alice_ = test_peer_factory.CreateTestPeer( std::move(alice_configurer), std::make_unique( - [this, bob_video_configs, alice_name]( + [this, alice_name, alice_subscription, bob_video_configs]( rtc::scoped_refptr transceiver) { - OnTrackCallback(alice_name, transceiver, bob_video_configs); + OnTrackCallback(alice_name, alice_subscription, transceiver, + bob_video_configs); }, [this]() { StartVideo(alice_video_sources_); }), alice_remote_audio_config, run_params.echo_emulation_config); bob_ = test_peer_factory.CreateTestPeer( std::move(bob_configurer), std::make_unique( - [this, alice_video_configs, - bob_name](rtc::scoped_refptr transceiver) { - OnTrackCallback(bob_name, transceiver, alice_video_configs); + [this, bob_name, bob_subscription, alice_video_configs]( + rtc::scoped_refptr transceiver) { + OnTrackCallback(bob_name, bob_subscription, transceiver, + alice_video_configs); }, [this]() { StartVideo(bob_video_sources_); }), bob_remote_audio_config, run_params.echo_emulation_config); @@ -448,6 +454,7 @@ std::string PeerConnectionE2EQualityTest::GetFieldTrials( void PeerConnectionE2EQualityTest::OnTrackCallback( absl::string_view peer_name, + VideoSubscription peer_subscription, rtc::scoped_refptr transceiver, std::vector remote_video_configs) { const rtc::scoped_refptr& track = diff --git a/test/pc/e2e/peer_connection_quality_test.h b/test/pc/e2e/peer_connection_quality_test.h index 532cec357f..ddada4b391 100644 --- a/test/pc/e2e/peer_connection_quality_test.h +++ b/test/pc/e2e/peer_connection_quality_test.h @@ -96,6 +96,7 @@ class PeerConnectionE2EQualityTest // enabled in Run(). std::string GetFieldTrials(const RunParams& run_params); void OnTrackCallback(absl::string_view peer_name, + VideoSubscription peer_subscription, rtc::scoped_refptr transceiver, std::vector remote_video_configs); // Have to be run on the signaling thread. diff --git a/test/pc/e2e/peer_connection_quality_test_test.cc b/test/pc/e2e/peer_connection_quality_test_test.cc new file mode 100644 index 0000000000..704fafd441 --- /dev/null +++ b/test/pc/e2e/peer_connection_quality_test_test.cc @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/peer_connection_quality_test.h" + +#include +#include +#include +#include + +#include "api/test/create_network_emulation_manager.h" +#include "api/test/metrics/global_metrics_logger_and_exporter.h" +#include "api/test/network_emulation_manager.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/units/time_delta.h" +#include "rtc_base/time_utils.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" +#include "test/testsupport/frame_reader.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::testing::Eq; +using ::testing::Test; + +using RunParams = + ::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::RunParams; +using VideoConfig = + ::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::VideoConfig; +using PeerConfigurer = ::webrtc::webrtc_pc_e2e:: + PeerConnectionE2EQualityTestFixture::PeerConfigurer; +using VideoDumpOptions = ::webrtc::webrtc_pc_e2e:: + PeerConnectionE2EQualityTestFixture::VideoDumpOptions; + +// Remove files and directories in a directory non-recursively. +void CleanDir(absl::string_view dir, size_t expected_output_files_count) { + absl::optional> dir_content = + test::ReadDirectory(dir); + if (expected_output_files_count == 0) { + ASSERT_FALSE(dir_content.has_value()) << "Empty directory is expected"; + } else { + ASSERT_TRUE(dir_content.has_value()) << "Test directory is empty!"; + EXPECT_EQ(dir_content->size(), expected_output_files_count); + for (const auto& entry : *dir_content) { + if (test::DirExists(entry)) { + EXPECT_TRUE(test::RemoveDir(entry)) + << "Failed to remove sub directory: " << entry; + } else if (test::FileExists(entry)) { + EXPECT_TRUE(test::RemoveFile(entry)) + << "Failed to remove file: " << entry; + } else { + FAIL() << "Can't remove unknown file type: " << entry; + } + } + } + EXPECT_TRUE(test::RemoveDir(dir)) << "Failed to remove directory: " << dir; +} + +class PeerConnectionE2EQualityTestTest : public Test { + protected: + ~PeerConnectionE2EQualityTestTest() override = default; + + void SetUp() override { + // Create an empty temporary directory for this test. + test_directory_ = test::JoinFilename( + test::OutputPath(), + "TestDir_PeerConnectionE2EQualityTestTest_" + + std::string( + testing::UnitTest::GetInstance()->current_test_info()->name())); + test::CreateDir(test_directory_); + } + + void TearDown() override { + CleanDir(test_directory_, expected_output_files_count_); + } + + void ExpectOutputFilesCount(size_t count) { + expected_output_files_count_ = count; + } + + std::string test_directory_; + size_t expected_output_files_count_ = 0; +}; + +TEST_F(PeerConnectionE2EQualityTestTest, OutputVideoIsDumpedWhenRequested) { + std::unique_ptr network_emulation = + CreateNetworkEmulationManager(TimeMode::kSimulated); + PeerConnectionE2EQualityTest fixture( + "test_case", *network_emulation->time_controller(), + /*audio_quality_analyzer=*/nullptr, /*video_quality_analyzer=*/nullptr, + test::GetGlobalMetricsLogger()); + + EmulatedEndpoint* alice_endpoint = + network_emulation->CreateEndpoint(EmulatedEndpointConfig()); + EmulatedEndpoint* bob_endpoint = + network_emulation->CreateEndpoint(EmulatedEndpointConfig()); + + network_emulation->CreateRoute( + alice_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()}, + bob_endpoint); + network_emulation->CreateRoute( + bob_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()}, + alice_endpoint); + + EmulatedNetworkManagerInterface* alice_network = + network_emulation->CreateEmulatedNetworkManagerInterface( + {alice_endpoint}); + EmulatedNetworkManagerInterface* bob_network = + network_emulation->CreateEmulatedNetworkManagerInterface({bob_endpoint}); + + fixture.AddPeer( + alice_network->network_dependencies(), [&](PeerConfigurer* peer) { + peer->SetName("alice"); + VideoConfig video("alice_video", 320, 180, 15); + video.output_dump_options = VideoDumpOptions(test_directory_); + peer->AddVideoConfig(std::move(video)); + }); + fixture.AddPeer(bob_network->network_dependencies(), + [&](PeerConfigurer* peer) { peer->SetName("bob"); }); + + fixture.Run(RunParams(TimeDelta::Seconds(2))); + + test::Y4mFrameReaderImpl frame_reader( + test::JoinFilename(test_directory_, "alice_video_bob_320x180_15.y4m"), + /*width=*/320, + /*height=*/180); + ASSERT_TRUE(frame_reader.Init()); + EXPECT_THAT(frame_reader.NumberOfFrames(), Eq(31)); // 2 seconds 15 fps + 1 + + ExpectOutputFilesCount(1); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc