diff --git a/api/test/peerconnection_quality_test_fixture.h b/api/test/peerconnection_quality_test_fixture.h index c75587243c..5ebe5d07b3 100644 --- a/api/test/peerconnection_quality_test_fixture.h +++ b/api/test/peerconnection_quality_test_fixture.h @@ -167,16 +167,18 @@ class PeerConnectionE2EQualityTestFixture { // Have to be unique among all specified configs for all peers in the call. // Will be auto generated if omitted. absl::optional stream_label; - // Only 1 from |generator|, |input_file_name| and |screen_share_config| can - // be specified. If none of them are specified, then |generator| will be set - // to VideoGeneratorType::kDefault. - // If specified generator of this type will be used to produce input video. + // Only 1 from |generator|, |input_file_name|, |screen_share_config| and + // |capturing_device_index| can be specified. If none of them are specified, + // then |generator| will be set to VideoGeneratorType::kDefault. If + // specified generator of this type will be used to produce input video. absl::optional generator; // If specified this file will be used as input. Input video will be played // in a circle. absl::optional input_file_name; // If specified screen share video stream will be created as input. absl::optional screen_share_config; + // If specified this capturing device will be used to get input video. + absl::optional capturing_device_index; // If presented video will be transfered in simulcast/SVC mode depending on // which encoder is used. // diff --git a/test/DEPS b/test/DEPS index 0f4fd2fcce..a5f4aae38b 100644 --- a/test/DEPS +++ b/test/DEPS @@ -65,5 +65,8 @@ specific_include_rules = { ".*sdp_changer\.(h|cc)": [ "+pc", "+p2p", + ], + ".*video_quality_analyzer_injection_helper\.(h|cc)": [ + "+pc", ] } diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index 3d34e7bc6c..fb3747cdfb 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -198,6 +198,7 @@ if (rtc_include_tests) { "../../../api/video:video_frame", "../../../api/video:video_rtp_headers", "../../../api/video_codecs:video_codecs_api", + "../../../pc:peerconnection", "../../../test:video_test_common", "../../../test:video_test_support", "//third_party/abseil-cpp/absl/memory", @@ -275,6 +276,8 @@ if (rtc_include_tests) { ":test_peer", ":video_quality_analyzer_injection_helper", "../..:field_trial", + "../..:platform_video_capturer", + "../..:video_test_common", "../../../api:audio_quality_analyzer_api", "../../../api:libjingle_peerconnection_api", "../../../api:media_stream_interface", diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index 2605461f77..fcef1fea3e 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -159,7 +159,8 @@ void DefaultVideoQualityAnalyzer::OnFramePreEncode( const webrtc::VideoFrame& frame) { rtc::CritScope crit(&lock_); auto it = frame_stats_.find(frame.id()); - RTC_DCHECK(it != frame_stats_.end()); + RTC_DCHECK(it != frame_stats_.end()) + << "Frame id=" << frame.id() << " not found"; frame_counters_.pre_encoded++; stream_frame_counters_[it->second.stream_label].pre_encoded++; it->second.pre_encode_time = Now(); diff --git a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc index b7d87b87fe..057e5b876c 100644 --- a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc +++ b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc @@ -11,6 +11,7 @@ #include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h" #include +#include #include "absl/memory/memory.h" #include "test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h" @@ -38,6 +39,65 @@ class VideoWriter final : public rtc::VideoSinkInterface { test::VideoFrameWriter* video_writer_; }; +class AnalyzingVideoSource : public rtc::VideoSourceInterface { + public: + AnalyzingVideoSource( + std::string stream_label, + VideoQualityAnalyzerInterface* analyzer, + std::unique_ptr test_capturer, + std::vector>> sinks) + : test_capturer_(std::move(test_capturer)), + analyzing_sink_(stream_label, analyzer, &broadcaster_), + sinks_(std::move(sinks)) { + for (auto& sink : sinks_) { + broadcaster_.AddOrUpdateSink(sink.get(), rtc::VideoSinkWants()); + } + } + + void AddOrUpdateSink(rtc::VideoSinkInterface* sink, + const rtc::VideoSinkWants& wants) override { + broadcaster_.AddOrUpdateSink(sink, wants); + test_capturer_->AddOrUpdateSink(&analyzing_sink_, broadcaster_.wants()); + } + + void RemoveSink(rtc::VideoSinkInterface* sink) override { + broadcaster_.RemoveSink(sink); + test_capturer_->AddOrUpdateSink(&analyzing_sink_, broadcaster_.wants()); + } + + private: + class AnalyzerCapturingVideoSink + : public rtc::VideoSinkInterface { + public: + AnalyzerCapturingVideoSink(std::string stream_label, + VideoQualityAnalyzerInterface* analyzer, + rtc::VideoBroadcaster* broadcaster) + : stream_label_(std::move(stream_label)), + analyzer_(analyzer), + broadcaster_(broadcaster) {} + ~AnalyzerCapturingVideoSink() override = default; + + void OnFrame(const VideoFrame& source_frame) override { + // Copy VideoFrame to be able to set id on it. + VideoFrame frame = source_frame; + uint16_t frame_id = analyzer_->OnFrameCaptured(stream_label_, frame); + frame.set_id(frame_id); + broadcaster_->OnFrame(frame); + } + + private: + const std::string stream_label_; + VideoQualityAnalyzerInterface* const analyzer_; + rtc::VideoBroadcaster* const broadcaster_; + }; + + rtc::VideoBroadcaster broadcaster_; + std::unique_ptr test_capturer_; + AnalyzerCapturingVideoSink analyzing_sink_; + const std::vector>> + sinks_; +}; + // Intercepts generated frames and passes them also to video quality analyzer // and to provided sinks. class AnalyzingFrameGenerator final : public test::FrameGenerator { @@ -142,11 +202,12 @@ VideoQualityAnalyzerInjectionHelper::WrapVideoDecoderFactory( analyzer_.get()); } -std::unique_ptr -VideoQualityAnalyzerInjectionHelper::WrapFrameGenerator( +rtc::scoped_refptr +VideoQualityAnalyzerInjectionHelper::CreateVideoTrackSource( const VideoConfig& config, - std::unique_ptr delegate, - test::VideoFrameWriter* writer) const { + std::unique_ptr capturer, + test::VideoFrameWriter* writer, + bool is_screencast) const { std::vector>> sinks; if (writer) { sinks.push_back(std::make_unique(writer)); @@ -156,9 +217,11 @@ VideoQualityAnalyzerInjectionHelper::WrapFrameGenerator( test::VideoRenderer::Create((*config.stream_label + "-capture").c_str(), config.width, config.height))); } - return std::make_unique( - std::move(*config.stream_label), std::move(delegate), analyzer_.get(), - std::move(sinks)); + return new rtc::RefCountedObject( + std::make_unique( + std::move(*config.stream_label), analyzer_.get(), std::move(capturer), + std::move(sinks)), + is_screencast); } std::unique_ptr> diff --git a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h index eb07a5df8d..b106a3c7f6 100644 --- a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h +++ b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h @@ -22,14 +22,43 @@ #include "api/video/video_sink_interface.h" #include "api/video_codecs/video_decoder_factory.h" #include "api/video_codecs/video_encoder_factory.h" +#include "pc/video_track_source.h" #include "test/frame_generator.h" #include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" #include "test/pc/e2e/analyzer/video/id_generator.h" +#include "test/test_video_capturer.h" #include "test/testsupport/video_frame_writer.h" namespace webrtc { namespace webrtc_pc_e2e { +class TestVideoCapturerVideoTrackSource : public VideoTrackSource { + public: + TestVideoCapturerVideoTrackSource( + std::unique_ptr> source, + bool is_screencast) + : VideoTrackSource(false /* remote */), + source_(std::move(source)), + is_screencast_(is_screencast) {} + + ~TestVideoCapturerVideoTrackSource() = default; + + void Start() { SetState(kLive); } + + void Stop() { SetState(kMuted); } + + bool is_screencast() const override { return is_screencast_; } + + protected: + rtc::VideoSourceInterface* source() override { + return source_.get(); + } + + private: + std::unique_ptr> source_; + const bool is_screencast_; +}; + // Provides factory methods for components, that will be used to inject // VideoQualityAnalyzerInterface into PeerConnection pipeline. class VideoQualityAnalyzerInjectionHelper : public StatsObserverInterface { @@ -54,16 +83,18 @@ class VideoQualityAnalyzerInjectionHelper : public StatsObserverInterface { std::unique_ptr WrapVideoDecoderFactory( std::unique_ptr delegate) const; - // Wraps frame generator, so video quality analyzer will gain access to the - // captured frames. If |writer| in not nullptr, will dump captured frames - // with provided writer. - std::unique_ptr WrapFrameGenerator( + // Creates video track source, that will allow video quality analyzer to get + // access to captured frames. If |writer| in not nullptr, will dump rendered + // frames with provided writer. + rtc::scoped_refptr CreateVideoTrackSource( const VideoConfig& config, - std::unique_ptr delegate, - test::VideoFrameWriter* writer) const; - // Creates sink, that will allow video quality analyzer to get access to the - // rendered frames. If |writer| in not nullptr, will dump rendered frames - // with provided writer. + std::unique_ptr capturer, + test::VideoFrameWriter* writer, + bool is_screencast) const; + + // Creates sink, that will allow video quality analyzer to get access to + // the rendered frames. If |writer| in not nullptr, will dump rendered + // frames with provided writer. std::unique_ptr> CreateVideoSink( const VideoConfig& config, test::VideoFrameWriter* writer) const; diff --git a/test/pc/e2e/peer_connection_quality_test.cc b/test/pc/e2e/peer_connection_quality_test.cc index 70bfec9b06..7913709f9c 100644 --- a/test/pc/e2e/peer_connection_quality_test.cc +++ b/test/pc/e2e/peer_connection_quality_test.cc @@ -30,9 +30,11 @@ #include "rtc_base/numerics/safe_conversions.h" #include "system_wrappers/include/cpu_info.h" #include "system_wrappers/include/field_trial.h" +#include "test/frame_generator_capturer.h" #include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" #include "test/pc/e2e/stats_poller.h" +#include "test/platform_video_capturer.h" #include "test/testsupport/file_utils.h" namespace webrtc { @@ -67,7 +69,9 @@ std::string VideoConfigSourcePresenceToString(const VideoConfig& video_config) { << "; video_config.input_file_name=" << video_config.input_file_name.has_value() << "; video_config.screen_share_config=" - << video_config.screen_share_config.has_value() << ";"; + << video_config.screen_share_config.has_value() + << "; video_config.capturing_device_index=" + << video_config.capturing_device_index.has_value() << ";"; return builder.str(); } @@ -418,7 +422,7 @@ void PeerConnectionE2EQualityTest::Run(RunParams run_params) { // Audio dumps. RTC_CHECK(!alice_); RTC_CHECK(!bob_); - // Ensuring that FrameGeneratorCapturerVideoTrackSource and VideoFrameWriter + // Ensuring that TestVideoCapturerVideoTrackSource and VideoFrameWriter // are destroyed on the right thread. RTC_CHECK(alice_video_sources_.empty()); RTC_CHECK(bob_video_sources_.empty()); @@ -434,7 +438,8 @@ void PeerConnectionE2EQualityTest::SetDefaultValuesForMissingParams( for (auto* p : params) { for (auto& video_config : p->video_configs) { if (!video_config.generator && !video_config.input_file_name && - !video_config.screen_share_config) { + !video_config.screen_share_config && + !video_config.capturing_device_index) { video_config.generator = VideoGeneratorType::kDefault; } if (!video_config.stream_label) { @@ -483,15 +488,16 @@ void PeerConnectionE2EQualityTest::ValidateParams(const RunParams& run_params, video_labels.insert(video_config.stream_label.value()).second; RTC_CHECK(inserted) << "Duplicate video_config.stream_label=" << video_config.stream_label.value(); - RTC_CHECK(video_config.generator || video_config.input_file_name || - video_config.screen_share_config) - << VideoConfigSourcePresenceToString(video_config); - RTC_CHECK(!(video_config.input_file_name && video_config.generator)) - << VideoConfigSourcePresenceToString(video_config); - RTC_CHECK( - !(video_config.input_file_name && video_config.screen_share_config)) - << VideoConfigSourcePresenceToString(video_config); - RTC_CHECK(!(video_config.screen_share_config && video_config.generator)) + int input_sources_count = 0; + if (video_config.generator) + ++input_sources_count; + if (video_config.input_file_name) + ++input_sources_count; + if (video_config.screen_share_config) + ++input_sources_count; + if (video_config.capturing_device_index) + ++input_sources_count; + RTC_CHECK_EQ(input_sources_count, 1) << VideoConfigSourcePresenceToString(video_config); if (video_config.screen_share_config) { @@ -678,38 +684,26 @@ void PeerConnectionE2EQualityTest::TearDownCallOnSignalingThread() { TearDownCall(); } -std::vector> +std::vector> PeerConnectionE2EQualityTest::MaybeAddMedia(TestPeer* peer) { MaybeAddAudio(peer); return MaybeAddVideo(peer); } -std::vector> +std::vector> PeerConnectionE2EQualityTest::MaybeAddVideo(TestPeer* peer) { // Params here valid because of pre-run validation. Params* params = peer->params(); - std::vector> out; + std::vector> out; for (auto video_config : params->video_configs) { - // Create video generator. - std::unique_ptr frame_generator = - CreateFrameGenerator(video_config); - - // Wrap it to inject video quality analyzer and enable dump of input video - // if required. + // Setup input video source into peer connection. + std::unique_ptr capturer = + CreateVideoCapturer(video_config); test::VideoFrameWriter* writer = MaybeCreateVideoWriter(video_config.input_dump_file_name, video_config); - frame_generator = - video_quality_analyzer_injection_helper_->WrapFrameGenerator( - video_config, std::move(frame_generator), writer); - - // Setup FrameGenerator into peer connection. - auto capturer = std::make_unique( - clock_, std::move(frame_generator), video_config.fps, - *task_queue_factory_); - capturer->Init(); - rtc::scoped_refptr source = - new rtc::RefCountedObject( - std::move(capturer), + rtc::scoped_refptr source = + video_quality_analyzer_injection_helper_->CreateVideoTrackSource( + video_config, std::move(capturer), writer, /*is_screencast=*/video_config.screen_share_config && video_config.screen_share_config->use_text_content_hint); out.push_back(source); @@ -738,9 +732,21 @@ PeerConnectionE2EQualityTest::MaybeAddVideo(TestPeer* peer) { return out; } -std::unique_ptr -PeerConnectionE2EQualityTest::CreateFrameGenerator( +std::unique_ptr +PeerConnectionE2EQualityTest::CreateVideoCapturer( const VideoConfig& video_config) { + if (video_config.capturing_device_index) { + std::unique_ptr capturer = + test::CreateVideoCapturer(video_config.width, video_config.height, + video_config.fps, + *video_config.capturing_device_index); + RTC_CHECK(capturer) + << "Failed to obtain input stream from capturing device #" + << *video_config.capturing_device_index; + return capturer; + } + + std::unique_ptr frame_generator = nullptr; if (video_config.generator) { absl::optional frame_generator_type = absl::nullopt; @@ -751,22 +757,27 @@ PeerConnectionE2EQualityTest::CreateFrameGenerator( } else if (video_config.generator == VideoGeneratorType::kI010) { frame_generator_type = test::FrameGenerator::OutputType::kI010; } - return test::FrameGenerator::CreateSquareGenerator( + frame_generator = test::FrameGenerator::CreateSquareGenerator( static_cast(video_config.width), static_cast(video_config.height), frame_generator_type, absl::nullopt); } if (video_config.input_file_name) { - return test::FrameGenerator::CreateFromYuvFile( + frame_generator = test::FrameGenerator::CreateFromYuvFile( std::vector(/*count=*/1, video_config.input_file_name.value()), video_config.width, video_config.height, /*frame_repeat_count=*/1); } if (video_config.screen_share_config) { - return CreateScreenShareFrameGenerator(video_config); + frame_generator = CreateScreenShareFrameGenerator(video_config); } - RTC_NOTREACHED() << "Unsupported video_config input source"; - return nullptr; + RTC_CHECK(frame_generator) << "Unsupported video_config input source"; + + auto capturer = std::make_unique( + clock_, std::move(frame_generator), video_config.fps, + *task_queue_factory_); + capturer->Init(); + return capturer; } std::unique_ptr @@ -955,8 +966,8 @@ void PeerConnectionE2EQualityTest::ExchangeIceCandidates( } void PeerConnectionE2EQualityTest::StartVideo( - const std::vector< - rtc::scoped_refptr>& sources) { + const std::vector>& + sources) { for (auto& source : sources) { if (source->state() != MediaSourceInterface::SourceState::kLive) { source->Start(); diff --git a/test/pc/e2e/peer_connection_quality_test.h b/test/pc/e2e/peer_connection_quality_test.h index 43a2f94647..f56283881c 100644 --- a/test/pc/e2e/peer_connection_quality_test.h +++ b/test/pc/e2e/peer_connection_quality_test.h @@ -20,7 +20,6 @@ #include "api/test/peerconnection_quality_test_fixture.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" -#include "pc/test/frame_generator_capturer_video_track_source.h" #include "rtc_base/task_queue_for_test.h" #include "rtc_base/task_utils/repeating_task.h" #include "rtc_base/thread.h" @@ -230,11 +229,11 @@ class PeerConnectionE2EQualityTest // Have to be run on the signaling thread. void SetupCallOnSignalingThread(const RunParams& run_params); void TearDownCallOnSignalingThread(); - std::vector> + std::vector> MaybeAddMedia(TestPeer* peer); - std::vector> + std::vector> MaybeAddVideo(TestPeer* peer); - std::unique_ptr CreateFrameGenerator( + std::unique_ptr CreateVideoCapturer( const VideoConfig& video_config); std::unique_ptr CreateScreenShareFrameGenerator( const VideoConfig& video_config); @@ -244,8 +243,8 @@ class PeerConnectionE2EQualityTest void ExchangeOfferAnswer(SignalingInterceptor* signaling_interceptor); void ExchangeIceCandidates(SignalingInterceptor* signaling_interceptor); void StartVideo( - const std::vector< - rtc::scoped_refptr>& sources); + const std::vector>& + sources); void TearDownCall(); test::VideoFrameWriter* MaybeCreateVideoWriter( absl::optional file_name, @@ -270,9 +269,9 @@ class PeerConnectionE2EQualityTest std::vector> quality_metrics_reporters_; - std::vector> + std::vector> alice_video_sources_; - std::vector> + std::vector> bob_video_sources_; std::vector> video_writers_; std::vector>>