diff --git a/api/test/peerconnection_quality_test_fixture.h b/api/test/peerconnection_quality_test_fixture.h index c75587243c..6ef4dfa38b 100644 --- a/api/test/peerconnection_quality_test_fixture.h +++ b/api/test/peerconnection_quality_test_fixture.h @@ -167,16 +167,22 @@ class PeerConnectionE2EQualityTestFixture { // Have to be unique among all specified configs for all peers in the call. // Will be auto generated if omitted. absl::optional stream_label; - // Only 1 from |generator|, |input_file_name| and |screen_share_config| can - // be specified. If none of them are specified, then |generator| will be set - // to VideoGeneratorType::kDefault. - // If specified generator of this type will be used to produce input video. + // Only 1 from |generator|, |input_file_name|, |screen_share_config| and + // |capturing_device_index| can be specified. If none of them are specified, + // then |generator| will be set to VideoGeneratorType::kDefault. If + // specified generator of this type will be used to produce input video. absl::optional generator; // If specified this file will be used as input. Input video will be played // in a circle. absl::optional input_file_name; // If specified screen share video stream will be created as input. absl::optional screen_share_config; + // If specified this capturing device will be used to get input video. The + // |capturing_device_index| is the index of required capturing device in OS + // provided list of video devices. On Linux and Windows the list will be + // obtained via webrtc::VideoCaptureModule::DeviceInfo, on Mac OS via + // [RTCCameraVideoCapturer captureDevices]. + absl::optional capturing_device_index; // If presented video will be transfered in simulcast/SVC mode depending on // which encoder is used. // diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index c6a54096af..5ab0f68003 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -275,6 +275,8 @@ if (rtc_include_tests) { ":test_peer", ":video_quality_analyzer_injection_helper", "../..:field_trial", + "../..:platform_video_capturer", + "../..:video_test_common", "../../../api:audio_quality_analyzer_api", "../../../api:libjingle_peerconnection_api", "../../../api:media_stream_interface", diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index 2605461f77..fcef1fea3e 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -159,7 +159,8 @@ void DefaultVideoQualityAnalyzer::OnFramePreEncode( const webrtc::VideoFrame& frame) { rtc::CritScope crit(&lock_); auto it = frame_stats_.find(frame.id()); - RTC_DCHECK(it != frame_stats_.end()); + RTC_DCHECK(it != frame_stats_.end()) + << "Frame id=" << frame.id() << " not found"; frame_counters_.pre_encoded++; stream_frame_counters_[it->second.stream_label].pre_encoded++; it->second.pre_encode_time = Now(); diff --git a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc index b7d87b87fe..436418b291 100644 --- a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc +++ b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc @@ -11,6 +11,7 @@ #include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h" #include +#include #include "absl/memory/memory.h" #include "test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h" @@ -38,6 +39,37 @@ class VideoWriter final : public rtc::VideoSinkInterface { test::VideoFrameWriter* video_writer_; }; +class AnalyzingFramePreprocessor + : public test::TestVideoCapturer::FramePreprocessor { + public: + AnalyzingFramePreprocessor( + std::string stream_label, + VideoQualityAnalyzerInterface* analyzer, + std::vector>> sinks) + : stream_label_(std::move(stream_label)), + analyzer_(analyzer), + sinks_(std::move(sinks)) {} + ~AnalyzingFramePreprocessor() override = default; + + VideoFrame Preprocess(const VideoFrame& source_frame) override { + // Copy VideoFrame to be able to set id on it. + VideoFrame frame = source_frame; + uint16_t frame_id = analyzer_->OnFrameCaptured(stream_label_, frame); + frame.set_id(frame_id); + + for (auto& sink : sinks_) { + sink->OnFrame(frame); + } + return frame; + } + + private: + const std::string stream_label_; + VideoQualityAnalyzerInterface* const analyzer_; + const std::vector>> + sinks_; +}; + // Intercepts generated frames and passes them also to video quality analyzer // and to provided sinks. class AnalyzingFrameGenerator final : public test::FrameGenerator { @@ -142,10 +174,9 @@ VideoQualityAnalyzerInjectionHelper::WrapVideoDecoderFactory( analyzer_.get()); } -std::unique_ptr -VideoQualityAnalyzerInjectionHelper::WrapFrameGenerator( +std::unique_ptr +VideoQualityAnalyzerInjectionHelper::CreateFramePreprocessor( const VideoConfig& config, - std::unique_ptr delegate, test::VideoFrameWriter* writer) const { std::vector>> sinks; if (writer) { @@ -156,9 +187,8 @@ VideoQualityAnalyzerInjectionHelper::WrapFrameGenerator( test::VideoRenderer::Create((*config.stream_label + "-capture").c_str(), config.width, config.height))); } - return std::make_unique( - std::move(*config.stream_label), std::move(delegate), analyzer_.get(), - std::move(sinks)); + return std::make_unique( + std::move(*config.stream_label), analyzer_.get(), std::move(sinks)); } std::unique_ptr> diff --git a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h index eb07a5df8d..4918768ea0 100644 --- a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h +++ b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h @@ -25,6 +25,7 @@ #include "test/frame_generator.h" #include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" #include "test/pc/e2e/analyzer/video/id_generator.h" +#include "test/test_video_capturer.h" #include "test/testsupport/video_frame_writer.h" namespace webrtc { @@ -54,16 +55,15 @@ class VideoQualityAnalyzerInjectionHelper : public StatsObserverInterface { std::unique_ptr WrapVideoDecoderFactory( std::unique_ptr delegate) const; - // Wraps frame generator, so video quality analyzer will gain access to the - // captured frames. If |writer| in not nullptr, will dump captured frames - // with provided writer. - std::unique_ptr WrapFrameGenerator( - const VideoConfig& config, - std::unique_ptr delegate, - test::VideoFrameWriter* writer) const; - // Creates sink, that will allow video quality analyzer to get access to the - // rendered frames. If |writer| in not nullptr, will dump rendered frames - // with provided writer. + // Creates VideoFrame preprocessor, that will allow video quality analyzer to + // get access to the captured frames. If |writer| in not nullptr, will dump + // captured frames with provided writer. + std::unique_ptr + CreateFramePreprocessor(const VideoConfig& config, + test::VideoFrameWriter* writer) const; + // Creates sink, that will allow video quality analyzer to get access to + // the rendered frames. If |writer| in not nullptr, will dump rendered + // frames with provided writer. std::unique_ptr> CreateVideoSink( const VideoConfig& config, test::VideoFrameWriter* writer) const; diff --git a/test/pc/e2e/peer_connection_quality_test.cc b/test/pc/e2e/peer_connection_quality_test.cc index eecb5aeb82..aa656c878e 100644 --- a/test/pc/e2e/peer_connection_quality_test.cc +++ b/test/pc/e2e/peer_connection_quality_test.cc @@ -30,9 +30,11 @@ #include "rtc_base/numerics/safe_conversions.h" #include "system_wrappers/include/cpu_info.h" #include "system_wrappers/include/field_trial.h" +#include "test/frame_generator_capturer.h" #include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" #include "test/pc/e2e/stats_poller.h" +#include "test/platform_video_capturer.h" #include "test/testsupport/file_utils.h" namespace webrtc { @@ -67,7 +69,9 @@ std::string VideoConfigSourcePresenceToString(const VideoConfig& video_config) { << "; video_config.input_file_name=" << video_config.input_file_name.has_value() << "; video_config.screen_share_config=" - << video_config.screen_share_config.has_value() << ";"; + << video_config.screen_share_config.has_value() + << "; video_config.capturing_device_index=" + << video_config.capturing_device_index.has_value() << ";"; return builder.str(); } @@ -420,7 +424,7 @@ void PeerConnectionE2EQualityTest::Run(RunParams run_params) { // Audio dumps. RTC_CHECK(!alice_); RTC_CHECK(!bob_); - // Ensuring that FrameGeneratorCapturerVideoTrackSource and VideoFrameWriter + // Ensuring that TestVideoCapturerVideoTrackSource and VideoFrameWriter // are destroyed on the right thread. RTC_CHECK(alice_video_sources_.empty()); RTC_CHECK(bob_video_sources_.empty()); @@ -436,7 +440,8 @@ void PeerConnectionE2EQualityTest::SetDefaultValuesForMissingParams( for (auto* p : params) { for (auto& video_config : p->video_configs) { if (!video_config.generator && !video_config.input_file_name && - !video_config.screen_share_config) { + !video_config.screen_share_config && + !video_config.capturing_device_index) { video_config.generator = VideoGeneratorType::kDefault; } if (!video_config.stream_label) { @@ -485,15 +490,16 @@ void PeerConnectionE2EQualityTest::ValidateParams(const RunParams& run_params, video_labels.insert(video_config.stream_label.value()).second; RTC_CHECK(inserted) << "Duplicate video_config.stream_label=" << video_config.stream_label.value(); - RTC_CHECK(video_config.generator || video_config.input_file_name || - video_config.screen_share_config) - << VideoConfigSourcePresenceToString(video_config); - RTC_CHECK(!(video_config.input_file_name && video_config.generator)) - << VideoConfigSourcePresenceToString(video_config); - RTC_CHECK( - !(video_config.input_file_name && video_config.screen_share_config)) - << VideoConfigSourcePresenceToString(video_config); - RTC_CHECK(!(video_config.screen_share_config && video_config.generator)) + int input_sources_count = 0; + if (video_config.generator) + ++input_sources_count; + if (video_config.input_file_name) + ++input_sources_count; + if (video_config.screen_share_config) + ++input_sources_count; + if (video_config.capturing_device_index) + ++input_sources_count; + RTC_CHECK_EQ(input_sources_count, 1) << VideoConfigSourcePresenceToString(video_config); if (video_config.screen_share_config) { @@ -680,37 +686,27 @@ void PeerConnectionE2EQualityTest::TearDownCallOnSignalingThread() { TearDownCall(); } -std::vector> +std::vector> PeerConnectionE2EQualityTest::MaybeAddMedia(TestPeer* peer) { MaybeAddAudio(peer); return MaybeAddVideo(peer); } -std::vector> +std::vector> PeerConnectionE2EQualityTest::MaybeAddVideo(TestPeer* peer) { // Params here valid because of pre-run validation. Params* params = peer->params(); - std::vector> out; + std::vector> out; for (auto video_config : params->video_configs) { - // Create video generator. - std::unique_ptr frame_generator = - CreateFrameGenerator(video_config); - - // Wrap it to inject video quality analyzer and enable dump of input video - // if required. + // Setup input video source into peer connection. test::VideoFrameWriter* writer = MaybeCreateVideoWriter(video_config.input_dump_file_name, video_config); - frame_generator = - video_quality_analyzer_injection_helper_->WrapFrameGenerator( - video_config, std::move(frame_generator), writer); - - // Setup FrameGenerator into peer connection. - auto capturer = std::make_unique( - clock_, std::move(frame_generator), video_config.fps, - *task_queue_factory_); - capturer->Init(); - rtc::scoped_refptr source = - new rtc::RefCountedObject( + std::unique_ptr capturer = CreateVideoCapturer( + video_config, + video_quality_analyzer_injection_helper_->CreateFramePreprocessor( + video_config, writer)); + rtc::scoped_refptr source = + new rtc::RefCountedObject( std::move(capturer), /*is_screencast=*/video_config.screen_share_config && video_config.screen_share_config->use_text_content_hint); @@ -740,9 +736,24 @@ PeerConnectionE2EQualityTest::MaybeAddVideo(TestPeer* peer) { return out; } -std::unique_ptr -PeerConnectionE2EQualityTest::CreateFrameGenerator( - const VideoConfig& video_config) { +std::unique_ptr +PeerConnectionE2EQualityTest::CreateVideoCapturer( + const VideoConfig& video_config, + std::unique_ptr + frame_preprocessor) { + if (video_config.capturing_device_index) { + std::unique_ptr capturer = + test::CreateVideoCapturer(video_config.width, video_config.height, + video_config.fps, + *video_config.capturing_device_index); + capturer->SetFramePreprocessor(std::move(frame_preprocessor)); + RTC_CHECK(capturer) + << "Failed to obtain input stream from capturing device #" + << *video_config.capturing_device_index; + return capturer; + } + + std::unique_ptr frame_generator = nullptr; if (video_config.generator) { absl::optional frame_generator_type = absl::nullopt; @@ -753,22 +764,28 @@ PeerConnectionE2EQualityTest::CreateFrameGenerator( } else if (video_config.generator == VideoGeneratorType::kI010) { frame_generator_type = test::FrameGenerator::OutputType::kI010; } - return test::FrameGenerator::CreateSquareGenerator( + frame_generator = test::FrameGenerator::CreateSquareGenerator( static_cast(video_config.width), static_cast(video_config.height), frame_generator_type, absl::nullopt); } if (video_config.input_file_name) { - return test::FrameGenerator::CreateFromYuvFile( + frame_generator = test::FrameGenerator::CreateFromYuvFile( std::vector(/*count=*/1, video_config.input_file_name.value()), video_config.width, video_config.height, /*frame_repeat_count=*/1); } if (video_config.screen_share_config) { - return CreateScreenShareFrameGenerator(video_config); + frame_generator = CreateScreenShareFrameGenerator(video_config); } - RTC_NOTREACHED() << "Unsupported video_config input source"; - return nullptr; + RTC_CHECK(frame_generator) << "Unsupported video_config input source"; + + auto capturer = std::make_unique( + clock_, std::move(frame_generator), video_config.fps, + *task_queue_factory_); + capturer->SetFramePreprocessor(std::move(frame_preprocessor)); + capturer->Init(); + return capturer; } std::unique_ptr @@ -957,8 +974,8 @@ void PeerConnectionE2EQualityTest::ExchangeIceCandidates( } void PeerConnectionE2EQualityTest::StartVideo( - const std::vector< - rtc::scoped_refptr>& sources) { + const std::vector>& + sources) { for (auto& source : sources) { if (source->state() != MediaSourceInterface::SourceState::kLive) { source->Start(); diff --git a/test/pc/e2e/peer_connection_quality_test.h b/test/pc/e2e/peer_connection_quality_test.h index 43a2f94647..dea571394c 100644 --- a/test/pc/e2e/peer_connection_quality_test.h +++ b/test/pc/e2e/peer_connection_quality_test.h @@ -20,7 +20,7 @@ #include "api/test/peerconnection_quality_test_fixture.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" -#include "pc/test/frame_generator_capturer_video_track_source.h" +#include "pc/video_track_source.h" #include "rtc_base/task_queue_for_test.h" #include "rtc_base/task_utils/repeating_task.h" #include "rtc_base/thread.h" @@ -159,6 +159,33 @@ class PeerConfigurerImpl final std::unique_ptr params_; }; +class TestVideoCapturerVideoTrackSource : public VideoTrackSource { + public: + TestVideoCapturerVideoTrackSource( + std::unique_ptr video_capturer, + bool is_screencast) + : VideoTrackSource(/*remote=*/false), + video_capturer_(std::move(video_capturer)), + is_screencast_(is_screencast) {} + + ~TestVideoCapturerVideoTrackSource() = default; + + void Start() { SetState(kLive); } + + void Stop() { SetState(kMuted); } + + bool is_screencast() const override { return is_screencast_; } + + protected: + rtc::VideoSourceInterface* source() override { + return video_capturer_.get(); + } + + private: + std::unique_ptr video_capturer_; + const bool is_screencast_; +}; + class PeerConnectionE2EQualityTest : public PeerConnectionE2EQualityTestFixture { public: @@ -230,12 +257,14 @@ class PeerConnectionE2EQualityTest // Have to be run on the signaling thread. void SetupCallOnSignalingThread(const RunParams& run_params); void TearDownCallOnSignalingThread(); - std::vector> + std::vector> MaybeAddMedia(TestPeer* peer); - std::vector> + std::vector> MaybeAddVideo(TestPeer* peer); - std::unique_ptr CreateFrameGenerator( - const VideoConfig& video_config); + std::unique_ptr CreateVideoCapturer( + const VideoConfig& video_config, + std::unique_ptr + frame_preprocessor); std::unique_ptr CreateScreenShareFrameGenerator( const VideoConfig& video_config); void MaybeAddAudio(TestPeer* peer); @@ -244,8 +273,8 @@ class PeerConnectionE2EQualityTest void ExchangeOfferAnswer(SignalingInterceptor* signaling_interceptor); void ExchangeIceCandidates(SignalingInterceptor* signaling_interceptor); void StartVideo( - const std::vector< - rtc::scoped_refptr>& sources); + const std::vector>& + sources); void TearDownCall(); test::VideoFrameWriter* MaybeCreateVideoWriter( absl::optional file_name, @@ -270,9 +299,9 @@ class PeerConnectionE2EQualityTest std::vector> quality_metrics_reporters_; - std::vector> + std::vector> alice_video_sources_; - std::vector> + std::vector> bob_video_sources_; std::vector> video_writers_; std::vector>> diff --git a/test/test_video_capturer.cc b/test/test_video_capturer.cc index fa43f9f7f0..a894cec99f 100644 --- a/test/test_video_capturer.cc +++ b/test/test_video_capturer.cc @@ -19,15 +19,16 @@ namespace webrtc { namespace test { -TestVideoCapturer::TestVideoCapturer() = default; TestVideoCapturer::~TestVideoCapturer() = default; -void TestVideoCapturer::OnFrame(const VideoFrame& frame) { +void TestVideoCapturer::OnFrame(const VideoFrame& original_frame) { int cropped_width = 0; int cropped_height = 0; int out_width = 0; int out_height = 0; + VideoFrame frame = MaybePreprocess(original_frame); + if (!video_adapter_.AdaptFrameResolution( frame.width(), frame.height(), frame.timestamp_us() * 1000, &cropped_width, &cropped_height, &out_width, &out_height)) { @@ -75,5 +76,14 @@ void TestVideoCapturer::UpdateVideoAdapter() { wants.target_pixel_count, wants.max_pixel_count, wants.max_framerate_fps); } +VideoFrame TestVideoCapturer::MaybePreprocess(const VideoFrame& frame) { + rtc::CritScope crit(&lock_); + if (preprocessor_ != nullptr) { + return preprocessor_->Preprocess(frame); + } else { + return frame; + } +} + } // namespace test } // namespace webrtc diff --git a/test/test_video_capturer.h b/test/test_video_capturer.h index 0f1886bb44..114767a43e 100644 --- a/test/test_video_capturer.h +++ b/test/test_video_capturer.h @@ -18,18 +18,29 @@ #include "api/video/video_source_interface.h" #include "media/base/video_adapter.h" #include "media/base/video_broadcaster.h" +#include "rtc_base/critical_section.h" namespace webrtc { namespace test { class TestVideoCapturer : public rtc::VideoSourceInterface { public: - TestVideoCapturer(); + class FramePreprocessor { + public: + virtual ~FramePreprocessor() = default; + + virtual VideoFrame Preprocess(const VideoFrame& frame) = 0; + }; + ~TestVideoCapturer() override; void AddOrUpdateSink(rtc::VideoSinkInterface* sink, const rtc::VideoSinkWants& wants) override; void RemoveSink(rtc::VideoSinkInterface* sink) override; + void SetFramePreprocessor(std::unique_ptr preprocessor) { + rtc::CritScope crit(&lock_); + preprocessor_ = std::move(preprocessor); + } protected: void OnFrame(const VideoFrame& frame); @@ -37,7 +48,10 @@ class TestVideoCapturer : public rtc::VideoSourceInterface { private: void UpdateVideoAdapter(); + VideoFrame MaybePreprocess(const VideoFrame& frame); + rtc::CriticalSection lock_; + std::unique_ptr preprocessor_ RTC_GUARDED_BY(lock_); rtc::VideoBroadcaster broadcaster_; cricket::VideoAdapter video_adapter_; };