From 5d6a06c1d29a2061bcf4b321ffceab477a404d51 Mon Sep 17 00:00:00 2001 From: ivica Date: Thu, 17 Sep 2015 05:30:24 -0700 Subject: [PATCH] Refactoring full stack and loopback tests Refactoring full stack, video and screenshare tests to use the same code basis for parametrization and initialization. This patch is done on top of recently commited full stack graphs CL https://codereview.webrtc.org/1289933003/, but virtually no changes have been made to full_stack_plot.py nor to the VideoAnalyzer in full stack, except moving it to video_quality_test.cc. Also, full_stack_samples.cc (build target) was removed and replaced with -output_filename and -duration cmdline arguments in video_loopback and screenshare_loopback. The important things to review: - video_quality_test.h Is the structure of Params good? (examples of usage can be found in full_stack.cc, video_loopback.cc and screenshare_loopback.cc) - video_quality_test.cc Is the initialization correct? The case for using Analyzer and using local renderer are different, can they be further merged? - webrtc_tests.gypi Reproducing the different bitrate settings the full stack and loopback tests had was a little bit tricky. To support both simultaneously, I added BitrateConfig to the Params struct, as well as separate start_bitrate and target_bitrate flags for loopback tests. Note: Side-by-side diff for video_quality_test.cc compares that file directly with the old full_stack.cc, so changes to VideoAnalyzer are clearly visible. Note: Recent CL I've committed added -num_temporal_layers and -sl_discard_threshold args to loopback tests. This was removed here. Support for streams and SVC will be added in a CL following this one. Review URL: https://codereview.webrtc.org/1308403003 Cr-Commit-Position: refs/heads/master@{#9969} --- webrtc/video/full_stack.cc | 904 ++------------------- webrtc/video/full_stack.h | 52 -- webrtc/video/full_stack_plot.py | 6 +- webrtc/video/full_stack_quality_sampler.cc | 142 ---- webrtc/video/loopback.cc | 183 ----- webrtc/video/loopback.h | 63 -- webrtc/video/screenshare_loopback.cc | 216 ++--- webrtc/video/video_loopback.cc | 116 ++- webrtc/video/video_quality_test.cc | 816 +++++++++++++++++++ webrtc/video/video_quality_test.h | 86 ++ webrtc/webrtc_tests.gypi | 47 +- 11 files changed, 1156 insertions(+), 1475 deletions(-) delete mode 100644 webrtc/video/full_stack.h delete mode 100644 webrtc/video/full_stack_quality_sampler.cc delete mode 100644 webrtc/video/loopback.cc delete mode 100644 webrtc/video/loopback.h create mode 100644 webrtc/video/video_quality_test.cc create mode 100644 webrtc/video/video_quality_test.h diff --git a/webrtc/video/full_stack.cc b/webrtc/video/full_stack.cc index 3fb1db66a5..9ae0bd97a6 100644 --- a/webrtc/video/full_stack.cc +++ b/webrtc/video/full_stack.cc @@ -9,831 +9,112 @@ */ #include -#include -#include - #include "testing/gtest/include/gtest/gtest.h" - -#include "webrtc/base/format_macros.h" -#include "webrtc/base/scoped_ptr.h" -#include "webrtc/base/thread_annotations.h" -#include "webrtc/call.h" -#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" -#include "webrtc/frame_callback.h" -#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" -#include "webrtc/system_wrappers/interface/clock.h" -#include "webrtc/system_wrappers/interface/cpu_info.h" -#include "webrtc/system_wrappers/interface/critical_section_wrapper.h" -#include "webrtc/system_wrappers/interface/event_wrapper.h" -#include "webrtc/system_wrappers/interface/sleep.h" -#include "webrtc/test/encoder_settings.h" -#include "webrtc/test/fake_encoder.h" -#include "webrtc/test/frame_generator.h" -#include "webrtc/test/frame_generator_capturer.h" -#include "webrtc/test/statistics.h" -#include "webrtc/test/testsupport/fileutils.h" -#include "webrtc/typedefs.h" -#include "webrtc/video/full_stack.h" +#include "webrtc/video/video_quality_test.h" namespace webrtc { static const int kFullStackTestDurationSecs = 60; -static const int kSendStatsPollingIntervalMs = 1000; -class VideoAnalyzer : public PacketReceiver, - public newapi::Transport, - public VideoRenderer, - public VideoCaptureInput, - public EncodedFrameObserver { +class FullStackTest : public VideoQualityTest { public: - VideoAnalyzer(VideoCaptureInput* input, - Transport* transport, - const char* test_label, - double avg_psnr_threshold, - double avg_ssim_threshold, - int duration_frames, - const std::string& graph_data_output_filename) - : input_(input), - transport_(transport), - receiver_(nullptr), - send_stream_(nullptr), - test_label_(test_label), - graph_data_output_filename_(graph_data_output_filename), - frames_to_process_(duration_frames), - frames_recorded_(0), - frames_processed_(0), - dropped_frames_(0), - last_render_time_(0), - rtp_timestamp_delta_(0), - avg_psnr_threshold_(avg_psnr_threshold), - avg_ssim_threshold_(avg_ssim_threshold), - comparison_available_event_(EventWrapper::Create()), - done_(EventWrapper::Create()) { - // Create thread pool for CPU-expensive PSNR/SSIM calculations. - - // Try to use about as many threads as cores, but leave kMinCoresLeft alone, - // so that we don't accidentally starve "real" worker threads (codec etc). - // Also, don't allocate more than kMaxComparisonThreads, even if there are - // spare cores. - - uint32_t num_cores = CpuInfo::DetectNumberOfCores(); - RTC_DCHECK_GE(num_cores, 1u); - static const uint32_t kMinCoresLeft = 4; - static const uint32_t kMaxComparisonThreads = 8; - - if (num_cores <= kMinCoresLeft) { - num_cores = 1; - } else { - num_cores -= kMinCoresLeft; - num_cores = std::min(num_cores, kMaxComparisonThreads); - } - - for (uint32_t i = 0; i < num_cores; ++i) { - rtc::scoped_ptr thread = - ThreadWrapper::CreateThread(&FrameComparisonThread, this, "Analyzer"); - EXPECT_TRUE(thread->Start()); - comparison_thread_pool_.push_back(thread.release()); - } - - stats_polling_thread_ = - ThreadWrapper::CreateThread(&PollStatsThread, this, "StatsPoller"); - EXPECT_TRUE(stats_polling_thread_->Start()); + void RunTest(const VideoQualityTest::Params ¶ms) { + RunWithAnalyzer(params); } - - ~VideoAnalyzer() { - for (ThreadWrapper* thread : comparison_thread_pool_) { - EXPECT_TRUE(thread->Stop()); - delete thread; - } - } - - virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; } - - DeliveryStatus DeliverPacket(MediaType media_type, - const uint8_t* packet, - size_t length, - const PacketTime& packet_time) override { - rtc::scoped_ptr parser(RtpHeaderParser::Create()); - RTPHeader header; - parser->Parse(packet, length, &header); - { - rtc::CritScope lock(&crit_); - recv_times_[header.timestamp - rtp_timestamp_delta_] = - Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); - } - - return receiver_->DeliverPacket(media_type, packet, length, packet_time); - } - - void IncomingCapturedFrame(const VideoFrame& video_frame) override { - VideoFrame copy = video_frame; - copy.set_timestamp(copy.ntp_time_ms() * 90); - - { - rtc::CritScope lock(&crit_); - if (first_send_frame_.IsZeroSize() && rtp_timestamp_delta_ == 0) - first_send_frame_ = copy; - - frames_.push_back(copy); - } - - input_->IncomingCapturedFrame(video_frame); - } - - bool SendRtp(const uint8_t* packet, size_t length) override { - rtc::scoped_ptr parser(RtpHeaderParser::Create()); - RTPHeader header; - parser->Parse(packet, length, &header); - - { - rtc::CritScope lock(&crit_); - if (rtp_timestamp_delta_ == 0) { - rtp_timestamp_delta_ = - header.timestamp - first_send_frame_.timestamp(); - first_send_frame_.Reset(); - } - uint32_t timestamp = header.timestamp - rtp_timestamp_delta_; - send_times_[timestamp] = - Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); - encoded_frame_sizes_[timestamp] += - length - (header.headerLength + header.paddingLength); - } - - return transport_->SendRtp(packet, length); - } - - bool SendRtcp(const uint8_t* packet, size_t length) override { - return transport_->SendRtcp(packet, length); - } - - void EncodedFrameCallback(const EncodedFrame& frame) override { - rtc::CritScope lock(&comparison_lock_); - if (frames_recorded_ < frames_to_process_) - encoded_frame_size_.AddSample(frame.length_); - } - - void RenderFrame(const VideoFrame& video_frame, - int time_to_render_ms) override { - int64_t render_time_ms = - Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); - uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_; - - rtc::CritScope lock(&crit_); - - while (frames_.front().timestamp() < send_timestamp) { - AddFrameComparison(frames_.front(), last_rendered_frame_, true, - render_time_ms); - frames_.pop_front(); - } - - VideoFrame reference_frame = frames_.front(); - frames_.pop_front(); - assert(!reference_frame.IsZeroSize()); - EXPECT_EQ(reference_frame.timestamp(), send_timestamp); - assert(reference_frame.timestamp() == send_timestamp); - - AddFrameComparison(reference_frame, video_frame, false, render_time_ms); - - last_rendered_frame_ = video_frame; - } - - bool IsTextureSupported() const override { return false; } - - void Wait() { - // Frame comparisons can be very expensive. Wait for test to be done, but - // at time-out check if frames_processed is going up. If so, give it more - // time, otherwise fail. Hopefully this will reduce test flakiness. - - int last_frames_processed = -1; - EventTypeWrapper eventType; - int iteration = 0; - while ((eventType = done_->Wait(FullStackTest::kDefaultTimeoutMs)) != - kEventSignaled) { - int frames_processed; - { - rtc::CritScope crit(&comparison_lock_); - frames_processed = frames_processed_; - } - - // Print some output so test infrastructure won't think we've crashed. - const char* kKeepAliveMessages[3] = { - "Uh, I'm-I'm not quite dead, sir.", - "Uh, I-I think uh, I could pull through, sir.", - "Actually, I think I'm all right to come with you--"}; - printf("- %s\n", kKeepAliveMessages[iteration++ % 3]); - - if (last_frames_processed == -1) { - last_frames_processed = frames_processed; - continue; - } - ASSERT_GT(frames_processed, last_frames_processed) - << "Analyzer stalled while waiting for test to finish."; - last_frames_processed = frames_processed; - } - - if (iteration > 0) - printf("- Farewell, sweet Concorde!\n"); - - // Signal stats polling thread if that is still waiting and stop it now, - // since it uses the send_stream_ reference that might be reclaimed after - // returning from this method. - done_->Set(); - EXPECT_TRUE(stats_polling_thread_->Stop()); - } - - VideoCaptureInput* input_; - Transport* transport_; - PacketReceiver* receiver_; - VideoSendStream* send_stream_; - - private: - struct FrameComparison { - FrameComparison() - : dropped(false), - send_time_ms(0), - recv_time_ms(0), - render_time_ms(0), - encoded_frame_size(0) {} - - FrameComparison(const VideoFrame& reference, - const VideoFrame& render, - bool dropped, - int64_t send_time_ms, - int64_t recv_time_ms, - int64_t render_time_ms, - size_t encoded_frame_size) - : reference(reference), - render(render), - dropped(dropped), - send_time_ms(send_time_ms), - recv_time_ms(recv_time_ms), - render_time_ms(render_time_ms), - encoded_frame_size(encoded_frame_size) {} - - VideoFrame reference; - VideoFrame render; - bool dropped; - int64_t send_time_ms; - int64_t recv_time_ms; - int64_t render_time_ms; - size_t encoded_frame_size; - }; - - struct Sample { - Sample(double dropped, - double input_time_ms, - double send_time_ms, - double recv_time_ms, - double encoded_frame_size, - double psnr, - double ssim, - double render_time_ms) - : dropped(dropped), - input_time_ms(input_time_ms), - send_time_ms(send_time_ms), - recv_time_ms(recv_time_ms), - encoded_frame_size(encoded_frame_size), - psnr(psnr), - ssim(ssim), - render_time_ms(render_time_ms) {} - - double dropped; - double input_time_ms; - double send_time_ms; - double recv_time_ms; - double encoded_frame_size; - double psnr; - double ssim; - double render_time_ms; - }; - - void AddFrameComparison(const VideoFrame& reference, - const VideoFrame& render, - bool dropped, - int64_t render_time_ms) - EXCLUSIVE_LOCKS_REQUIRED(crit_) { - int64_t send_time_ms = send_times_[reference.timestamp()]; - send_times_.erase(reference.timestamp()); - int64_t recv_time_ms = recv_times_[reference.timestamp()]; - recv_times_.erase(reference.timestamp()); - - size_t encoded_size = encoded_frame_sizes_[reference.timestamp()]; - encoded_frame_sizes_.erase(reference.timestamp()); - - VideoFrame reference_copy; - VideoFrame render_copy; - reference_copy.CopyFrame(reference); - render_copy.CopyFrame(render); - - rtc::CritScope crit(&comparison_lock_); - comparisons_.push_back(FrameComparison(reference_copy, render_copy, dropped, - send_time_ms, recv_time_ms, - render_time_ms, encoded_size)); - comparison_available_event_->Set(); - } - - static bool PollStatsThread(void* obj) { - return static_cast(obj)->PollStats(); - } - - bool PollStats() { - switch (done_->Wait(kSendStatsPollingIntervalMs)) { - case kEventSignaled: - case kEventError: - done_->Set(); // Make sure main thread is also signaled. - return false; - case kEventTimeout: - break; - default: - RTC_NOTREACHED(); - } - - VideoSendStream::Stats stats = send_stream_->GetStats(); - - rtc::CritScope crit(&comparison_lock_); - encode_frame_rate_.AddSample(stats.encode_frame_rate); - encode_time_ms.AddSample(stats.avg_encode_time_ms); - encode_usage_percent.AddSample(stats.encode_usage_percent); - media_bitrate_bps.AddSample(stats.media_bitrate_bps); - - return true; - } - - static bool FrameComparisonThread(void* obj) { - return static_cast(obj)->CompareFrames(); - } - - bool CompareFrames() { - if (AllFramesRecorded()) - return false; - - VideoFrame reference; - VideoFrame render; - FrameComparison comparison; - - if (!PopComparison(&comparison)) { - // Wait until new comparison task is available, or test is done. - // If done, wake up remaining threads waiting. - comparison_available_event_->Wait(1000); - if (AllFramesRecorded()) { - comparison_available_event_->Set(); - return false; - } - return true; // Try again. - } - - PerformFrameComparison(comparison); - - if (FrameProcessed()) { - PrintResults(); - if (!graph_data_output_filename_.empty()) - PrintSamplesToFile(); - done_->Set(); - comparison_available_event_->Set(); - return false; - } - - return true; - } - - bool PopComparison(FrameComparison* comparison) { - rtc::CritScope crit(&comparison_lock_); - // If AllFramesRecorded() is true, it means we have already popped - // frames_to_process_ frames from comparisons_, so there is no more work - // for this thread to be done. frames_processed_ might still be lower if - // all comparisons are not done, but those frames are currently being - // worked on by other threads. - if (comparisons_.empty() || AllFramesRecorded()) - return false; - - *comparison = comparisons_.front(); - comparisons_.pop_front(); - - FrameRecorded(); - return true; - } - - // Increment counter for number of frames received for comparison. - void FrameRecorded() { - rtc::CritScope crit(&comparison_lock_); - ++frames_recorded_; - } - - // Returns true if all frames to be compared have been taken from the queue. - bool AllFramesRecorded() { - rtc::CritScope crit(&comparison_lock_); - assert(frames_recorded_ <= frames_to_process_); - return frames_recorded_ == frames_to_process_; - } - - // Increase count of number of frames processed. Returns true if this was the - // last frame to be processed. - bool FrameProcessed() { - rtc::CritScope crit(&comparison_lock_); - ++frames_processed_; - assert(frames_processed_ <= frames_to_process_); - return frames_processed_ == frames_to_process_; - } - - void PrintResults() { - rtc::CritScope crit(&comparison_lock_); - PrintResult("psnr", psnr_, " dB"); - PrintResult("ssim", ssim_, ""); - PrintResult("sender_time", sender_time_, " ms"); - printf("RESULT dropped_frames: %s = %d frames\n", test_label_, - dropped_frames_); - PrintResult("receiver_time", receiver_time_, " ms"); - PrintResult("total_delay_incl_network", end_to_end_, " ms"); - PrintResult("time_between_rendered_frames", rendered_delta_, " ms"); - PrintResult("encoded_frame_size", encoded_frame_size_, " bytes"); - PrintResult("encode_frame_rate", encode_frame_rate_, " fps"); - PrintResult("encode_time", encode_time_ms, " ms"); - PrintResult("encode_usage_percent", encode_usage_percent, " percent"); - PrintResult("media_bitrate", media_bitrate_bps, " bps"); - - EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_); - EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_); - } - - void PerformFrameComparison(const FrameComparison& comparison) { - // Perform expensive psnr and ssim calculations while not holding lock. - double psnr = I420PSNR(&comparison.reference, &comparison.render); - double ssim = I420SSIM(&comparison.reference, &comparison.render); - - int64_t input_time_ms = comparison.reference.ntp_time_ms(); - - rtc::CritScope crit(&comparison_lock_); - if (!graph_data_output_filename_.empty()) { - samples_.push_back(Sample( - comparison.dropped, input_time_ms, comparison.send_time_ms, - comparison.recv_time_ms, comparison.encoded_frame_size, psnr, ssim, - comparison.render_time_ms)); - } - psnr_.AddSample(psnr); - ssim_.AddSample(ssim); - - if (comparison.dropped) { - ++dropped_frames_; - return; - } - if (last_render_time_ != 0) - rendered_delta_.AddSample(comparison.render_time_ms - last_render_time_); - last_render_time_ = comparison.render_time_ms; - - sender_time_.AddSample(comparison.send_time_ms - input_time_ms); - receiver_time_.AddSample(comparison.render_time_ms - - comparison.recv_time_ms); - end_to_end_.AddSample(comparison.render_time_ms - input_time_ms); - encoded_frame_size_.AddSample(comparison.encoded_frame_size); - } - - void PrintResult(const char* result_type, - test::Statistics stats, - const char* unit) { - printf("RESULT %s: %s = {%f, %f}%s\n", - result_type, - test_label_, - stats.Mean(), - stats.StandardDeviation(), - unit); - } - - void PrintSamplesToFile(void) { - FILE* out = fopen(graph_data_output_filename_.c_str(), "w"); - RTC_CHECK(out != nullptr) << "Couldn't open file: " - << graph_data_output_filename_; - - rtc::CritScope crit(&comparison_lock_); - std::sort(samples_.begin(), samples_.end(), - [](const Sample& A, const Sample& B) - -> bool { return A.input_time_ms < B.input_time_ms; }); - - fprintf(out, "%s\n", test_label_); - fprintf(out, "%" PRIuS "\n", samples_.size()); - fprintf(out, - "dropped " - "input_time_ms " - "send_time_ms " - "recv_time_ms " - "encoded_frame_size " - "psnr " - "ssim " - "render_time_ms\n"); - for (const Sample& sample : samples_) { - fprintf(out, "%lf %lf %lf %lf %lf %lf %lf %lf\n", sample.dropped, - sample.input_time_ms, sample.send_time_ms, sample.recv_time_ms, - sample.encoded_frame_size, sample.psnr, sample.ssim, - sample.render_time_ms); - } - fclose(out); - } - - const char* const test_label_; - std::string graph_data_output_filename_; - std::vector samples_ GUARDED_BY(comparison_lock_); - test::Statistics sender_time_ GUARDED_BY(comparison_lock_); - test::Statistics receiver_time_ GUARDED_BY(comparison_lock_); - test::Statistics psnr_ GUARDED_BY(comparison_lock_); - test::Statistics ssim_ GUARDED_BY(comparison_lock_); - test::Statistics end_to_end_ GUARDED_BY(comparison_lock_); - test::Statistics rendered_delta_ GUARDED_BY(comparison_lock_); - test::Statistics encoded_frame_size_ GUARDED_BY(comparison_lock_); - test::Statistics encode_frame_rate_ GUARDED_BY(comparison_lock_); - test::Statistics encode_time_ms GUARDED_BY(comparison_lock_); - test::Statistics encode_usage_percent GUARDED_BY(comparison_lock_); - test::Statistics media_bitrate_bps GUARDED_BY(comparison_lock_); - - const int frames_to_process_; - int frames_recorded_; - int frames_processed_; - int dropped_frames_; - int64_t last_render_time_; - uint32_t rtp_timestamp_delta_; - - rtc::CriticalSection crit_; - std::deque frames_ GUARDED_BY(crit_); - std::deque send_stats_ GUARDED_BY(crit_); - VideoFrame last_rendered_frame_ GUARDED_BY(crit_); - std::map send_times_ GUARDED_BY(crit_); - std::map recv_times_ GUARDED_BY(crit_); - std::map encoded_frame_sizes_ GUARDED_BY(crit_); - VideoFrame first_send_frame_ GUARDED_BY(crit_); - const double avg_psnr_threshold_; - const double avg_ssim_threshold_; - - rtc::CriticalSection comparison_lock_; - std::vector comparison_thread_pool_; - rtc::scoped_ptr stats_polling_thread_; - const rtc::scoped_ptr comparison_available_event_; - std::deque comparisons_ GUARDED_BY(comparison_lock_); - const rtc::scoped_ptr done_; }; -void FullStackTest::RunTest(const FullStackTestParams& params) { - // TODO(ivica): Add num_temporal_layers as a param. - unsigned char num_temporal_layers = - params.graph_data_output_filename.empty() ? 2 : 1; - - test::DirectTransport send_transport(params.link); - test::DirectTransport recv_transport(params.link); - VideoAnalyzer analyzer(nullptr, &send_transport, params.test_label, - params.avg_psnr_threshold, params.avg_ssim_threshold, - params.test_durations_secs * params.clip.fps, - params.graph_data_output_filename); - - CreateCalls(Call::Config(), Call::Config()); - - analyzer.SetReceiver(receiver_call_->Receiver()); - send_transport.SetReceiver(&analyzer); - recv_transport.SetReceiver(sender_call_->Receiver()); - - CreateSendConfig(1, &analyzer); - - rtc::scoped_ptr encoder; - if (params.codec == "VP8") { - encoder = - rtc::scoped_ptr(VideoEncoder::Create(VideoEncoder::kVp8)); - send_config_.encoder_settings.encoder = encoder.get(); - send_config_.encoder_settings.payload_name = "VP8"; - } else if (params.codec == "VP9") { - encoder = - rtc::scoped_ptr(VideoEncoder::Create(VideoEncoder::kVp9)); - send_config_.encoder_settings.encoder = encoder.get(); - send_config_.encoder_settings.payload_name = "VP9"; - } else { - RTC_NOTREACHED() << "Codec not supported!"; - return; - } - send_config_.encoder_settings.payload_type = 124; - - send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs; - send_config_.rtp.rtx.ssrcs.push_back(kSendRtxSsrcs[0]); - send_config_.rtp.rtx.payload_type = kSendRtxPayloadType; - - VideoStream* stream = &encoder_config_.streams[0]; - stream->width = params.clip.width; - stream->height = params.clip.height; - stream->min_bitrate_bps = params.min_bitrate_bps; - stream->target_bitrate_bps = params.target_bitrate_bps; - stream->max_bitrate_bps = params.max_bitrate_bps; - stream->max_framerate = params.clip.fps; - - VideoCodecVP8 vp8_settings; - VideoCodecVP9 vp9_settings; - if (params.mode == ContentMode::kScreensharingStaticImage || - params.mode == ContentMode::kScreensharingScrollingImage) { - encoder_config_.content_type = VideoEncoderConfig::ContentType::kScreen; - encoder_config_.min_transmit_bitrate_bps = 400 * 1000; - if (params.codec == "VP8") { - vp8_settings = VideoEncoder::GetDefaultVp8Settings(); - vp8_settings.denoisingOn = false; - vp8_settings.frameDroppingOn = false; - vp8_settings.numberOfTemporalLayers = num_temporal_layers; - encoder_config_.encoder_specific_settings = &vp8_settings; - } else if (params.codec == "VP9") { - vp9_settings = VideoEncoder::GetDefaultVp9Settings(); - vp9_settings.denoisingOn = false; - vp9_settings.frameDroppingOn = false; - vp9_settings.numberOfTemporalLayers = num_temporal_layers; - encoder_config_.encoder_specific_settings = &vp9_settings; - } - - stream->temporal_layer_thresholds_bps.clear(); - if (num_temporal_layers > 1) { - stream->temporal_layer_thresholds_bps.push_back( - stream->target_bitrate_bps); - } - } - - CreateMatchingReceiveConfigs(&recv_transport); - receive_configs_[0].renderer = &analyzer; - receive_configs_[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs; - receive_configs_[0].rtp.rtx[kSendRtxPayloadType].ssrc = kSendRtxSsrcs[0]; - receive_configs_[0].rtp.rtx[kSendRtxPayloadType].payload_type = - kSendRtxPayloadType; - - for (auto& config : receive_configs_) - config.pre_decode_callback = &analyzer; - CreateStreams(); - analyzer.input_ = send_stream_->Input(); - analyzer.send_stream_ = send_stream_; - - std::vector slides; - slides.push_back(test::ResourcePath("web_screenshot_1850_1110", "yuv")); - slides.push_back(test::ResourcePath("presentation_1850_1110", "yuv")); - slides.push_back(test::ResourcePath("photo_1850_1110", "yuv")); - slides.push_back(test::ResourcePath("difficult_photo_1850_1110", "yuv")); - size_t kSlidesWidth = 1850; - size_t kSlidesHeight = 1110; - - Clock* clock = Clock::GetRealTimeClock(); - rtc::scoped_ptr frame_generator; - - switch (params.mode) { - case ContentMode::kRealTimeVideo: - frame_generator.reset(test::FrameGenerator::CreateFromYuvFile( - std::vector(1, - test::ResourcePath(params.clip.name, "yuv")), - params.clip.width, params.clip.height, 1)); - break; - case ContentMode::kScreensharingScrollingImage: - frame_generator.reset( - test::FrameGenerator::CreateScrollingInputFromYuvFiles( - clock, slides, kSlidesWidth, kSlidesHeight, params.clip.width, - params.clip.height, 2000, - 8000)); // Scroll for 2 seconds, then pause for 8. - break; - case ContentMode::kScreensharingStaticImage: - frame_generator.reset(test::FrameGenerator::CreateFromYuvFile( - slides, kSlidesWidth, kSlidesHeight, - 10 * params.clip.fps)); // Cycle image every 10 seconds. - break; - } - - ASSERT_TRUE(frame_generator.get() != nullptr); - frame_generator_capturer_.reset(new test::FrameGeneratorCapturer( - clock, &analyzer, frame_generator.release(), params.clip.fps)); - ASSERT_TRUE(frame_generator_capturer_->Init()); - - Start(); - - analyzer.Wait(); - - send_transport.StopSending(); - recv_transport.StopSending(); - - Stop(); - - DestroyStreams(); -} TEST_F(FullStackTest, ParisQcifWithoutPacketLoss) { - FullStackTestParams paris_qcif = {"net_delay_0_0_plr_0", - {"paris_qcif", 176, 144, 30}, - ContentMode::kRealTimeVideo, - 300000, - 300000, - 300000, - 36.0, - 0.96, - kFullStackTestDurationSecs, - "VP8"}; + VideoQualityTest::Params paris_qcif = { + {176, 144, 30, 300000, 300000, 300000, "VP8", 1}, + {"paris_qcif"}, + {}, + {"net_delay_0_0_plr_0", 36.0, 0.96, kFullStackTestDurationSecs}}; RunTest(paris_qcif); } TEST_F(FullStackTest, ForemanCifWithoutPacketLoss) { // TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif. - FullStackTestParams foreman_cif = {"foreman_cif_net_delay_0_0_plr_0", - {"foreman_cif", 352, 288, 30}, - ContentMode::kRealTimeVideo, - 700000, - 700000, - 700000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP8"}; + VideoQualityTest::Params foreman_cif = { + {352, 288, 30, 700000, 700000, 700000, "VP8", 1}, + {"foreman_cif"}, + {}, + {"foreman_cif_net_delay_0_0_plr_0", 0.0, 0.0, kFullStackTestDurationSecs} + }; RunTest(foreman_cif); } TEST_F(FullStackTest, ForemanCifPlr5) { - FullStackTestParams foreman_cif = {"foreman_cif_delay_50_0_plr_5", - {"foreman_cif", 352, 288, 30}, - ContentMode::kRealTimeVideo, - 30000, - 500000, - 2000000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP8"}; - foreman_cif.link.loss_percent = 5; - foreman_cif.link.queue_delay_ms = 50; + VideoQualityTest::Params foreman_cif = { + {352, 288, 30, 30000, 500000, 2000000, "VP8", 1}, + {"foreman_cif"}, + {}, + {"foreman_cif_delay_50_0_plr_5", 0.0, 0.0, kFullStackTestDurationSecs}}; + foreman_cif.pipe.loss_percent = 5; + foreman_cif.pipe.queue_delay_ms = 50; RunTest(foreman_cif); } TEST_F(FullStackTest, ForemanCif500kbps) { - FullStackTestParams foreman_cif = {"foreman_cif_500kbps", - {"foreman_cif", 352, 288, 30}, - ContentMode::kRealTimeVideo, - 30000, - 500000, - 2000000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP8"}; - foreman_cif.link.queue_length_packets = 0; - foreman_cif.link.queue_delay_ms = 0; - foreman_cif.link.link_capacity_kbps = 500; + VideoQualityTest::Params foreman_cif = { + {352, 288, 30, 30000, 500000, 2000000, "VP8", 1}, + {"foreman_cif"}, + {}, + {"foreman_cif_500kbps", 0.0, 0.0, kFullStackTestDurationSecs}}; + foreman_cif.pipe.queue_length_packets = 0; + foreman_cif.pipe.queue_delay_ms = 0; + foreman_cif.pipe.link_capacity_kbps = 500; RunTest(foreman_cif); } TEST_F(FullStackTest, ForemanCif500kbpsLimitedQueue) { - FullStackTestParams foreman_cif = {"foreman_cif_500kbps_32pkts_queue", - {"foreman_cif", 352, 288, 30}, - ContentMode::kRealTimeVideo, - 30000, - 500000, - 2000000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP8"}; - foreman_cif.link.queue_length_packets = 32; - foreman_cif.link.queue_delay_ms = 0; - foreman_cif.link.link_capacity_kbps = 500; + VideoQualityTest::Params foreman_cif = { + {352, 288, 30, 30000, 500000, 2000000, "VP8", 1}, + {"foreman_cif"}, + {}, + {"foreman_cif_500kbps_32pkts_queue", 0.0, 0.0, kFullStackTestDurationSecs} + }; + foreman_cif.pipe.queue_length_packets = 32; + foreman_cif.pipe.queue_delay_ms = 0; + foreman_cif.pipe.link_capacity_kbps = 500; RunTest(foreman_cif); } TEST_F(FullStackTest, ForemanCif500kbps100ms) { - FullStackTestParams foreman_cif = {"foreman_cif_500kbps_100ms", - {"foreman_cif", 352, 288, 30}, - ContentMode::kRealTimeVideo, - 30000, - 500000, - 2000000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP8"}; - foreman_cif.link.queue_length_packets = 0; - foreman_cif.link.queue_delay_ms = 100; - foreman_cif.link.link_capacity_kbps = 500; + VideoQualityTest::Params foreman_cif = { + {352, 288, 30, 30000, 500000, 2000000, "VP8", 1}, + {"foreman_cif"}, + {}, + {"foreman_cif_500kbps_100ms", 0.0, 0.0, kFullStackTestDurationSecs}}; + foreman_cif.pipe.queue_length_packets = 0; + foreman_cif.pipe.queue_delay_ms = 100; + foreman_cif.pipe.link_capacity_kbps = 500; RunTest(foreman_cif); } TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueue) { - FullStackTestParams foreman_cif = {"foreman_cif_500kbps_100ms_32pkts_queue", - {"foreman_cif", 352, 288, 30}, - ContentMode::kRealTimeVideo, - 30000, - 500000, - 2000000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP8"}; - foreman_cif.link.queue_length_packets = 32; - foreman_cif.link.queue_delay_ms = 100; - foreman_cif.link.link_capacity_kbps = 500; + VideoQualityTest::Params foreman_cif = { + {352, 288, 30, 30000, 500000, 2000000, "VP8", 1}, + {"foreman_cif"}, + {}, + {"foreman_cif_500kbps_100ms_32pkts_queue", 0.0, 0.0, + kFullStackTestDurationSecs}}; + foreman_cif.pipe.queue_length_packets = 32; + foreman_cif.pipe.queue_delay_ms = 100; + foreman_cif.pipe.link_capacity_kbps = 500; RunTest(foreman_cif); } TEST_F(FullStackTest, ForemanCif1000kbps100msLimitedQueue) { - FullStackTestParams foreman_cif = {"foreman_cif_1000kbps_100ms_32pkts_queue", - {"foreman_cif", 352, 288, 30}, - ContentMode::kRealTimeVideo, - 30000, - 2000000, - 2000000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP8"}; - foreman_cif.link.queue_length_packets = 32; - foreman_cif.link.queue_delay_ms = 100; - foreman_cif.link.link_capacity_kbps = 1000; + VideoQualityTest::Params foreman_cif = { + {352, 288, 30, 30000, 500000, 2000000, "VP8", 1}, + {"foreman_cif"}, + {}, + {"foreman_cif_1000kbps_100ms_32pkts_queue", 0.0, 0.0, + kFullStackTestDurationSecs}}; + foreman_cif.pipe.queue_length_packets = 32; + foreman_cif.pipe.queue_delay_ms = 100; + foreman_cif.pipe.link_capacity_kbps = 1000; RunTest(foreman_cif); } @@ -841,49 +122,30 @@ TEST_F(FullStackTest, ForemanCif1000kbps100msLimitedQueue) { // https://code.google.com/p/chromium/issues/detail?id=513170 #include "webrtc/test/testsupport/gtest_disable.h" TEST_F(FullStackTest, DISABLED_ON_ANDROID(ScreenshareSlidesVP8_2TL)) { - FullStackTestParams screenshare_params = { - "screenshare_slides", - {"screenshare_slides", 1850, 1110, 5}, - ContentMode::kScreensharingStaticImage, - 50000, - 200000, - 2000000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP8"}; - RunTest(screenshare_params); + VideoQualityTest::Params screenshare = { + {1850, 1110, 5, 50000, 200000, 2000000, "VP8", 2, 400000}, + {}, // Video-specific. + {true, 10}, // Screenshare-specific. + {"screenshare_slides", 0.0, 0.0, kFullStackTestDurationSecs}}; + RunTest(screenshare); } TEST_F(FullStackTest, DISABLED_ON_ANDROID(ScreenshareSlidesVP8_2TL_Scroll)) { - FullStackTestParams screenshare_params = { - "screenshare_slides_scrolling", - // Crop height by two, scrolling vertically only. - {"screenshare_slides_scrolling", 1850, 1110 / 2, 5}, - ContentMode::kScreensharingScrollingImage, - 50000, - 200000, - 2000000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP8"}; - RunTest(screenshare_params); + VideoQualityTest::Params config = { + {1850, 1110 / 2, 5, 50000, 200000, 2000000, "VP8", 2, 400000}, + {}, + {true, 10, 2}, + {"screenshare_slides_scrolling", 0.0, 0.0, kFullStackTestDurationSecs}}; + RunTest(config); } // Disabled on Android along with VP8 screenshare above. TEST_F(FullStackTest, DISABLED_ON_ANDROID(ScreenshareSlidesVP9_2TL)) { - FullStackTestParams screenshare_params = { - "screenshare_slides_vp9_2tl", - {"screenshare_slides", 1850, 1110, 5}, - ContentMode::kScreensharingStaticImage, - 50000, - 200000, - 2000000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP9"}; - RunTest(screenshare_params); + VideoQualityTest::Params screenshare = { + {1850, 1110, 5, 50000, 200000, 2000000, "VP9", 2, 400000}, + {}, + {true, 10}, + {"screenshare_slides_vp9_2tl", 0.0, 0.0, kFullStackTestDurationSecs}}; + RunTest(screenshare); } } // namespace webrtc diff --git a/webrtc/video/full_stack.h b/webrtc/video/full_stack.h deleted file mode 100644 index 56ef4b329f..0000000000 --- a/webrtc/video/full_stack.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ -#ifndef WEBRTC_VIDEO_FULL_STACK_H_ -#define WEBRTC_VIDEO_FULL_STACK_H_ - -#include - -#include "webrtc/test/call_test.h" -#include "webrtc/test/direct_transport.h" - -namespace webrtc { - -enum class ContentMode { - kRealTimeVideo, - kScreensharingStaticImage, - kScreensharingScrollingImage, -}; - -struct FullStackTestParams { - const char* test_label; - struct { - const char* name; - size_t width, height; - int fps; - } clip; - ContentMode mode; - int min_bitrate_bps; - int target_bitrate_bps; - int max_bitrate_bps; - double avg_psnr_threshold; - double avg_ssim_threshold; - int test_durations_secs; - std::string codec; - FakeNetworkPipe::Config link; - std::string graph_data_output_filename; -}; - -class FullStackTest : public test::CallTest { - protected: - void RunTest(const FullStackTestParams& params); -}; - -} // namespace webrtc - -#endif // WEBRTC_VIDEO_FULL_STACK_H_ diff --git a/webrtc/video/full_stack_plot.py b/webrtc/video/full_stack_plot.py index 92b50a44a1..0e8990157f 100755 --- a/webrtc/video/full_stack_plot.py +++ b/webrtc/video/full_stack_plot.py @@ -7,7 +7,7 @@ # in the file PATENTS. All contributing project authors may # be found in the AUTHORS file in the root of the source tree. -"""Generate graphs for data generated by full_stack_quality_sampler.cc. +"""Generate graphs for data generated by loopback tests. Usage examples: Show end to end time for a single full stack test. @@ -76,7 +76,6 @@ _fields = [ name_to_id = {field[1]: field[0] for field in _fields} id_to_title = {field[0]: field[2] for field in _fields} - def field_arg_to_id(arg): if arg == "none": return None @@ -297,7 +296,6 @@ load_files.cache = {} def get_parser(): class CustomAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string=None): if "ordered_args" not in namespace: namespace.ordered_args = [] @@ -331,7 +329,7 @@ def get_parser(): "Otherwise, a window will be shown.") parser.add_argument( "files", nargs="+", action=CustomAction, - help="List of text-based files generated by full_stack.cc") + help="List of text-based files generated by loopback tests.") return parser diff --git a/webrtc/video/full_stack_quality_sampler.cc b/webrtc/video/full_stack_quality_sampler.cc deleted file mode 100644 index a17975699a..0000000000 --- a/webrtc/video/full_stack_quality_sampler.cc +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "gflags/gflags.h" -#include "webrtc/test/field_trial.h" -#include "webrtc/test/run_test.h" -#include "webrtc/video/full_stack.h" - -namespace webrtc { -namespace flags { - -DEFINE_string(title, "Full stack graph", "Graph title."); -std::string Title() { - return static_cast(FLAGS_title); -} - -DEFINE_string(filename, "graph_data.txt", "Name of a target graph data file."); -std::string Filename() { - return static_cast(FLAGS_filename); -} - -DEFINE_string(clip_name, "screenshare_slides", "Clip name, resource name."); -std::string ClipName() { - return static_cast(FLAGS_clip_name); -} - -DEFINE_int32(width, 1850, "Video width (crops source)."); -size_t Width() { - return static_cast(FLAGS_width); -} - -DEFINE_int32(height, 1110, "Video height (crops source)."); -size_t Height() { - return static_cast(FLAGS_height); -} - -DEFINE_int32(fps, 5, "Frames per second."); -int Fps() { - return static_cast(FLAGS_fps); -} - -DEFINE_int32( - content_mode, - 1, - "0 - real time video, 1 - screenshare static, 2 - screenshare scrolling."); -ContentMode ContentModeFlag() { - switch (FLAGS_content_mode) { - case 0: - return ContentMode::kRealTimeVideo; - case 1: - return ContentMode::kScreensharingStaticImage; - case 2: - return ContentMode::kScreensharingScrollingImage; - default: - RTC_NOTREACHED() << "Unknown content mode!"; - return ContentMode::kScreensharingStaticImage; - } -} - -DEFINE_int32(test_duration, 60, "Duration of the test in seconds."); -int TestDuration() { - return static_cast(FLAGS_test_duration); -} - -DEFINE_int32(min_bitrate, 50000, "Minimum video bitrate."); -int MinBitrate() { - return static_cast(FLAGS_min_bitrate); -} - -DEFINE_int32(target_bitrate, - 500000, - "Target video bitrate. (Default value here different than in full " - "stack tests!)"); -int TargetBitrate() { - return static_cast(FLAGS_target_bitrate); -} - -DEFINE_int32(max_bitrate, - 500000, - "Maximum video bitrate. (Default value here different than in " - "full stack tests!)"); -int MaxBitrate() { - return static_cast(FLAGS_max_bitrate); -} - -DEFINE_string(codec, "VP9", "Video codec to use."); -std::string Codec() { - return static_cast(FLAGS_codec); -} - -DEFINE_string( - force_fieldtrials, - "", - "Field trials control experimental feature code which can be forced. " - "E.g. running with --force_fieldtrials=WebRTC-FooFeature/Enable/" - " will assign the group Enable to field trial WebRTC-FooFeature. Multiple " - "trials are separated by \"/\""); -} // namespace flags - -class FullStackGenGraph : public FullStackTest { - public: - void TestBody() override { - std::string title = flags::Title(); - std::string clip_name = flags::ClipName(); - FullStackTestParams params = { - title.c_str(), - {clip_name.c_str(), flags::Width(), flags::Height(), flags::Fps()}, - flags::ContentModeFlag(), - flags::MinBitrate(), - flags::TargetBitrate(), - flags::MaxBitrate(), - 0.0, // avg_psnr_threshold - 0.0, // avg_ssim_threshold - flags::TestDuration(), - flags::Codec()}; - params.graph_data_output_filename = flags::Filename(); - - RunTest(params); - } -}; - -void FullStackRun(void) { - FullStackGenGraph full_stack; - full_stack.TestBody(); -} -} // namespace webrtc - -int main(int argc, char* argv[]) { - ::testing::InitGoogleTest(&argc, argv); - google::ParseCommandLineFlags(&argc, &argv, true); - webrtc::test::InitFieldTrialsFromString( - webrtc::flags::FLAGS_force_fieldtrials); - webrtc::test::RunTest(webrtc::FullStackRun); - return 0; -} diff --git a/webrtc/video/loopback.cc b/webrtc/video/loopback.cc deleted file mode 100644 index 9083500c1e..0000000000 --- a/webrtc/video/loopback.cc +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include - -#include - -#include "webrtc/video/loopback.h" - -#include "testing/gtest/include/gtest/gtest.h" - -#include "webrtc/base/checks.h" -#include "webrtc/base/scoped_ptr.h" -#include "webrtc/call.h" -#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" -#include "webrtc/modules/rtp_rtcp/source/rtp_format.h" -#include "webrtc/modules/rtp_rtcp/source/rtp_utility.h" -#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" -#include "webrtc/system_wrappers/interface/clock.h" -#include "webrtc/test/encoder_settings.h" -#include "webrtc/test/fake_encoder.h" -#include "webrtc/test/layer_filtering_transport.h" -#include "webrtc/test/run_loop.h" -#include "webrtc/test/testsupport/trace_to_stderr.h" -#include "webrtc/test/video_capturer.h" -#include "webrtc/test/video_renderer.h" -#include "webrtc/typedefs.h" - -namespace webrtc { -namespace test { - -static const int kAbsSendTimeExtensionId = 7; - -static const uint32_t kSendSsrc = 0x654321; -static const uint32_t kSendRtxSsrc = 0x654322; -static const uint32_t kReceiverLocalSsrc = 0x123456; - -static const uint8_t kRtxVideoPayloadType = 96; -static const uint8_t kVideoPayloadTypeVP8 = 124; -static const uint8_t kVideoPayloadTypeVP9 = 125; - -Loopback::Loopback(const Config& config) - : config_(config), clock_(Clock::GetRealTimeClock()) { -} - -Loopback::~Loopback() { -} - -void Loopback::Run() { - rtc::scoped_ptr trace_to_stderr_; - if (config_.logs) - trace_to_stderr_.reset(new test::TraceToStderr); - - rtc::scoped_ptr local_preview( - test::VideoRenderer::Create("Local Preview", config_.width, - config_.height)); - rtc::scoped_ptr loopback_video( - test::VideoRenderer::Create("Loopback Video", config_.width, - config_.height)); - - Call::Config call_config; - call_config.bitrate_config.min_bitrate_bps = - static_cast(config_.min_bitrate_kbps) * 1000; - call_config.bitrate_config.start_bitrate_bps = - static_cast(config_.start_bitrate_kbps) * 1000; - call_config.bitrate_config.max_bitrate_bps = - static_cast(config_.max_bitrate_kbps) * 1000; - rtc::scoped_ptr call(Call::Create(call_config)); - - FakeNetworkPipe::Config pipe_config; - pipe_config.loss_percent = config_.loss_percent; - pipe_config.link_capacity_kbps = config_.link_capacity_kbps; - pipe_config.queue_length_packets = config_.queue_size; - pipe_config.queue_delay_ms = config_.avg_propagation_delay_ms; - pipe_config.delay_standard_deviation_ms = config_.std_propagation_delay_ms; - LayerFilteringTransport send_transport( - pipe_config, kVideoPayloadTypeVP8, kVideoPayloadTypeVP9, - static_cast(config_.tl_discard_threshold), - static_cast(config_.sl_discard_threshold)); - - // Loopback, call sends to itself. - send_transport.SetReceiver(call->Receiver()); - - VideoSendStream::Config send_config(&send_transport); - send_config.rtp.ssrcs.push_back(kSendSsrc); - send_config.rtp.rtx.ssrcs.push_back(kSendRtxSsrc); - send_config.rtp.rtx.payload_type = kRtxVideoPayloadType; - send_config.rtp.nack.rtp_history_ms = 1000; - send_config.rtp.extensions.push_back( - RtpExtension(RtpExtension::kAbsSendTime, kAbsSendTimeExtensionId)); - - send_config.local_renderer = local_preview.get(); - rtc::scoped_ptr encoder; - if (config_.codec == "VP8") { - encoder.reset(VideoEncoder::Create(VideoEncoder::kVp8)); - } else if (config_.codec == "VP9") { - encoder.reset(VideoEncoder::Create(VideoEncoder::kVp9)); - } else { - // Codec not supported. - RTC_NOTREACHED() << "Codec not supported!"; - return; - } - const int payload_type = - config_.codec == "VP8" ? kVideoPayloadTypeVP8 : kVideoPayloadTypeVP9; - send_config.encoder_settings.encoder = encoder.get(); - send_config.encoder_settings.payload_name = config_.codec; - send_config.encoder_settings.payload_type = payload_type; - - VideoEncoderConfig encoder_config(CreateEncoderConfig()); - - VideoSendStream* send_stream = - call->CreateVideoSendStream(send_config, encoder_config); - - rtc::scoped_ptr capturer(CreateCapturer(send_stream)); - - VideoReceiveStream::Config receive_config(&send_transport); - receive_config.rtp.remote_ssrc = send_config.rtp.ssrcs[0]; - receive_config.rtp.local_ssrc = kReceiverLocalSsrc; - receive_config.rtp.nack.rtp_history_ms = 1000; - receive_config.rtp.remb = true; - receive_config.rtp.rtx[payload_type].ssrc = kSendRtxSsrc; - receive_config.rtp.rtx[payload_type].payload_type = kRtxVideoPayloadType; - receive_config.rtp.extensions.push_back( - RtpExtension(RtpExtension::kAbsSendTime, kAbsSendTimeExtensionId)); - receive_config.renderer = loopback_video.get(); - VideoReceiveStream::Decoder decoder = - test::CreateMatchingDecoder(send_config.encoder_settings); - receive_config.decoders.push_back(decoder); - - VideoReceiveStream* receive_stream = - call->CreateVideoReceiveStream(receive_config); - - receive_stream->Start(); - send_stream->Start(); - capturer->Start(); - - test::PressEnterToContinue(); - - capturer->Stop(); - send_stream->Stop(); - receive_stream->Stop(); - - call->DestroyVideoReceiveStream(receive_stream); - call->DestroyVideoSendStream(send_stream); - - delete decoder.decoder; - - send_transport.StopSending(); -} - -VideoEncoderConfig Loopback::CreateEncoderConfig() { - VideoEncoderConfig encoder_config; - encoder_config.streams = test::CreateVideoStreams(1); - VideoStream* stream = &encoder_config.streams[0]; - stream->width = config_.width; - stream->height = config_.height; - stream->min_bitrate_bps = static_cast(config_.min_bitrate_kbps) * 1000; - stream->max_bitrate_bps = static_cast(config_.max_bitrate_kbps) * 1000; - stream->target_bitrate_bps = - static_cast(config_.max_bitrate_kbps) * 1000; - stream->max_framerate = config_.fps; - stream->max_qp = 56; - if (config_.num_temporal_layers != 0) { - stream->temporal_layer_thresholds_bps.resize(config_.num_temporal_layers - - 1); - } - return encoder_config; -} - -test::VideoCapturer* Loopback::CreateCapturer(VideoSendStream* send_stream) { - return test::VideoCapturer::Create(send_stream->Input(), config_.width, - config_.height, config_.fps, clock_); -} - -} // namespace test -} // namespace webrtc diff --git a/webrtc/video/loopback.h b/webrtc/video/loopback.h deleted file mode 100644 index c671de4e98..0000000000 --- a/webrtc/video/loopback.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include - -#include "webrtc/config.h" - -namespace webrtc { - -class VideoSendStream; -class Clock; - -namespace test { - -class VideoCapturer; - -class Loopback { - public: - struct Config { - size_t width; - size_t height; - int32_t fps; - size_t min_bitrate_kbps; - size_t start_bitrate_kbps; - size_t max_bitrate_kbps; - int32_t min_transmit_bitrate_kbps; - std::string codec; - size_t num_temporal_layers; - size_t num_spatial_layers; - // Discard all temporal/spatial layers with id greater or equal the - // threshold. 0 to disable. - size_t tl_discard_threshold; - size_t sl_discard_threshold; - int32_t loss_percent; - int32_t link_capacity_kbps; - int32_t queue_size; - int32_t avg_propagation_delay_ms; - int32_t std_propagation_delay_ms; - bool logs; - }; - - explicit Loopback(const Config& config); - virtual ~Loopback(); - - void Run(); - - protected: - virtual VideoEncoderConfig CreateEncoderConfig(); - virtual VideoCapturer* CreateCapturer(VideoSendStream* send_stream); - - const Config config_; - Clock* const clock_; -}; - -} // namespace test -} // namespace webrtc diff --git a/webrtc/video/screenshare_loopback.cc b/webrtc/video/screenshare_loopback.cc index 2dfadd1d6c..34f68947f6 100644 --- a/webrtc/video/screenshare_loopback.cc +++ b/webrtc/video/screenshare_loopback.cc @@ -10,20 +10,12 @@ #include -#include - #include "gflags/gflags.h" #include "testing/gtest/include/gtest/gtest.h" -#include "webrtc/base/checks.h" #include "webrtc/test/field_trial.h" -#include "webrtc/test/frame_generator.h" -#include "webrtc/test/frame_generator_capturer.h" #include "webrtc/test/run_test.h" -#include "webrtc/test/testsupport/fileutils.h" -#include "webrtc/typedefs.h" -#include "webrtc/video/loopback.h" -#include "webrtc/video/video_send_stream.h" +#include "webrtc/video/video_quality_test.h" namespace webrtc { namespace flags { @@ -58,49 +50,41 @@ int ScrollDuration() { return static_cast(FLAGS_scroll_duration); } -DEFINE_int32(min_bitrate, 50, "Minimum video bitrate."); -size_t MinBitrate() { - return static_cast(FLAGS_min_bitrate); +DEFINE_int32(min_bitrate, 50, "Call and stream min bitrate in kbps."); +int MinBitrateKbps() { + return static_cast(FLAGS_min_bitrate); } -DEFINE_int32(tl0_bitrate, 200, "Temporal layer 0 target bitrate."); -size_t StartBitrate() { - return static_cast(FLAGS_tl0_bitrate); +DEFINE_int32(start_bitrate, 200, "Call start bitrate in kbps."); +int StartBitrateKbps() { + return static_cast(FLAGS_start_bitrate); } -DEFINE_int32(tl1_bitrate, 2000, "Temporal layer 1 target bitrate."); -size_t MaxBitrate() { - return static_cast(FLAGS_tl1_bitrate); +DEFINE_int32(target_bitrate, 2000, "Stream target bitrate in kbps."); +int TargetBitrateKbps() { + return static_cast(FLAGS_target_bitrate); +} + +DEFINE_int32(max_bitrate, 2000, "Call and stream max bitrate in kbps."); +int MaxBitrateKbps() { + return static_cast(FLAGS_max_bitrate); } DEFINE_int32(num_temporal_layers, 2, "Number of temporal layers to use."); -int NumTemporalLayers() { - return static_cast(FLAGS_num_temporal_layers); -} - -DEFINE_int32(num_spatial_layers, 1, "Number of spatial layers to use."); -int NumSpatialLayers() { - return static_cast(FLAGS_num_spatial_layers); +size_t NumTemporalLayers() { + return static_cast(FLAGS_num_temporal_layers); } DEFINE_int32( tl_discard_threshold, 0, "Discard TLs with id greater or equal the threshold. 0 to disable."); -int TLDiscardThreshold() { - return static_cast(FLAGS_tl_discard_threshold); -} - -DEFINE_int32( - sl_discard_threshold, - 0, - "Discard SLs with id greater or equal the threshold. 0 to disable."); -int SLDiscardThreshold() { - return static_cast(FLAGS_sl_discard_threshold); +size_t TLDiscardThreshold() { + return static_cast(FLAGS_tl_discard_threshold); } DEFINE_int32(min_transmit_bitrate, 400, "Min transmit bitrate incl. padding."); -int MinTransmitBitrate() { +int MinTransmitBitrateKbps() { return FLAGS_min_transmit_bitrate; } @@ -117,7 +101,7 @@ int LossPercent() { DEFINE_int32(link_capacity, 0, "Capacity (kbps) of the fake link. 0 means infinite."); -int LinkCapacity() { +int LinkCapacityKbps() { return static_cast(FLAGS_link_capacity); } @@ -142,6 +126,19 @@ int StdPropagationDelayMs() { DEFINE_bool(logs, false, "print logs to stderr"); +DEFINE_string( + output_filename, + "", + "Name of a target graph data file. If set, no preview will be shown."); +std::string OutputFilename() { + return static_cast(FLAGS_output_filename); +} + +DEFINE_int32(duration, 60, "Duration of the test in seconds."); +int DurationSecs() { + return static_cast(FLAGS_duration); +} + DEFINE_string( force_fieldtrials, "", @@ -151,115 +148,44 @@ DEFINE_string( "trials are separated by \"/\""); } // namespace flags -class ScreenshareLoopback : public test::Loopback { - public: - explicit ScreenshareLoopback(const Config& config) : Loopback(config) { - RTC_CHECK_GE(config.num_temporal_layers, 1u); - RTC_CHECK_LE(config.num_temporal_layers, 2u); - RTC_CHECK_GE(config.num_spatial_layers, 1u); - RTC_CHECK_LE(config.num_spatial_layers, 5u); - RTC_CHECK(config.num_spatial_layers == 1 || config.codec == "VP9"); - RTC_CHECK(config.num_spatial_layers == 1 || - config.num_temporal_layers == 1); - RTC_CHECK_LT(config.tl_discard_threshold, config.num_temporal_layers); - RTC_CHECK_LT(config.sl_discard_threshold, config.num_spatial_layers); - - vp8_settings_ = VideoEncoder::GetDefaultVp8Settings(); - vp8_settings_.denoisingOn = false; - vp8_settings_.frameDroppingOn = false; - vp8_settings_.numberOfTemporalLayers = - static_cast(config.num_temporal_layers); - - vp9_settings_ = VideoEncoder::GetDefaultVp9Settings(); - vp9_settings_.denoisingOn = false; - vp9_settings_.frameDroppingOn = false; - vp9_settings_.numberOfTemporalLayers = - static_cast(config.num_temporal_layers); - vp9_settings_.numberOfSpatialLayers = - static_cast(config.num_spatial_layers); - } - virtual ~ScreenshareLoopback() {} - - protected: - VideoEncoderConfig CreateEncoderConfig() override { - VideoEncoderConfig encoder_config(test::Loopback::CreateEncoderConfig()); - VideoStream* stream = &encoder_config.streams[0]; - encoder_config.content_type = VideoEncoderConfig::ContentType::kScreen; - encoder_config.min_transmit_bitrate_bps = flags::MinTransmitBitrate(); - int num_temporal_layers; - if (config_.codec == "VP8") { - encoder_config.encoder_specific_settings = &vp8_settings_; - num_temporal_layers = vp8_settings_.numberOfTemporalLayers; - } else if (config_.codec == "VP9") { - encoder_config.encoder_specific_settings = &vp9_settings_; - num_temporal_layers = vp9_settings_.numberOfTemporalLayers; - } else { - RTC_NOTREACHED() << "Codec not supported!"; - abort(); - } - stream->temporal_layer_thresholds_bps.clear(); - stream->target_bitrate_bps = - static_cast(config_.start_bitrate_kbps) * 1000; - if (num_temporal_layers == 2) { - stream->temporal_layer_thresholds_bps.push_back( - stream->target_bitrate_bps); - } - return encoder_config; - } - - test::VideoCapturer* CreateCapturer(VideoSendStream* send_stream) override { - std::vector slides; - slides.push_back(test::ResourcePath("web_screenshot_1850_1110", "yuv")); - slides.push_back(test::ResourcePath("presentation_1850_1110", "yuv")); - slides.push_back(test::ResourcePath("photo_1850_1110", "yuv")); - slides.push_back(test::ResourcePath("difficult_photo_1850_1110", "yuv")); - - // Fixed for input resolution for prerecorded screenshare content. - const size_t kWidth = 1850; - const size_t kHeight = 1110; - RTC_CHECK_LE(flags::Width(), kWidth); - RTC_CHECK_LE(flags::Height(), kHeight); - RTC_CHECK_GT(flags::SlideChangeInterval(), 0); - const int kPauseDurationMs = - (flags::SlideChangeInterval() - flags::ScrollDuration()) * 1000; - RTC_CHECK_LE(flags::ScrollDuration(), flags::SlideChangeInterval()); - - test::FrameGenerator* frame_generator = - test::FrameGenerator::CreateScrollingInputFromYuvFiles( - Clock::GetRealTimeClock(), slides, kWidth, kHeight, flags::Width(), - flags::Height(), flags::ScrollDuration() * 1000, kPauseDurationMs); - - test::FrameGeneratorCapturer* capturer(new test::FrameGeneratorCapturer( - clock_, send_stream->Input(), frame_generator, flags::Fps())); - EXPECT_TRUE(capturer->Init()); - return capturer; - } - - VideoCodecVP8 vp8_settings_; - VideoCodecVP9 vp9_settings_; -}; - void Loopback() { - test::Loopback::Config config{flags::Width(), - flags::Height(), - flags::Fps(), - flags::MinBitrate(), - flags::StartBitrate(), - flags::MaxBitrate(), - flags::MinTransmitBitrate(), - flags::Codec(), - flags::NumTemporalLayers(), - flags::NumSpatialLayers(), - flags::TLDiscardThreshold(), - flags::SLDiscardThreshold(), - flags::LossPercent(), - flags::LinkCapacity(), - flags::QueueSize(), - flags::AvgPropagationDelayMs(), - flags::StdPropagationDelayMs(), - flags::FLAGS_logs}; - ScreenshareLoopback loopback(config); - loopback.Run(); + FakeNetworkPipe::Config pipe_config; + pipe_config.loss_percent = flags::LossPercent(); + pipe_config.link_capacity_kbps = flags::LinkCapacityKbps(); + pipe_config.queue_length_packets = flags::QueueSize(); + pipe_config.queue_delay_ms = flags::AvgPropagationDelayMs(); + pipe_config.delay_standard_deviation_ms = flags::StdPropagationDelayMs(); + + Call::Config::BitrateConfig call_bitrate_config; + call_bitrate_config.min_bitrate_bps = flags::MinBitrateKbps() * 1000; + call_bitrate_config.start_bitrate_bps = flags::StartBitrateKbps() * 1000; + call_bitrate_config.max_bitrate_bps = flags::MaxBitrateKbps() * 1000; + + VideoQualityTest::Params params{ + { + flags::Width(), + flags::Height(), + flags::Fps(), + flags::MinBitrateKbps() * 1000, + flags::TargetBitrateKbps() * 1000, + flags::MaxBitrateKbps() * 1000, + flags::Codec(), + flags::NumTemporalLayers(), + flags::MinTransmitBitrateKbps() * 1000, + call_bitrate_config, + flags::TLDiscardThreshold() + }, + {}, // Video specific. + {true, flags::SlideChangeInterval(), flags::ScrollDuration()}, + {"screenshare", 0.0, 0.0, flags::DurationSecs(), flags::OutputFilename()}, + pipe_config, + flags::FLAGS_logs}; + + VideoQualityTest test; + if (flags::OutputFilename().empty()) + test.RunWithVideoRenderer(params); + else + test.RunWithAnalyzer(params); } } // namespace webrtc diff --git a/webrtc/video/video_loopback.cc b/webrtc/video/video_loopback.cc index 789b5cceac..c5e9254c08 100644 --- a/webrtc/video/video_loopback.cc +++ b/webrtc/video/video_loopback.cc @@ -18,7 +18,7 @@ #include "webrtc/test/field_trial.h" #include "webrtc/test/run_test.h" #include "webrtc/typedefs.h" -#include "webrtc/video/loopback.h" +#include "webrtc/video/video_quality_test.h" namespace webrtc { @@ -39,24 +39,25 @@ int Fps() { return static_cast(FLAGS_fps); } -DEFINE_int32(min_bitrate, 50, "Minimum video bitrate."); -size_t MinBitrate() { - return static_cast(FLAGS_min_bitrate); +DEFINE_int32(min_bitrate, 50, "Call and stream min bitrate in kbps."); +int MinBitrateKbps() { + return static_cast(FLAGS_min_bitrate); } -DEFINE_int32(start_bitrate, 300, "Video starting bitrate."); -size_t StartBitrate() { - return static_cast(FLAGS_start_bitrate); +DEFINE_int32(start_bitrate, 300, "Call start bitrate in kbps."); +int StartBitrateKbps() { + return static_cast(FLAGS_start_bitrate); } -DEFINE_int32(max_bitrate, 800, "Maximum video bitrate."); -size_t MaxBitrate() { - return static_cast(FLAGS_max_bitrate); +DEFINE_int32(target_bitrate, 800, "Stream target bitrate in kbps."); +int TargetBitrateKbps() { + return static_cast(FLAGS_target_bitrate); } -int MinTransmitBitrate() { - return 0; -} // No min padding for regular video. +DEFINE_int32(max_bitrate, 800, "Call and stream max bitrate in kbps."); +int MaxBitrateKbps() { + return static_cast(FLAGS_max_bitrate); +} DEFINE_string(codec, "VP8", "Video codec to use."); std::string Codec() { @@ -71,7 +72,7 @@ int LossPercent() { DEFINE_int32(link_capacity, 0, "Capacity (kbps) of the fake link. 0 means infinite."); -int LinkCapacity() { +int LinkCapacityKbps() { return static_cast(FLAGS_link_capacity); } @@ -105,33 +106,82 @@ DEFINE_string( "trials are separated by \"/\""); DEFINE_int32(num_temporal_layers, - 0, + 1, "Number of temporal layers. Set to 1-4 to override."); - size_t NumTemporalLayers() { return static_cast(FLAGS_num_temporal_layers); } +DEFINE_int32( + tl_discard_threshold, + 0, + "Discard TLs with id greater or equal the threshold. 0 to disable."); +size_t TLDiscardThreshold() { + return static_cast(FLAGS_tl_discard_threshold); +} + +DEFINE_string(clip, + "", + "Name of the clip to show. If empty, using chroma generator."); +std::string Clip() { + return static_cast(FLAGS_clip); +} + +DEFINE_string( + output_filename, + "", + "Name of a target graph data file. If set, no preview will be shown."); +std::string OutputFilename() { + return static_cast(FLAGS_output_filename); +} + +DEFINE_int32(duration, 60, "Duration of the test in seconds."); +int DurationSecs() { + return static_cast(FLAGS_duration); +} + } // namespace flags void Loopback() { - test::Loopback::Config config{flags::Width(), - flags::Height(), - flags::Fps(), - flags::MinBitrate(), - flags::StartBitrate(), - flags::MaxBitrate(), - 0, // No min transmit bitrate. - flags::Codec(), - flags::NumTemporalLayers(), - flags::LossPercent(), - flags::LinkCapacity(), - flags::QueueSize(), - flags::AvgPropagationDelayMs(), - flags::StdPropagationDelayMs(), - flags::FLAGS_logs}; - test::Loopback loopback(config); - loopback.Run(); + FakeNetworkPipe::Config pipe_config; + pipe_config.loss_percent = flags::LossPercent(); + pipe_config.link_capacity_kbps = flags::LinkCapacityKbps(); + pipe_config.queue_length_packets = flags::QueueSize(); + pipe_config.queue_delay_ms = flags::AvgPropagationDelayMs(); + pipe_config.delay_standard_deviation_ms = flags::StdPropagationDelayMs(); + + Call::Config::BitrateConfig call_bitrate_config; + call_bitrate_config.min_bitrate_bps = flags::MinBitrateKbps() * 1000; + call_bitrate_config.start_bitrate_bps = flags::StartBitrateKbps() * 1000; + call_bitrate_config.max_bitrate_bps = flags::MaxBitrateKbps() * 1000; + + std::string clip = flags::Clip(); + std::string graph_title = clip.empty() ? "" : "video " + clip; + VideoQualityTest::Params params{ + { + flags::Width(), + flags::Height(), + flags::Fps(), + flags::MinBitrateKbps() * 1000, + flags::TargetBitrateKbps() * 1000, + flags::MaxBitrateKbps() * 1000, + flags::Codec(), + flags::NumTemporalLayers(), + 0, // No min transmit bitrate. + call_bitrate_config, + flags::TLDiscardThreshold() + }, + {clip}, + {}, // Screenshare specific. + {graph_title, 0.0, 0.0, flags::DurationSecs(), flags::OutputFilename()}, + pipe_config, + flags::FLAGS_logs}; + + VideoQualityTest test; + if (flags::OutputFilename().empty()) + test.RunWithVideoRenderer(params); + else + test.RunWithAnalyzer(params); } } // namespace webrtc diff --git a/webrtc/video/video_quality_test.cc b/webrtc/video/video_quality_test.cc new file mode 100644 index 0000000000..d440858155 --- /dev/null +++ b/webrtc/video/video_quality_test.cc @@ -0,0 +1,816 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include + +#include +#include +#include +#include + +#include "testing/gtest/include/gtest/gtest.h" + +#include "webrtc/base/checks.h" +#include "webrtc/base/format_macros.h" +#include "webrtc/base/scoped_ptr.h" +#include "webrtc/call.h" +#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" +#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" +#include "webrtc/system_wrappers/interface/cpu_info.h" +#include "webrtc/test/layer_filtering_transport.h" +#include "webrtc/test/run_loop.h" +#include "webrtc/test/statistics.h" +#include "webrtc/test/testsupport/fileutils.h" +#include "webrtc/test/video_renderer.h" +#include "webrtc/video/video_quality_test.h" + +namespace webrtc { + +static const int kSendStatsPollingIntervalMs = 1000; +static const int kPayloadTypeVP8 = 123; +static const int kPayloadTypeVP9 = 124; + +class VideoAnalyzer : public PacketReceiver, + public newapi::Transport, + public VideoRenderer, + public VideoCaptureInput, + public EncodedFrameObserver { + public: + VideoAnalyzer(VideoCaptureInput* input, + Transport* transport, + const std::string& test_label, + double avg_psnr_threshold, + double avg_ssim_threshold, + int duration_frames, + FILE* graph_data_output_file) + : input_(input), + transport_(transport), + receiver_(nullptr), + send_stream_(nullptr), + test_label_(test_label), + graph_data_output_file_(graph_data_output_file), + frames_to_process_(duration_frames), + frames_recorded_(0), + frames_processed_(0), + dropped_frames_(0), + last_render_time_(0), + rtp_timestamp_delta_(0), + avg_psnr_threshold_(avg_psnr_threshold), + avg_ssim_threshold_(avg_ssim_threshold), + comparison_available_event_(EventWrapper::Create()), + done_(EventWrapper::Create()) { + // Create thread pool for CPU-expensive PSNR/SSIM calculations. + + // Try to use about as many threads as cores, but leave kMinCoresLeft alone, + // so that we don't accidentally starve "real" worker threads (codec etc). + // Also, don't allocate more than kMaxComparisonThreads, even if there are + // spare cores. + + uint32_t num_cores = CpuInfo::DetectNumberOfCores(); + RTC_DCHECK_GE(num_cores, 1u); + static const uint32_t kMinCoresLeft = 4; + static const uint32_t kMaxComparisonThreads = 8; + + if (num_cores <= kMinCoresLeft) { + num_cores = 1; + } else { + num_cores -= kMinCoresLeft; + num_cores = std::min(num_cores, kMaxComparisonThreads); + } + + for (uint32_t i = 0; i < num_cores; ++i) { + rtc::scoped_ptr thread = + ThreadWrapper::CreateThread(&FrameComparisonThread, this, "Analyzer"); + EXPECT_TRUE(thread->Start()); + comparison_thread_pool_.push_back(thread.release()); + } + + stats_polling_thread_ = + ThreadWrapper::CreateThread(&PollStatsThread, this, "StatsPoller"); + EXPECT_TRUE(stats_polling_thread_->Start()); + } + + ~VideoAnalyzer() { + for (ThreadWrapper* thread : comparison_thread_pool_) { + EXPECT_TRUE(thread->Stop()); + delete thread; + } + } + + virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; } + + DeliveryStatus DeliverPacket(MediaType media_type, + const uint8_t* packet, + size_t length, + const PacketTime& packet_time) override { + rtc::scoped_ptr parser(RtpHeaderParser::Create()); + RTPHeader header; + parser->Parse(packet, length, &header); + { + rtc::CritScope lock(&crit_); + recv_times_[header.timestamp - rtp_timestamp_delta_] = + Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); + } + + return receiver_->DeliverPacket(media_type, packet, length, packet_time); + } + + void IncomingCapturedFrame(const VideoFrame& video_frame) override { + VideoFrame copy = video_frame; + copy.set_timestamp(copy.ntp_time_ms() * 90); + + { + rtc::CritScope lock(&crit_); + if (first_send_frame_.IsZeroSize() && rtp_timestamp_delta_ == 0) + first_send_frame_ = copy; + + frames_.push_back(copy); + } + + input_->IncomingCapturedFrame(video_frame); + } + + bool SendRtp(const uint8_t* packet, size_t length) override { + rtc::scoped_ptr parser(RtpHeaderParser::Create()); + RTPHeader header; + parser->Parse(packet, length, &header); + + { + rtc::CritScope lock(&crit_); + if (rtp_timestamp_delta_ == 0) { + rtp_timestamp_delta_ = header.timestamp - first_send_frame_.timestamp(); + first_send_frame_.Reset(); + } + uint32_t timestamp = header.timestamp - rtp_timestamp_delta_; + send_times_[timestamp] = + Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); + encoded_frame_sizes_[timestamp] += + length - (header.headerLength + header.paddingLength); + } + + return transport_->SendRtp(packet, length); + } + + bool SendRtcp(const uint8_t* packet, size_t length) override { + return transport_->SendRtcp(packet, length); + } + + void EncodedFrameCallback(const EncodedFrame& frame) override { + rtc::CritScope lock(&comparison_lock_); + if (frames_recorded_ < frames_to_process_) + encoded_frame_size_.AddSample(frame.length_); + } + + void RenderFrame(const VideoFrame& video_frame, + int time_to_render_ms) override { + int64_t render_time_ms = + Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); + uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_; + + rtc::CritScope lock(&crit_); + + while (frames_.front().timestamp() < send_timestamp) { + AddFrameComparison(frames_.front(), last_rendered_frame_, true, + render_time_ms); + frames_.pop_front(); + } + + VideoFrame reference_frame = frames_.front(); + frames_.pop_front(); + assert(!reference_frame.IsZeroSize()); + EXPECT_EQ(reference_frame.timestamp(), send_timestamp); + assert(reference_frame.timestamp() == send_timestamp); + + AddFrameComparison(reference_frame, video_frame, false, render_time_ms); + + last_rendered_frame_ = video_frame; + } + + bool IsTextureSupported() const override { return false; } + + void Wait() { + // Frame comparisons can be very expensive. Wait for test to be done, but + // at time-out check if frames_processed is going up. If so, give it more + // time, otherwise fail. Hopefully this will reduce test flakiness. + + int last_frames_processed = -1; + EventTypeWrapper eventType; + int iteration = 0; + while ((eventType = done_->Wait(VideoQualityTest::kDefaultTimeoutMs)) != + kEventSignaled) { + int frames_processed; + { + rtc::CritScope crit(&comparison_lock_); + frames_processed = frames_processed_; + } + + // Print some output so test infrastructure won't think we've crashed. + const char* kKeepAliveMessages[3] = { + "Uh, I'm-I'm not quite dead, sir.", + "Uh, I-I think uh, I could pull through, sir.", + "Actually, I think I'm all right to come with you--"}; + printf("- %s\n", kKeepAliveMessages[iteration++ % 3]); + + if (last_frames_processed == -1) { + last_frames_processed = frames_processed; + continue; + } + ASSERT_GT(frames_processed, last_frames_processed) + << "Analyzer stalled while waiting for test to finish."; + last_frames_processed = frames_processed; + } + + if (iteration > 0) + printf("- Farewell, sweet Concorde!\n"); + + // Signal stats polling thread if that is still waiting and stop it now, + // since it uses the send_stream_ reference that might be reclaimed after + // returning from this method. + done_->Set(); + EXPECT_TRUE(stats_polling_thread_->Stop()); + } + + VideoCaptureInput* input_; + Transport* transport_; + PacketReceiver* receiver_; + VideoSendStream* send_stream_; + + private: + struct FrameComparison { + FrameComparison() + : dropped(false), + send_time_ms(0), + recv_time_ms(0), + render_time_ms(0), + encoded_frame_size(0) {} + + FrameComparison(const VideoFrame& reference, + const VideoFrame& render, + bool dropped, + int64_t send_time_ms, + int64_t recv_time_ms, + int64_t render_time_ms, + size_t encoded_frame_size) + : reference(reference), + render(render), + dropped(dropped), + send_time_ms(send_time_ms), + recv_time_ms(recv_time_ms), + render_time_ms(render_time_ms), + encoded_frame_size(encoded_frame_size) {} + + VideoFrame reference; + VideoFrame render; + bool dropped; + int64_t send_time_ms; + int64_t recv_time_ms; + int64_t render_time_ms; + size_t encoded_frame_size; + }; + + struct Sample { + Sample(double dropped, + double input_time_ms, + double send_time_ms, + double recv_time_ms, + double encoded_frame_size, + double psnr, + double ssim, + double render_time_ms) + : dropped(dropped), + input_time_ms(input_time_ms), + send_time_ms(send_time_ms), + recv_time_ms(recv_time_ms), + encoded_frame_size(encoded_frame_size), + psnr(psnr), + ssim(ssim), + render_time_ms(render_time_ms) {} + + double dropped; + double input_time_ms; + double send_time_ms; + double recv_time_ms; + double encoded_frame_size; + double psnr; + double ssim; + double render_time_ms; + }; + + void AddFrameComparison(const VideoFrame& reference, + const VideoFrame& render, + bool dropped, + int64_t render_time_ms) + EXCLUSIVE_LOCKS_REQUIRED(crit_) { + int64_t send_time_ms = send_times_[reference.timestamp()]; + send_times_.erase(reference.timestamp()); + int64_t recv_time_ms = recv_times_[reference.timestamp()]; + recv_times_.erase(reference.timestamp()); + + size_t encoded_size = encoded_frame_sizes_[reference.timestamp()]; + encoded_frame_sizes_.erase(reference.timestamp()); + + VideoFrame reference_copy; + VideoFrame render_copy; + reference_copy.CopyFrame(reference); + render_copy.CopyFrame(render); + + rtc::CritScope crit(&comparison_lock_); + comparisons_.push_back(FrameComparison(reference_copy, render_copy, dropped, + send_time_ms, recv_time_ms, + render_time_ms, encoded_size)); + comparison_available_event_->Set(); + } + + static bool PollStatsThread(void* obj) { + return static_cast(obj)->PollStats(); + } + + bool PollStats() { + switch (done_->Wait(kSendStatsPollingIntervalMs)) { + case kEventSignaled: + case kEventError: + done_->Set(); // Make sure main thread is also signaled. + return false; + case kEventTimeout: + break; + default: + RTC_NOTREACHED(); + } + + VideoSendStream::Stats stats = send_stream_->GetStats(); + + rtc::CritScope crit(&comparison_lock_); + encode_frame_rate_.AddSample(stats.encode_frame_rate); + encode_time_ms.AddSample(stats.avg_encode_time_ms); + encode_usage_percent.AddSample(stats.encode_usage_percent); + media_bitrate_bps.AddSample(stats.media_bitrate_bps); + + return true; + } + + static bool FrameComparisonThread(void* obj) { + return static_cast(obj)->CompareFrames(); + } + + bool CompareFrames() { + if (AllFramesRecorded()) + return false; + + VideoFrame reference; + VideoFrame render; + FrameComparison comparison; + + if (!PopComparison(&comparison)) { + // Wait until new comparison task is available, or test is done. + // If done, wake up remaining threads waiting. + comparison_available_event_->Wait(1000); + if (AllFramesRecorded()) { + comparison_available_event_->Set(); + return false; + } + return true; // Try again. + } + + PerformFrameComparison(comparison); + + if (FrameProcessed()) { + PrintResults(); + if (graph_data_output_file_) + PrintSamplesToFile(); + done_->Set(); + comparison_available_event_->Set(); + return false; + } + + return true; + } + + bool PopComparison(FrameComparison* comparison) { + rtc::CritScope crit(&comparison_lock_); + // If AllFramesRecorded() is true, it means we have already popped + // frames_to_process_ frames from comparisons_, so there is no more work + // for this thread to be done. frames_processed_ might still be lower if + // all comparisons are not done, but those frames are currently being + // worked on by other threads. + if (comparisons_.empty() || AllFramesRecorded()) + return false; + + *comparison = comparisons_.front(); + comparisons_.pop_front(); + + FrameRecorded(); + return true; + } + + // Increment counter for number of frames received for comparison. + void FrameRecorded() { + rtc::CritScope crit(&comparison_lock_); + ++frames_recorded_; + } + + // Returns true if all frames to be compared have been taken from the queue. + bool AllFramesRecorded() { + rtc::CritScope crit(&comparison_lock_); + assert(frames_recorded_ <= frames_to_process_); + return frames_recorded_ == frames_to_process_; + } + + // Increase count of number of frames processed. Returns true if this was the + // last frame to be processed. + bool FrameProcessed() { + rtc::CritScope crit(&comparison_lock_); + ++frames_processed_; + assert(frames_processed_ <= frames_to_process_); + return frames_processed_ == frames_to_process_; + } + + void PrintResults() { + rtc::CritScope crit(&comparison_lock_); + PrintResult("psnr", psnr_, " dB"); + PrintResult("ssim", ssim_, ""); + PrintResult("sender_time", sender_time_, " ms"); + printf("RESULT dropped_frames: %s = %d frames\n", test_label_.c_str(), + dropped_frames_); + PrintResult("receiver_time", receiver_time_, " ms"); + PrintResult("total_delay_incl_network", end_to_end_, " ms"); + PrintResult("time_between_rendered_frames", rendered_delta_, " ms"); + PrintResult("encoded_frame_size", encoded_frame_size_, " bytes"); + PrintResult("encode_frame_rate", encode_frame_rate_, " fps"); + PrintResult("encode_time", encode_time_ms, " ms"); + PrintResult("encode_usage_percent", encode_usage_percent, " percent"); + PrintResult("media_bitrate", media_bitrate_bps, " bps"); + + EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_); + EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_); + } + + void PerformFrameComparison(const FrameComparison& comparison) { + // Perform expensive psnr and ssim calculations while not holding lock. + double psnr = I420PSNR(&comparison.reference, &comparison.render); + double ssim = I420SSIM(&comparison.reference, &comparison.render); + + int64_t input_time_ms = comparison.reference.ntp_time_ms(); + + rtc::CritScope crit(&comparison_lock_); + if (graph_data_output_file_) { + samples_.push_back( + Sample(comparison.dropped, input_time_ms, comparison.send_time_ms, + comparison.recv_time_ms, comparison.encoded_frame_size, psnr, + ssim, comparison.render_time_ms)); + } + psnr_.AddSample(psnr); + ssim_.AddSample(ssim); + + if (comparison.dropped) { + ++dropped_frames_; + return; + } + if (last_render_time_ != 0) + rendered_delta_.AddSample(comparison.render_time_ms - last_render_time_); + last_render_time_ = comparison.render_time_ms; + + sender_time_.AddSample(comparison.send_time_ms - input_time_ms); + receiver_time_.AddSample(comparison.render_time_ms - + comparison.recv_time_ms); + end_to_end_.AddSample(comparison.render_time_ms - input_time_ms); + encoded_frame_size_.AddSample(comparison.encoded_frame_size); + } + + void PrintResult(const char* result_type, + test::Statistics stats, + const char* unit) { + printf("RESULT %s: %s = {%f, %f}%s\n", + result_type, + test_label_.c_str(), + stats.Mean(), + stats.StandardDeviation(), + unit); + } + + void PrintSamplesToFile(void) { + FILE* out = graph_data_output_file_; + rtc::CritScope crit(&comparison_lock_); + std::sort(samples_.begin(), samples_.end(), + [](const Sample& A, const Sample& B) -> bool { + return A.input_time_ms < B.input_time_ms; + }); + + fprintf(out, "%s\n", test_label_.c_str()); + fprintf(out, "%" PRIuS "\n", samples_.size()); + fprintf(out, + "dropped " + "input_time_ms " + "send_time_ms " + "recv_time_ms " + "encoded_frame_size " + "psnr " + "ssim " + "render_time_ms\n"); + for (const Sample& sample : samples_) { + fprintf(out, "%lf %lf %lf %lf %lf %lf %lf %lf\n", sample.dropped, + sample.input_time_ms, sample.send_time_ms, sample.recv_time_ms, + sample.encoded_frame_size, sample.psnr, sample.ssim, + sample.render_time_ms); + } + } + + const std::string test_label_; + FILE* const graph_data_output_file_; + std::vector samples_ GUARDED_BY(comparison_lock_); + test::Statistics sender_time_ GUARDED_BY(comparison_lock_); + test::Statistics receiver_time_ GUARDED_BY(comparison_lock_); + test::Statistics psnr_ GUARDED_BY(comparison_lock_); + test::Statistics ssim_ GUARDED_BY(comparison_lock_); + test::Statistics end_to_end_ GUARDED_BY(comparison_lock_); + test::Statistics rendered_delta_ GUARDED_BY(comparison_lock_); + test::Statistics encoded_frame_size_ GUARDED_BY(comparison_lock_); + test::Statistics encode_frame_rate_ GUARDED_BY(comparison_lock_); + test::Statistics encode_time_ms GUARDED_BY(comparison_lock_); + test::Statistics encode_usage_percent GUARDED_BY(comparison_lock_); + test::Statistics media_bitrate_bps GUARDED_BY(comparison_lock_); + + const int frames_to_process_; + int frames_recorded_; + int frames_processed_; + int dropped_frames_; + int64_t last_render_time_; + uint32_t rtp_timestamp_delta_; + + rtc::CriticalSection crit_; + std::deque frames_ GUARDED_BY(crit_); + VideoFrame last_rendered_frame_ GUARDED_BY(crit_); + std::map send_times_ GUARDED_BY(crit_); + std::map recv_times_ GUARDED_BY(crit_); + std::map encoded_frame_sizes_ GUARDED_BY(crit_); + VideoFrame first_send_frame_ GUARDED_BY(crit_); + const double avg_psnr_threshold_; + const double avg_ssim_threshold_; + + rtc::CriticalSection comparison_lock_; + std::vector comparison_thread_pool_; + rtc::scoped_ptr stats_polling_thread_; + const rtc::scoped_ptr comparison_available_event_; + std::deque comparisons_ GUARDED_BY(comparison_lock_); + const rtc::scoped_ptr done_; +}; + +VideoQualityTest::VideoQualityTest() : clock_(Clock::GetRealTimeClock()) {} + +void VideoQualityTest::ValidateParams(const Params& params) { + RTC_CHECK_GE(params.common.max_bitrate_bps, params.common.target_bitrate_bps); + RTC_CHECK_GE(params.common.target_bitrate_bps, params.common.min_bitrate_bps); + RTC_CHECK_LT(params.common.tl_discard_threshold, + params.common.num_temporal_layers); +} + +void VideoQualityTest::TestBody() {} + +void VideoQualityTest::SetupFullStack(const Params& params, + newapi::Transport* send_transport, + newapi::Transport* recv_transport) { + if (params.logs) + trace_to_stderr_.reset(new test::TraceToStderr); + + CreateSendConfig(1, send_transport); + + int payload_type; + if (params.common.codec == "VP8") { + encoder_.reset(VideoEncoder::Create(VideoEncoder::kVp8)); + payload_type = kPayloadTypeVP8; + } else if (params.common.codec == "VP9") { + encoder_.reset(VideoEncoder::Create(VideoEncoder::kVp9)); + payload_type = kPayloadTypeVP9; + } else { + RTC_NOTREACHED() << "Codec not supported!"; + return; + } + send_config_.encoder_settings.encoder = encoder_.get(); + send_config_.encoder_settings.payload_name = params.common.codec; + send_config_.encoder_settings.payload_type = payload_type; + + send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs; + send_config_.rtp.rtx.ssrcs.push_back(kSendRtxSsrcs[0]); + send_config_.rtp.rtx.payload_type = kSendRtxPayloadType; + + // Automatically fill out streams[0] with params. + VideoStream* stream = &encoder_config_.streams[0]; + stream->width = params.common.width; + stream->height = params.common.height; + stream->min_bitrate_bps = params.common.min_bitrate_bps; + stream->target_bitrate_bps = params.common.target_bitrate_bps; + stream->max_bitrate_bps = params.common.max_bitrate_bps; + stream->max_framerate = static_cast(params.common.fps); + + stream->temporal_layer_thresholds_bps.clear(); + if (params.common.num_temporal_layers > 1) { + stream->temporal_layer_thresholds_bps.push_back(stream->target_bitrate_bps); + } + + CreateMatchingReceiveConfigs(recv_transport); + + receive_configs_[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs; + receive_configs_[0].rtp.rtx[kSendRtxPayloadType].ssrc = kSendRtxSsrcs[0]; + receive_configs_[0].rtp.rtx[kSendRtxPayloadType].payload_type = + kSendRtxPayloadType; + + encoder_config_.min_transmit_bitrate_bps = params.common.min_transmit_bps; +} + +void VideoQualityTest::SetupScreenshare(const Params& params) { + RTC_CHECK(params.screenshare.enabled); + + // Fill out codec settings. + encoder_config_.content_type = VideoEncoderConfig::ContentType::kScreen; + if (params.common.codec == "VP8") { + codec_settings_.VP8 = VideoEncoder::GetDefaultVp8Settings(); + codec_settings_.VP8.denoisingOn = false; + codec_settings_.VP8.frameDroppingOn = false; + codec_settings_.VP8.numberOfTemporalLayers = + static_cast(params.common.num_temporal_layers); + encoder_config_.encoder_specific_settings = &codec_settings_.VP8; + } else if (params.common.codec == "VP9") { + codec_settings_.VP9 = VideoEncoder::GetDefaultVp9Settings(); + codec_settings_.VP9.denoisingOn = false; + codec_settings_.VP9.frameDroppingOn = false; + codec_settings_.VP9.numberOfTemporalLayers = + static_cast(params.common.num_temporal_layers); + encoder_config_.encoder_specific_settings = &codec_settings_.VP9; + } + + // Setup frame generator. + const size_t kWidth = 1850; + const size_t kHeight = 1110; + std::vector slides; + slides.push_back(test::ResourcePath("web_screenshot_1850_1110", "yuv")); + slides.push_back(test::ResourcePath("presentation_1850_1110", "yuv")); + slides.push_back(test::ResourcePath("photo_1850_1110", "yuv")); + slides.push_back(test::ResourcePath("difficult_photo_1850_1110", "yuv")); + + if (params.screenshare.scroll_duration == 0) { + // Cycle image every slide_change_interval seconds. + frame_generator_.reset(test::FrameGenerator::CreateFromYuvFile( + slides, kWidth, kHeight, + params.screenshare.slide_change_interval * params.common.fps)); + } else { + RTC_CHECK_LE(params.common.width, kWidth); + RTC_CHECK_LE(params.common.height, kHeight); + RTC_CHECK_GT(params.screenshare.slide_change_interval, 0); + const int kPauseDurationMs = (params.screenshare.slide_change_interval - + params.screenshare.scroll_duration) * 1000; + RTC_CHECK_LE(params.screenshare.scroll_duration, + params.screenshare.slide_change_interval); + + frame_generator_.reset( + test::FrameGenerator::CreateScrollingInputFromYuvFiles( + clock_, slides, kWidth, kHeight, params.common.width, + params.common.height, params.screenshare.scroll_duration * 1000, + kPauseDurationMs)); + } +} + +void VideoQualityTest::CreateCapturer(const Params& params, + VideoCaptureInput* input) { + if (params.screenshare.enabled) { + frame_generator_capturer_.reset(new test::FrameGeneratorCapturer( + clock_, input, frame_generator_.release(), params.common.fps)); + EXPECT_TRUE(frame_generator_capturer_->Init()); + } else { + if (params.video.clip_name.empty()) { + frame_generator_capturer_.reset(test::FrameGeneratorCapturer::Create( + input, params.common.width, params.common.height, params.common.fps, + clock_)); + EXPECT_TRUE(frame_generator_capturer_->Init()); + } else { + frame_generator_capturer_.reset( + test::FrameGeneratorCapturer::CreateFromYuvFile( + input, test::ResourcePath(params.video.clip_name, "yuv"), + params.common.width, params.common.height, params.common.fps, + clock_)); + ASSERT_TRUE(frame_generator_capturer_.get() != nullptr) + << "Could not create capturer for " << params.video.clip_name + << ".yuv. Is this resource file present?"; + } + } +} + +void VideoQualityTest::RunWithAnalyzer(const Params& params) { + // TODO(ivica): Merge with RunWithRenderer and use a flag / argument to + // differentiate between the analyzer and the renderer case. + ValidateParams(params); + + FILE* graph_data_output_file = nullptr; + if (!params.analyzer.graph_data_output_filename.empty()) { + graph_data_output_file = + fopen(params.analyzer.graph_data_output_filename.c_str(), "w"); + RTC_CHECK(graph_data_output_file != nullptr) + << "Can't open the file " + << params.analyzer.graph_data_output_filename << "!"; + } + + test::LayerFilteringTransport send_transport( + params.pipe, kPayloadTypeVP8, kPayloadTypeVP9, + static_cast(params.common.tl_discard_threshold), 0); + test::DirectTransport recv_transport(params.pipe); + VideoAnalyzer analyzer( + nullptr, &send_transport, params.analyzer.test_label, + params.analyzer.avg_psnr_threshold, params.analyzer.avg_ssim_threshold, + params.analyzer.test_durations_secs * params.common.fps, + graph_data_output_file); + + Call::Config call_config; + call_config.bitrate_config = params.common.call_bitrate_config; + CreateCalls(call_config, call_config); + + analyzer.SetReceiver(receiver_call_->Receiver()); + send_transport.SetReceiver(&analyzer); + recv_transport.SetReceiver(sender_call_->Receiver()); + + SetupFullStack(params, &analyzer, &recv_transport); + receive_configs_[0].renderer = &analyzer; + for (auto& config : receive_configs_) + config.pre_decode_callback = &analyzer; + + if (params.screenshare.enabled) + SetupScreenshare(params); + + CreateCapturer(params, &analyzer); + + CreateStreams(); + analyzer.input_ = send_stream_->Input(); + analyzer.send_stream_ = send_stream_; + + Start(); + + analyzer.Wait(); + + send_transport.StopSending(); + recv_transport.StopSending(); + + Stop(); + + DestroyStreams(); + + if (graph_data_output_file) + fclose(graph_data_output_file); +} + +void VideoQualityTest::RunWithVideoRenderer(const Params& params) { + ValidateParams(params); + + rtc::scoped_ptr local_preview( + test::VideoRenderer::Create("Local Preview", params.common.width, + params.common.height)); + rtc::scoped_ptr loopback_video( + test::VideoRenderer::Create("Loopback Video", params.common.width, + params.common.height)); + + // TODO(ivica): Remove bitrate_config and use the default Call::Config(), to + // match the full stack tests. + Call::Config call_config; + call_config.bitrate_config = params.common.call_bitrate_config; + rtc::scoped_ptr call(Call::Create(call_config)); + + test::LayerFilteringTransport transport( + params.pipe, kPayloadTypeVP8, kPayloadTypeVP9, + static_cast(params.common.tl_discard_threshold), 0); + // TODO(ivica): Use two calls to be able to merge with RunWithAnalyzer or at + // least share as much code as possible. That way this test would also match + // the full stack tests better. + transport.SetReceiver(call->Receiver()); + + SetupFullStack(params, &transport, &transport); + send_config_.local_renderer = local_preview.get(); + receive_configs_[0].renderer = loopback_video.get(); + + if (params.screenshare.enabled) + SetupScreenshare(params); + + send_stream_ = call->CreateVideoSendStream(send_config_, encoder_config_); + CreateCapturer(params, send_stream_->Input()); + + VideoReceiveStream* receive_stream = + call->CreateVideoReceiveStream(receive_configs_[0]); + + receive_stream->Start(); + send_stream_->Start(); + frame_generator_capturer_->Start(); + + test::PressEnterToContinue(); + + frame_generator_capturer_->Stop(); + send_stream_->Stop(); + receive_stream->Stop(); + + call->DestroyVideoReceiveStream(receive_stream); + call->DestroyVideoSendStream(send_stream_); + + transport.StopSending(); +} + +} // namespace webrtc diff --git a/webrtc/video/video_quality_test.h b/webrtc/video/video_quality_test.h new file mode 100644 index 0000000000..d606de7cac --- /dev/null +++ b/webrtc/video/video_quality_test.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef WEBRTC_VIDEO_VIDEO_QUALITY_TEST_H_ +#define WEBRTC_VIDEO_VIDEO_QUALITY_TEST_H_ + +#include + +#include "webrtc/test/call_test.h" +#include "webrtc/test/frame_generator.h" +#include "webrtc/test/testsupport/trace_to_stderr.h" + +namespace webrtc { + +class VideoQualityTest : public test::CallTest { + public: + // Parameters are grouped into smaller structs to make it easier to set + // the desired elements and skip unused, using aggregate initialization. + // Unfortunately, C++11 (as opposed to C11) doesn't support unnamed structs, + // which makes the implementation of VideoQualityTest a bit uglier. + struct Params { + struct { + size_t width; + size_t height; + int32_t fps; + int min_bitrate_bps; + int target_bitrate_bps; + int max_bitrate_bps; + std::string codec; + size_t num_temporal_layers; + + int min_transmit_bps; + Call::Config::BitrateConfig call_bitrate_config; + size_t tl_discard_threshold; + } common; + struct { // Video-specific settings. + std::string clip_name; + } video; + struct { // Screenshare-specific settings. + bool enabled; + int32_t slide_change_interval; + int32_t scroll_duration; + } screenshare; + struct { // Analyzer settings. + std::string test_label; + double avg_psnr_threshold; + double avg_ssim_threshold; + int test_durations_secs; + std::string graph_data_output_filename; + } analyzer; + FakeNetworkPipe::Config pipe; + bool logs; + }; + + VideoQualityTest(); + void RunWithAnalyzer(const Params& params); + void RunWithVideoRenderer(const Params& params); + + protected: + // No-op implementation to be able to instantiate this class from non-TEST_F + // locations. + void TestBody() override; + + void CreateCapturer(const Params& params, VideoCaptureInput* input); + void ValidateParams(const Params& params); + void SetupFullStack(const Params& params, + newapi::Transport* send_transport, + newapi::Transport* recv_transport); + void SetupScreenshare(const Params& params); + + rtc::scoped_ptr trace_to_stderr_; + rtc::scoped_ptr frame_generator_; + rtc::scoped_ptr encoder_; + VideoCodecUnion codec_settings_; + Clock* const clock_; +}; + +} // namespace webrtc + +#endif // WEBRTC_VIDEO_VIDEO_QUALITY_TEST_H_ diff --git a/webrtc/webrtc_tests.gypi b/webrtc/webrtc_tests.gypi index 8186415b6c..2515ebd101 100644 --- a/webrtc/webrtc_tests.gypi +++ b/webrtc/webrtc_tests.gypi @@ -45,45 +45,26 @@ ], }, { - 'target_name': 'full_stack_quality_sampler', - 'type': 'executable', - 'sources': [ - 'test/mac/run_test.mm', - 'test/run_test.cc', - 'test/run_test.h', - 'video/full_stack.cc', - 'video/full_stack_quality_sampler.cc', - ], - 'conditions': [ - ['OS=="mac"', { - 'sources!': [ - 'test/run_test.cc', - ], - }], - ], - 'dependencies': [ - '<(DEPTH)/testing/gtest.gyp:gtest', - '<(DEPTH)/third_party/gflags/gflags.gyp:gflags', - '<(webrtc_root)/modules/modules.gyp:video_capture', - 'test/webrtc_test_common.gyp:webrtc_test_common', - 'test/test.gyp:test_main', - 'webrtc', - ], - }, - { - 'target_name': 'loopback_base', + 'target_name': 'video_quality_test', 'type': 'static_library', 'sources': [ - 'video/loopback.cc', - 'video/loopback.h', + 'video/video_quality_test.cc', + 'video/video_quality_test.h', ], 'dependencies': [ '<(DEPTH)/testing/gtest.gyp:gtest', - '<(webrtc_root)/modules/modules.gyp:video_capture_module_internal_impl', '<(webrtc_root)/modules/modules.gyp:video_render', + '<(webrtc_root)/modules/modules.gyp:video_capture_module_internal_impl', '<(webrtc_root)/system_wrappers/system_wrappers.gyp:system_wrappers', 'webrtc', ], + 'conditions': [ + ['OS=="android"', { + 'dependencies!': [ + '<(webrtc_root)/modules/modules.gyp:video_capture_module_internal_impl', + ], + }], + ], }, { 'target_name': 'video_loopback', @@ -102,7 +83,7 @@ }], ], 'dependencies': [ - 'loopback_base', + 'video_quality_test', '<(DEPTH)/testing/gtest.gyp:gtest', '<(DEPTH)/third_party/gflags/gflags.gyp:gflags', 'test/webrtc_test_common.gyp:webrtc_test_common', @@ -128,7 +109,7 @@ }], ], 'dependencies': [ - 'loopback_base', + 'video_quality_test', '<(DEPTH)/testing/gtest.gyp:gtest', '<(DEPTH)/third_party/gflags/gflags.gyp:gflags', 'test/webrtc_test_common.gyp:webrtc_test_common', @@ -236,11 +217,13 @@ '<(webrtc_root)/modules/modules.gyp:video_capture', '<(webrtc_root)/test/test.gyp:channel_transport', '<(webrtc_root)/voice_engine/voice_engine.gyp:voice_engine', + 'video_quality_test', 'modules/modules.gyp:neteq_test_support', 'modules/modules.gyp:bwe_simulator', 'modules/modules.gyp:rtp_rtcp', 'test/test.gyp:test_main', 'test/webrtc_test_common.gyp:webrtc_test_common', + 'test/webrtc_test_common.gyp:webrtc_test_renderer', 'tools/tools.gyp:agc_manager', 'webrtc', ],