diff --git a/modules/video_coding/utility/ivf_file_reader.cc b/modules/video_coding/utility/ivf_file_reader.cc index 8703a29c37..9667bb7cec 100644 --- a/modules/video_coding/utility/ivf_file_reader.cc +++ b/modules/video_coding/utility/ivf_file_reader.cc @@ -122,6 +122,8 @@ absl::optional IvfFileReader::NextFrame() { // is missing it means there is a bug in error handling. RTC_DCHECK(next_frame_header_); int64_t current_timestamp = next_frame_header_->timestamp; + // The first frame from the file should be marked as Key frame. + bool is_first_frame = num_read_frames_ == 0; while (next_frame_header_ && current_timestamp == next_frame_header_->timestamp) { // Resize payload to fit next spatial layer. @@ -165,6 +167,10 @@ absl::optional IvfFileReader::NextFrame() { for (size_t i = 0; i < layer_sizes.size(); ++i) { image.SetSpatialLayerFrameSize(static_cast(i), layer_sizes[i]); } + if (is_first_frame) { + image._frameType = VideoFrameType::kVideoFrameKey; + } + image._completeFrame = true; return image; } diff --git a/modules/video_coding/utility/ivf_file_reader.h b/modules/video_coding/utility/ivf_file_reader.h index 05b1d79cdf..eb5a21d55d 100644 --- a/modules/video_coding/utility/ivf_file_reader.h +++ b/modules/video_coding/utility/ivf_file_reader.h @@ -40,6 +40,9 @@ class IvfFileReader { bool HasMoreFrames() const { return num_read_frames_ < num_frames_; } bool HasError() const { return has_error_; } + uint16_t GetFrameWidth() const { return width_; } + uint16_t GetFrameHeight() const { return height_; } + bool Close(); private: diff --git a/test/BUILD.gn b/test/BUILD.gn index f2e21ceef3..7656ecc1d4 100644 --- a/test/BUILD.gn +++ b/test/BUILD.gn @@ -292,6 +292,8 @@ if (rtc_include_tests) { sources = [ "testsupport/frame_reader.h", "testsupport/frame_writer.h", + "testsupport/ivf_video_frame_generator.cc", + "testsupport/ivf_video_frame_generator.h", "testsupport/mock/mock_frame_reader.h", "testsupport/video_frame_writer.cc", "testsupport/video_frame_writer.h", @@ -306,13 +308,25 @@ if (rtc_include_tests) { ":test_support", ":video_test_common", "../api:scoped_refptr", + "../api/video:encoded_image", "../api/video:video_frame", "../api/video:video_frame_i420", + "../api/video_codecs:video_codecs_api", "../common_video", + "../media:rtc_media_base", + "../modules/video_coding:video_codec_interface", + "../modules/video_coding:video_coding_utility", + "../modules/video_coding:webrtc_h264", + "../modules/video_coding:webrtc_vp8", + "../modules/video_coding:webrtc_vp9", "../rtc_base:checks", "../rtc_base:criticalsection", "../rtc_base:logging", "../rtc_base:rtc_base_approved", + "../rtc_base:rtc_event", + "../rtc_base/synchronization:sequence_checker", + "../rtc_base/system:file_wrapper", + "//third_party/abseil-cpp/absl/types:optional", ] if (!is_ios) { @@ -379,15 +393,28 @@ if (rtc_include_tests) { "../api:scoped_refptr", "../api:simulcast_test_fixture_api", "../api/test/video:function_video_factory", + "../api/video:encoded_image", "../api/video:video_frame", "../api/video:video_frame_i420", + "../api/video_codecs:video_codecs_api", "../call:video_stream_api", + "../common_video", + "../media:rtc_media_base", "../modules/rtp_rtcp", + "../modules/rtp_rtcp:rtp_rtcp_format", "../modules/video_coding:simulcast_test_fixture_impl", + "../modules/video_coding:video_codec_interface", + "../modules/video_coding:video_coding_utility", + "../modules/video_coding:webrtc_h264", + "../modules/video_coding:webrtc_vp8", + "../modules/video_coding:webrtc_vp9", + "../rtc_base:criticalsection", + "../rtc_base:rtc_event", "../rtc_base/system:file_wrapper", "time_controller:time_controller", "//third_party/abseil-cpp/absl/flags:flag", "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", ] sources = [ "call_config_utils_unittest.cc", @@ -397,6 +424,7 @@ if (rtc_include_tests) { "frame_generator_unittest.cc", "rtp_file_reader_unittest.cc", "rtp_file_writer_unittest.cc", + "testsupport/ivf_video_frame_generator_unittest.cc", "testsupport/perf_test_unittest.cc", "testsupport/test_artifacts_unittest.cc", "testsupport/video_frame_writer_unittest.cc", diff --git a/test/testsupport/ivf_video_frame_generator.cc b/test/testsupport/ivf_video_frame_generator.cc new file mode 100644 index 0000000000..54d8bc3a62 --- /dev/null +++ b/test/testsupport/ivf_video_frame_generator.cc @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/testsupport/ivf_video_frame_generator.h" + +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "api/video_codecs/video_codec.h" +#include "media/base/media_constants.h" +#include "modules/video_coding/codecs/h264/include/h264.h" +#include "modules/video_coding/codecs/vp8/include/vp8.h" +#include "modules/video_coding/codecs/vp9/include/vp9.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/checks.h" +#include "rtc_base/system/file_wrapper.h" + +namespace webrtc { +namespace test { +namespace { + +constexpr int kMaxNextFrameWaitTemeoutMs = 1000; + +} // namespace + +IvfVideoFrameGenerator::IvfVideoFrameGenerator(const std::string& file_name) + : callback_(this), + file_reader_(IvfFileReader::Create(FileWrapper::OpenReadOnly(file_name))), + video_decoder_(CreateVideoDecoder(file_reader_->GetVideoCodecType())), + width_(file_reader_->GetFrameWidth()), + height_(file_reader_->GetFrameHeight()) { + RTC_CHECK(video_decoder_) << "No decoder found for file's video codec type"; + VideoCodec codec_settings; + codec_settings.codecType = file_reader_->GetVideoCodecType(); + codec_settings.width = file_reader_->GetFrameWidth(); + codec_settings.height = file_reader_->GetFrameHeight(); + RTC_CHECK_EQ(video_decoder_->RegisterDecodeCompleteCallback(&callback_), + WEBRTC_VIDEO_CODEC_OK); + RTC_CHECK_EQ( + video_decoder_->InitDecode(&codec_settings, /*number_of_cores=*/1), + WEBRTC_VIDEO_CODEC_OK); + sequence_checker_.Detach(); +} +IvfVideoFrameGenerator::~IvfVideoFrameGenerator() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + if (!file_reader_) { + return; + } + file_reader_->Close(); + file_reader_.reset(); + // Reset decoder to prevent it from async access to |this|. + video_decoder_.reset(); + { + rtc::CritScope crit(&lock_); + next_frame_ = absl::nullopt; + // Set event in case another thread is waiting on it. + next_frame_decoded_.Set(); + } +} + +VideoFrame* IvfVideoFrameGenerator::NextFrame() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + next_frame_decoded_.Reset(); + if (!file_reader_) { + return nullptr; + } + if (!file_reader_->HasMoreFrames()) { + file_reader_->Reset(); + } + absl::optional image = file_reader_->NextFrame(); + if (!image) { + return nullptr; + } + RTC_DCHECK(image); + // Last parameter is undocumented and there is no usage of it found. + RTC_DCHECK_EQ(WEBRTC_VIDEO_CODEC_OK, + video_decoder_->Decode(*image, /*missing_frames=*/false, + /*render_time_ms=*/0)); + bool decoded = next_frame_decoded_.Wait(kMaxNextFrameWaitTemeoutMs); + RTC_CHECK(decoded) << "Failed to decode next frame in " + << kMaxNextFrameWaitTemeoutMs << "ms. Can't continue"; + + rtc::CritScope crit(&lock_); + if (width_ != static_cast(next_frame_->width()) || + height_ != static_cast(next_frame_->height())) { + // Video adapter has requested a down-scale. Allocate a new buffer and + // return scaled version. + rtc::scoped_refptr scaled_buffer = + I420Buffer::Create(width_, height_); + scaled_buffer->ScaleFrom(*next_frame_->video_frame_buffer()->ToI420()); + next_frame_ = VideoFrame::Builder() + .set_video_frame_buffer(scaled_buffer) + .set_rotation(kVideoRotation_0) + .set_timestamp_us(next_frame_->timestamp_us()) + .set_id(next_frame_->id()) + .build(); + } + return &next_frame_.value(); +} + +void IvfVideoFrameGenerator::ChangeResolution(size_t width, size_t height) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + width_ = width; + height_ = height; +} + +int32_t IvfVideoFrameGenerator::DecodedCallback::Decoded( + VideoFrame& decoded_image) { + Decoded(decoded_image, 0, 0); + return WEBRTC_VIDEO_CODEC_OK; +} +int32_t IvfVideoFrameGenerator::DecodedCallback::Decoded( + VideoFrame& decoded_image, + int64_t decode_time_ms) { + Decoded(decoded_image, decode_time_ms, 0); + return WEBRTC_VIDEO_CODEC_OK; +} +void IvfVideoFrameGenerator::DecodedCallback::Decoded( + VideoFrame& decoded_image, + absl::optional decode_time_ms, + absl::optional qp) { + reader_->OnFrameDecoded(decoded_image); +} + +void IvfVideoFrameGenerator::OnFrameDecoded(const VideoFrame& decoded_frame) { + rtc::CritScope crit(&lock_); + next_frame_ = decoded_frame; + next_frame_decoded_.Set(); +} + +std::unique_ptr IvfVideoFrameGenerator::CreateVideoDecoder( + VideoCodecType codec_type) { + if (codec_type == VideoCodecType::kVideoCodecVP8) { + return VP8Decoder::Create(); + } + if (codec_type == VideoCodecType::kVideoCodecVP9) { + return VP9Decoder::Create(); + } + if (codec_type == VideoCodecType::kVideoCodecH264) { + return H264Decoder::Create(); + } + return nullptr; +} + +} // namespace test +} // namespace webrtc diff --git a/test/testsupport/ivf_video_frame_generator.h b/test/testsupport/ivf_video_frame_generator.h new file mode 100644 index 0000000000..f1f00d7380 --- /dev/null +++ b/test/testsupport/ivf_video_frame_generator.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_TESTSUPPORT_IVF_VIDEO_FRAME_GENERATOR_H_ +#define TEST_TESTSUPPORT_IVF_VIDEO_FRAME_GENERATOR_H_ + +#include +#include + +#include "absl/types/optional.h" +#include "api/video/video_codec_type.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/video_decoder.h" +#include "modules/video_coding/utility/ivf_file_reader.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/event.h" +#include "rtc_base/synchronization/sequence_checker.h" +#include "test/frame_generator.h" + +namespace webrtc { +namespace test { + +// All methods except constructor must be used from the same thread. +class IvfVideoFrameGenerator : public FrameGenerator { + public: + explicit IvfVideoFrameGenerator(const std::string& file_name); + ~IvfVideoFrameGenerator() override; + + VideoFrame* NextFrame() override; + void ChangeResolution(size_t width, size_t height) override; + + private: + class DecodedCallback : public DecodedImageCallback { + public: + explicit DecodedCallback(IvfVideoFrameGenerator* reader) + : reader_(reader) {} + + int32_t Decoded(VideoFrame& decoded_image) override; + int32_t Decoded(VideoFrame& decoded_image, int64_t decode_time_ms) override; + void Decoded(VideoFrame& decoded_image, + absl::optional decode_time_ms, + absl::optional qp) override; + + private: + IvfVideoFrameGenerator* const reader_; + }; + + void OnFrameDecoded(const VideoFrame& decoded_frame); + static std::unique_ptr CreateVideoDecoder( + VideoCodecType codec_type); + + DecodedCallback callback_; + std::unique_ptr file_reader_; + std::unique_ptr video_decoder_; + + size_t width_; + size_t height_; + + rtc::Event next_frame_decoded_; + SequenceChecker sequence_checker_; + + rtc::CriticalSection lock_; + absl::optional next_frame_ RTC_GUARDED_BY(lock_); +}; + +} // namespace test +} // namespace webrtc + +#endif // TEST_TESTSUPPORT_IVF_VIDEO_FRAME_GENERATOR_H_ diff --git a/test/testsupport/ivf_video_frame_generator_unittest.cc b/test/testsupport/ivf_video_frame_generator_unittest.cc new file mode 100644 index 0000000000..fd469f29a8 --- /dev/null +++ b/test/testsupport/ivf_video_frame_generator_unittest.cc @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "absl/types/optional.h" +#include "api/video/encoded_image.h" +#include "api/video/video_codec_type.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "media/base/codec.h" +#include "media/base/media_constants.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/video_coding/codecs/vp8/include/vp8.h" +#include "modules/video_coding/codecs/vp9/include/vp9.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/utility/ivf_file_writer.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/event.h" +#include "test/frame_generator.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" +#include "test/testsupport/ivf_video_frame_generator.h" +#include "test/video_codec_settings.h" + +#if defined(WEBRTC_USE_H264) +#include "modules/video_coding/codecs/h264/include/h264.h" +#endif + +namespace webrtc { +namespace test { +namespace { + +constexpr int kWidth = 320; +constexpr int kHeight = 240; +constexpr int kVideoFramesCount = 30; +constexpr int kMaxFramerate = 30; +constexpr int kMaxFrameEncodeWaitTimeoutMs = 2000; +static const VideoEncoder::Capabilities kCapabilities(false); + +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +constexpr double kExpectedMinPsnr = 36; +#else +constexpr double kExpectedMinPsnr = 39; +#endif + +class IvfFileWriterEncodedCallback : public EncodedImageCallback { + public: + IvfFileWriterEncodedCallback(const std::string& file_name, + VideoCodecType video_codec_type, + int expected_frames_count) + : file_writer_( + IvfFileWriter::Wrap(FileWrapper::OpenWriteOnly(file_name), 0)), + video_codec_type_(video_codec_type), + expected_frames_count_(expected_frames_count) { + EXPECT_TRUE(file_writer_.get()); + } + ~IvfFileWriterEncodedCallback() { EXPECT_TRUE(file_writer_->Close()); } + + Result OnEncodedImage(const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info, + const RTPFragmentationHeader* fragmentation) override { + EXPECT_TRUE(file_writer_->WriteFrame(encoded_image, video_codec_type_)); + + rtc::CritScope crit(&lock_); + received_frames_count_++; + RTC_CHECK_LE(received_frames_count_, expected_frames_count_); + if (received_frames_count_ == expected_frames_count_) { + expected_frames_count_received_.Set(); + } + return Result(Result::Error::OK); + } + + bool WaitForExpectedFramesReceived(int timeout_ms) { + return expected_frames_count_received_.Wait(timeout_ms); + } + + private: + std::unique_ptr file_writer_; + const VideoCodecType video_codec_type_; + const int expected_frames_count_; + + rtc::CriticalSection lock_; + int received_frames_count_ RTC_GUARDED_BY(lock_) = 0; + rtc::Event expected_frames_count_received_; +}; + +class IvfVideoFrameGeneratorTest : public ::testing::Test { + protected: + void SetUp() override { + file_name_ = + webrtc::test::TempFilename(webrtc::test::OutputPath(), "test_file.ivf"); + } + void TearDown() override { webrtc::test::RemoveFile(file_name_); } + + void CreateTestVideoFile(VideoCodecType video_codec_type, + std::unique_ptr video_encoder) { + std::unique_ptr frame_generator = + test::FrameGenerator::CreateSquareGenerator( + kWidth, kHeight, test::FrameGenerator::OutputType::kI420, + absl::nullopt); + + VideoCodec codec_settings; + webrtc::test::CodecSettings(video_codec_type, &codec_settings); + codec_settings.width = kWidth; + codec_settings.height = kHeight; + codec_settings.maxFramerate = kMaxFramerate; + const uint32_t kBitrateBps = 500000; + VideoBitrateAllocation bitrate_allocation; + bitrate_allocation.SetBitrate(0, 0, kBitrateBps); + + IvfFileWriterEncodedCallback ivf_writer_callback( + file_name_, video_codec_type, kVideoFramesCount); + + video_encoder->RegisterEncodeCompleteCallback(&ivf_writer_callback); + video_encoder->SetRates(VideoEncoder::RateControlParameters( + bitrate_allocation, static_cast(codec_settings.maxFramerate))); + ASSERT_EQ(WEBRTC_VIDEO_CODEC_OK, + video_encoder->InitEncode( + &codec_settings, + VideoEncoder::Settings(kCapabilities, /*number_of_cores=*/1, + /*max_payload_size=*/0))); + + uint32_t last_frame_timestamp = 0; + + for (int i = 0; i < kVideoFramesCount; ++i) { + VideoFrame* frame = frame_generator->NextFrame(); + const uint32_t timestamp = + last_frame_timestamp + + kVideoPayloadTypeFrequency / codec_settings.maxFramerate; + frame->set_timestamp(timestamp); + + last_frame_timestamp = timestamp; + + ASSERT_EQ(WEBRTC_VIDEO_CODEC_OK, video_encoder->Encode(*frame, nullptr)); + video_frames_.push_back(*frame); + } + + ASSERT_TRUE(ivf_writer_callback.WaitForExpectedFramesReceived( + kMaxFrameEncodeWaitTimeoutMs)); + } + + std::string file_name_; + std::vector video_frames_; +}; + +} // namespace + +TEST_F(IvfVideoFrameGeneratorTest, Vp8) { + CreateTestVideoFile(VideoCodecType::kVideoCodecVP8, VP8Encoder::Create()); + IvfVideoFrameGenerator generator(file_name_); + for (size_t i = 0; i < video_frames_.size(); ++i) { + auto& expected_frame = video_frames_[i]; + VideoFrame* actual_frame = generator.NextFrame(); + EXPECT_TRUE(actual_frame); + EXPECT_GT(I420PSNR(&expected_frame, actual_frame), kExpectedMinPsnr); + } +} + +TEST_F(IvfVideoFrameGeneratorTest, Vp8DoubleRead) { + CreateTestVideoFile(VideoCodecType::kVideoCodecVP8, VP8Encoder::Create()); + IvfVideoFrameGenerator generator(file_name_); + for (size_t i = 0; i < video_frames_.size() * 2; ++i) { + auto& expected_frame = video_frames_[i % video_frames_.size()]; + VideoFrame* actual_frame = generator.NextFrame(); + EXPECT_TRUE(actual_frame); + EXPECT_GT(I420PSNR(&expected_frame, actual_frame), kExpectedMinPsnr); + } +} + +TEST_F(IvfVideoFrameGeneratorTest, Vp9) { + CreateTestVideoFile(VideoCodecType::kVideoCodecVP9, VP9Encoder::Create()); + IvfVideoFrameGenerator generator(file_name_); + for (size_t i = 0; i < video_frames_.size(); ++i) { + auto& expected_frame = video_frames_[i]; + VideoFrame* actual_frame = generator.NextFrame(); + EXPECT_TRUE(actual_frame); + EXPECT_GT(I420PSNR(&expected_frame, actual_frame), kExpectedMinPsnr); + } +} + +#if defined(WEBRTC_USE_H264) +TEST_F(IvfVideoFrameGeneratorTest, H264) { + CreateTestVideoFile( + VideoCodecType::kVideoCodecH264, + H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName))); + IvfVideoFrameGenerator generator(file_name_); + for (size_t i = 0; i < video_frames_.size(); ++i) { + auto& expected_frame = video_frames_[i]; + VideoFrame* actual_frame = generator.NextFrame(); + EXPECT_TRUE(actual_frame); + EXPECT_GT(I420PSNR(&expected_frame, actual_frame), kExpectedMinPsnr); + } +} +#endif + +} // namespace test +} // namespace webrtc