zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'upstream-master'

Browse Source 
Bug: 153469641
Test: none
Change-Id: Ic33e363deb0d1ac4bb4d57c3c239eb2e45370802
This commit is contained in:
Jorge E. Moreira
2020-07-17 14:12:45 -07:00
parent 477c82e10a cc57b935cd
commit 85770cd25b
9929 changed files with 1031155 additions and 779502 deletions
Download Patch File Download Diff File Expand all files Collapse all files

966
modules/video_coding/BUILD.gn Normal file
View File

@ -0,0 +1,966 @@
# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
import("../../webrtc.gni")
rtc_library("encoded_frame") {
visibility = [ "*" ]
sources = [
"encoded_frame.cc",
"encoded_frame.h",
]
deps = [
":codec_globals_headers",
":video_codec_interface",
"../../api/video:encoded_image",
"../../api/video:video_frame",
"../../api/video:video_frame_i420",
"../../api/video:video_rtp_headers",
"../../modules:module_api",
"../../modules:module_api_public",
"../../modules/rtp_rtcp:rtp_video_header",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"../../rtc_base/experiments:alr_experiment",
"../../rtc_base/experiments:rtt_mult_experiment",
"../../rtc_base/system:rtc_export",
"../../system_wrappers",
"//third_party/abseil-cpp/absl/types:optional",
"//third_party/abseil-cpp/absl/types:variant",
]
}
rtc_library("frame_dependencies_calculator") {
sources = [
"frame_dependencies_calculator.cc",
"frame_dependencies_calculator.h",
]
deps = [
"../../api:array_view",
"../../api/video:video_frame_type",
"../../common_video/generic_frame_descriptor",
"../../rtc_base:checks",
"../../rtc_base:logging",
"//third_party/abseil-cpp/absl/algorithm:container",
"//third_party/abseil-cpp/absl/container:inlined_vector",
"//third_party/abseil-cpp/absl/types:optional",
]
}
rtc_library("nack_module") {
visibility = [ "*" ]
sources = [
"histogram.cc",
"histogram.h",
"nack_module.cc",
"nack_module.h",
]
deps = [
"..:module_api",
"../../api/units:time_delta",
"../../api/units:timestamp",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"../../rtc_base:rtc_numerics",
"../../rtc_base/experiments:field_trial_parser",
"../../system_wrappers",
"../../system_wrappers:field_trial",
"../utility",
]
}
rtc_library("video_coding") {
visibility = [ "*" ]
deps = [
"..:module_fec_api",
"../../api:array_view",
"../../api:scoped_refptr",
"../../api/video:encoded_image",
"../../api/video:video_adaptation",
"../../api/video:video_bitrate_allocation",
"../../api/video:video_bitrate_allocator_factory",
"../../rtc_base:deprecation",
"../../rtc_base/task_utils:to_queued_task",
"../../system_wrappers:field_trial",
"../../system_wrappers:metrics",
"../rtp_rtcp:rtp_video_header",
"//third_party/abseil-cpp/absl/base:core_headers",
"//third_party/abseil-cpp/absl/memory",
]
sources = [
"codec_timer.cc",
"codec_timer.h",
"decoder_database.cc",
"decoder_database.h",
"fec_controller_default.cc",
"fec_controller_default.h",
"fec_rate_table.h",
"frame_buffer2.cc",
"frame_buffer2.h",
"frame_object.cc",
"frame_object.h",
"generic_decoder.cc",
"generic_decoder.h",
"h264_sprop_parameter_sets.cc",
"h264_sprop_parameter_sets.h",
"h264_sps_pps_tracker.cc",
"h264_sps_pps_tracker.h",
"include/video_codec_initializer.h",
"inter_frame_delay.cc",
"inter_frame_delay.h",
"internal_defines.h",
"jitter_estimator.cc",
"jitter_estimator.h",
"loss_notification_controller.cc",
"loss_notification_controller.h",
"media_opt_util.cc",
"media_opt_util.h",
"packet_buffer.cc",
"packet_buffer.h",
"rtp_frame_reference_finder.cc",
"rtp_frame_reference_finder.h",
"rtt_filter.cc",
"rtt_filter.h",
"timestamp_map.cc",
"timestamp_map.h",
"timing.cc",
"timing.h",
"unique_timestamp_counter.cc",
"unique_timestamp_counter.h",
"video_codec_initializer.cc",
"video_receiver2.cc",
"video_receiver2.h",
]
deps += [
":codec_globals_headers",
":encoded_frame",
":video_codec_interface",
":video_coding_utility",
":webrtc_vp9_helpers",
"..:module_api",
"..:module_api_public",
"../../api:fec_controller_api",
"../../api:rtp_headers",
"../../api:rtp_packet_info",
"../../api/units:data_rate",
"../../api/units:time_delta",
"../../api/video:builtin_video_bitrate_allocator_factory",
"../../api/video:encoded_frame",
"../../api/video:video_adaptation",
"../../api/video:video_bitrate_allocator",
"../../api/video:video_frame",
"../../api/video:video_frame_i420",
"../../api/video:video_frame_type",
"../../api/video:video_rtp_headers",
"../../api/video_codecs:video_codecs_api",
"../../common_video",
"../../rtc_base",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"../../rtc_base:rtc_numerics",
"../../rtc_base:rtc_task_queue",
"../../rtc_base/experiments:alr_experiment",
"../../rtc_base/experiments:field_trial_parser",
"../../rtc_base/experiments:jitter_upper_bound_experiment",
"../../rtc_base/experiments:min_video_bitrate_experiment",
"../../rtc_base/experiments:rate_control_settings",
"../../rtc_base/experiments:rtt_mult_experiment",
"../../rtc_base/synchronization:sequence_checker",
"../../rtc_base/task_utils:repeating_task",
"../../rtc_base/third_party/base64",
"../../rtc_base/time:timestamp_extrapolator",
"../../system_wrappers",
"../rtp_rtcp",
"../rtp_rtcp:rtp_rtcp_format",
"//third_party/abseil-cpp/absl/container:inlined_vector",
"//third_party/abseil-cpp/absl/types:optional",
"//third_party/abseil-cpp/absl/types:variant",
]
}
rtc_library("video_codec_interface") {
visibility = [ "*" ]
sources = [
"include/video_codec_interface.cc",
"include/video_codec_interface.h",
"include/video_coding_defines.h",
"include/video_error_codes.h",
"video_coding_defines.cc",
]
deps = [
":codec_globals_headers",
"..:module_api",
"../../api/video:video_frame",
"../../api/video:video_rtp_headers",
"../../api/video_codecs:video_codecs_api",
"../../common_video",
"../../common_video/generic_frame_descriptor",
"../../rtc_base/system:rtc_export",
"//third_party/abseil-cpp/absl/types:optional",
]
}
rtc_library("video_coding_legacy") {
visibility = [ ":video_coding_unittests" ]
sources = [
"decoding_state.cc",
"decoding_state.h",
"event_wrapper.cc",
"event_wrapper.h",
"frame_buffer.cc",
"frame_buffer.h",
"include/video_coding.h",
"jitter_buffer.cc",
"jitter_buffer.h",
"jitter_buffer_common.h",
"packet.cc",
"packet.h",
"receiver.cc",
"receiver.h",
"session_info.cc",
"session_info.h",
"video_coding_impl.cc",
"video_coding_impl.h",
"video_receiver.cc",
]
deps = [
":codec_globals_headers",
":encoded_frame",
":video_codec_interface",
":video_coding",
"..:module_api",
"..:module_api_public",
"../../:webrtc_common",
"../../api:rtp_headers",
"../../api:rtp_packet_info",
"../../api/video:encoded_image",
"../../api/video:video_frame",
"../../api/video:video_frame_type",
"../../api/video:video_rtp_headers",
"../../api/video_codecs:video_codecs_api",
"../../common_video",
"../../modules/rtp_rtcp:rtp_video_header",
"../../rtc_base:checks",
"../../rtc_base:logging",
"../../rtc_base:rtc_base_approved",
"../../rtc_base:rtc_event",
"../../rtc_base/synchronization:sequence_checker",
"../../system_wrappers",
"../rtp_rtcp:rtp_rtcp_format",
"../rtp_rtcp:rtp_video_header",
"../utility",
"//third_party/abseil-cpp/absl/memory",
"//third_party/abseil-cpp/absl/types:optional",
"//third_party/abseil-cpp/absl/types:variant",
]
}
rtc_source_set("codec_globals_headers") {
visibility = [ "*" ]
sources = [
"codecs/h264/include/h264_globals.h",
"codecs/interface/common_constants.h",
"codecs/vp8/include/vp8_globals.h",
"codecs/vp9/include/vp9_globals.h",
]
deps = [ "../../rtc_base:checks" ]
}
rtc_library("video_coding_utility") {
visibility = [ "*" ]
sources = [
"utility/decoded_frames_history.cc",
"utility/decoded_frames_history.h",
"utility/frame_dropper.cc",
"utility/frame_dropper.h",
"utility/framerate_controller.cc",
"utility/framerate_controller.h",
"utility/ivf_file_reader.cc",
"utility/ivf_file_reader.h",
"utility/ivf_file_writer.cc",
"utility/ivf_file_writer.h",
"utility/quality_scaler.cc",
"utility/quality_scaler.h",
"utility/simulcast_rate_allocator.cc",
"utility/simulcast_rate_allocator.h",
"utility/simulcast_utility.cc",
"utility/simulcast_utility.h",
"utility/vp8_header_parser.cc",
"utility/vp8_header_parser.h",
"utility/vp9_uncompressed_header_parser.cc",
"utility/vp9_uncompressed_header_parser.h",
]
deps = [
":video_codec_interface",
"..:module_api",
"../../api:scoped_refptr",
"../../api/video:encoded_frame",
"../../api/video:encoded_image",
"../../api/video:video_adaptation",
"../../api/video:video_bitrate_allocation",
"../../api/video:video_bitrate_allocator",
"../../api/video_codecs:video_codecs_api",
"../../common_video",
"../../modules/rtp_rtcp",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"../../rtc_base:rtc_numerics",
"../../rtc_base:rtc_task_queue",
"../../rtc_base:weak_ptr",
"../../rtc_base/experiments:quality_scaler_settings",
"../../rtc_base/experiments:quality_scaling_experiment",
"../../rtc_base/experiments:rate_control_settings",
"../../rtc_base/experiments:stable_target_rate_experiment",
"../../rtc_base/synchronization:sequence_checker",
"../../rtc_base/system:arch",
"../../rtc_base/system:file_wrapper",
"../../rtc_base/task_utils:repeating_task",
"../../rtc_base/task_utils:to_queued_task",
"../../system_wrappers:field_trial",
"../rtp_rtcp:rtp_rtcp_format",
"//third_party/abseil-cpp/absl/types:optional",
]
}
rtc_library("webrtc_h264") {
visibility = [ "*" ]
sources = [
"codecs/h264/h264.cc",
"codecs/h264/h264_color_space.cc",
"codecs/h264/h264_color_space.h",
"codecs/h264/h264_decoder_impl.cc",
"codecs/h264/h264_decoder_impl.h",
"codecs/h264/h264_encoder_impl.cc",
"codecs/h264/h264_encoder_impl.h",
"codecs/h264/include/h264.h",
]
defines = []
deps = [
":video_codec_interface",
":video_coding_utility",
"../../api/video:video_frame",
"../../api/video:video_frame_i010",
"../../api/video:video_frame_i420",
"../../api/video:video_rtp_headers",
"../../api/video_codecs:video_codecs_api",
"../../common_video",
"../../media:rtc_h264_profile_id",
"../../media:rtc_media_base",
"../../rtc_base",
"../../rtc_base:checks",
"../../rtc_base/system:rtc_export",
"../../system_wrappers:field_trial",
"../../system_wrappers:metrics",
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
"//third_party/libyuv",
]
if (rtc_use_h264) {
deps += [
"//third_party/ffmpeg",
"//third_party/openh264:encoder",
]
if (!build_with_mozilla) {
deps += [ "../../media:rtc_media_base" ]
}
}
}
rtc_library("webrtc_multiplex") {
sources = [
"codecs/multiplex/augmented_video_frame_buffer.cc",
"codecs/multiplex/include/augmented_video_frame_buffer.h",
"codecs/multiplex/include/multiplex_decoder_adapter.h",
"codecs/multiplex/include/multiplex_encoder_adapter.h",
"codecs/multiplex/multiplex_decoder_adapter.cc",
"codecs/multiplex/multiplex_encoded_image_packer.cc",
"codecs/multiplex/multiplex_encoded_image_packer.h",
"codecs/multiplex/multiplex_encoder_adapter.cc",
]
deps = [
":video_codec_interface",
":video_coding_utility",
"..:module_api",
"../../api:fec_controller_api",
"../../api:scoped_refptr",
"../../api/video:encoded_image",
"../../api/video:video_frame",
"../../api/video:video_frame_i420",
"../../api/video:video_rtp_headers",
"../../api/video_codecs:video_codecs_api",
"../../common_video",
"../../media:rtc_media_base",
"../../rtc_base",
"../../rtc_base:checks",
"../rtp_rtcp:rtp_rtcp_format",
]
}
# This target includes the internal SW codec.
rtc_library("webrtc_vp8") {
visibility = [ "*" ]
poisonous = [ "software_video_codecs" ]
sources = [
"codecs/vp8/include/vp8.h",
"codecs/vp8/libvpx_interface.cc",
"codecs/vp8/libvpx_interface.h",
"codecs/vp8/libvpx_vp8_decoder.cc",
"codecs/vp8/libvpx_vp8_decoder.h",
"codecs/vp8/libvpx_vp8_encoder.cc",
"codecs/vp8/libvpx_vp8_encoder.h",
]
deps = [
":codec_globals_headers",
":video_codec_interface",
":video_coding_utility",
":webrtc_vp8_temporal_layers",
"..:module_api",
"../..:webrtc_common",
"../../api:fec_controller_api",
"../../api:scoped_refptr",
"../../api/video:encoded_image",
"../../api/video:video_frame",
"../../api/video:video_frame_i420",
"../../api/video:video_rtp_headers",
"../../api/video_codecs:video_codecs_api",
"../../api/video_codecs:vp8_temporal_layers_factory",
"../../common_video",
"../../rtc_base:checks",
"../../rtc_base:deprecation",
"../../rtc_base:rtc_base_approved",
"../../rtc_base:rtc_numerics",
"../../rtc_base/experiments:cpu_speed_experiment",
"../../rtc_base/experiments:field_trial_parser",
"../../rtc_base/experiments:rate_control_settings",
"../../system_wrappers:field_trial",
"../../system_wrappers:metrics",
"//third_party/abseil-cpp/absl/types:optional",
"//third_party/libyuv",
]
if (rtc_build_libvpx) {
deps += [ rtc_libvpx_dir ]
}
}
rtc_library("webrtc_vp8_temporal_layers") {
visibility = [ "*" ]
sources = [
"codecs/vp8/default_temporal_layers.cc",
"codecs/vp8/default_temporal_layers.h",
"codecs/vp8/include/temporal_layers_checker.h",
"codecs/vp8/screenshare_layers.cc",
"codecs/vp8/screenshare_layers.h",
"codecs/vp8/temporal_layers.h",
"codecs/vp8/temporal_layers_checker.cc",
]
deps = [
":codec_globals_headers",
":video_codec_interface",
":video_coding_utility",
"..:module_api",
"../..:webrtc_common",
"../../api:fec_controller_api",
"../../api/video_codecs:video_codecs_api",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"../../rtc_base:rtc_numerics",
"../../system_wrappers:field_trial",
"../../system_wrappers:metrics",
"//third_party/abseil-cpp/absl/types:optional",
]
}
# This target includes VP9 files that may be used for any VP9 codec, internal SW or external HW.
rtc_library("webrtc_vp9_helpers") {
sources = [
"codecs/vp9/svc_config.cc",
"codecs/vp9/svc_config.h",
"codecs/vp9/svc_rate_allocator.cc",
"codecs/vp9/svc_rate_allocator.h",
]
deps = [
":codec_globals_headers",
":video_codec_interface",
"../..:webrtc_common",
"../../api/video:video_bitrate_allocation",
"../../api/video:video_bitrate_allocator",
"../../api/video:video_codec_constants",
"../../api/video_codecs:video_codecs_api",
"../../common_video",
"../../rtc_base:checks",
"../../rtc_base/experiments:stable_target_rate_experiment",
"//third_party/abseil-cpp/absl/container:inlined_vector",
]
}
rtc_library("webrtc_vp9") {
visibility = [ "*" ]
poisonous = [ "software_video_codecs" ]
sources = [
"codecs/vp9/include/vp9.h",
"codecs/vp9/vp9.cc",
"codecs/vp9/vp9_frame_buffer_pool.cc",
"codecs/vp9/vp9_frame_buffer_pool.h",
"codecs/vp9/vp9_impl.cc",
"codecs/vp9/vp9_impl.h",
]
deps = [
":video_codec_interface",
":video_coding_utility",
":webrtc_vp9_helpers",
"..:module_api",
"../..:webrtc_common",
"../../api:fec_controller_api",
"../../api:scoped_refptr",
"../../api/video:video_frame",
"../../api/video:video_frame_i010",
"../../api/video:video_rtp_headers",
"../../api/video_codecs:video_codecs_api",
"../../common_video",
"../../media:rtc_media_base",
"../../media:rtc_vp9_profile",
"../../rtc_base",
"../../rtc_base:checks",
"../../rtc_base/experiments:rate_control_settings",
"../../system_wrappers:field_trial",
"../rtp_rtcp:rtp_rtcp_format",
"//third_party/abseil-cpp/absl/memory",
]
if (rtc_build_libvpx) {
deps += [ rtc_libvpx_dir ]
}
}
if (rtc_include_tests) {
if (is_android) {
rtc_library("android_codec_factory_helper") {
sources = [
"codecs/test/android_codec_factory_helper.cc",
"codecs/test/android_codec_factory_helper.h",
]
deps = [
"../../api/video_codecs:video_codecs_api",
"../../rtc_base:checks",
"../../rtc_base:ignore_wundef",
"../../rtc_base:rtc_base_approved",
"../../sdk/android:native_api_base",
"../../sdk/android:native_api_codecs",
"../../sdk/android:native_api_jni",
"//base",
]
}
}
if (is_ios || is_mac) {
rtc_library("objc_codec_factory_helper") {
sources = [
"codecs/test/objc_codec_factory_helper.h",
"codecs/test/objc_codec_factory_helper.mm",
]
deps = [
"../../api/video:video_frame",
"../../api/video:video_rtp_headers",
"../../api/video_codecs:video_codecs_api",
"../../media:rtc_audio_video",
"../../media:rtc_media_base",
"../../modules:module_api",
"../../rtc_base:rtc_base_approved",
"../../sdk:native_api",
"../../sdk:peerconnectionfactory_base_objc",
"../../sdk:videocodec_objc",
"../../sdk:videosource_objc",
"../../sdk:videotoolbox_objc",
]
}
}
rtc_library("simulcast_test_fixture_impl") {
testonly = true
sources = [
"utility/simulcast_test_fixture_impl.cc",
"utility/simulcast_test_fixture_impl.h",
]
deps = [
":video_codec_interface",
":video_coding",
":video_coding_utility",
"../../:webrtc_common",
"../../api:mock_video_decoder",
"../../api:mock_video_encoder",
"../../api:simulcast_test_fixture_api",
"../../api/video:encoded_image",
"../../api/video:video_frame",
"../../api/video:video_frame_i420",
"../../api/video:video_rtp_headers",
"../../api/video_codecs:video_codecs_api",
"../../common_video",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"../../test:test_support",
]
}
rtc_library("video_codecs_test_framework") {
testonly = true
sources = [
"codecs/test/video_codec_unittest.cc",
"codecs/test/video_codec_unittest.h",
"codecs/test/videoprocessor.cc",
"codecs/test/videoprocessor.h",
]
deps = [
":codec_globals_headers",
":video_codec_interface",
":video_coding",
":video_coding_utility",
":videocodec_test_stats_impl",
":webrtc_vp9_helpers",
"..:module_api",
"../../api:create_frame_generator",
"../../api:frame_generator_api",
"../../api:scoped_refptr",
"../../api:videocodec_test_fixture_api",
"../../api/task_queue",
"../../api/video:builtin_video_bitrate_allocator_factory",
"../../api/video:encoded_image",
"../../api/video:video_bitrate_allocation",
"../../api/video:video_bitrate_allocator",
"../../api/video:video_bitrate_allocator_factory",
"../../api/video:video_frame",
"../../api/video:video_frame_i420",
"../../api/video:video_rtp_headers",
"../../api/video_codecs:video_codecs_api",
"../../common_video",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"../../rtc_base:rtc_task_queue",
"../../rtc_base/synchronization:sequence_checker",
"../../rtc_base/task_utils:to_queued_task",
"../../test:test_support",
"../../test:video_test_common",
"../../test:video_test_support",
"../rtp_rtcp:rtp_rtcp_format",
"//third_party/abseil-cpp/absl/types:optional",
"//third_party/libyuv",
]
}
video_coding_modules_tests_resources = []
if (is_android) {
video_coding_modules_tests_resources += [
"../../resources/foreman_128x96.yuv",
"../../resources/foreman_160x120.yuv",
"../../resources/foreman_176x144.yuv",
"../../resources/foreman_240x136.yuv",
"../../resources/foreman_320x240.yuv",
"../../resources/foreman_480x272.yuv",
]
}
if (!is_android) {
video_coding_modules_tests_resources += [
"../../resources/ConferenceMotion_1280_720_50.yuv",
"../../resources/FourPeople_1280x720_30.yuv",
]
}
num_video_coding_modules_tests_resources = 0
foreach(i, video_coding_modules_tests_resources) {
num_video_coding_modules_tests_resources += 1
}
if (num_video_coding_modules_tests_resources > 0) {
if (is_ios || is_mac) {
bundle_data("video_coding_modules_tests_resources_bundle_data") {
testonly = true
sources = video_coding_modules_tests_resources
outputs = [ "{{bundle_resources_dir}}/{{source_file_part}}" ]
}
}
}
rtc_library("videocodec_test_impl") {
testonly = true
sources = [
"codecs/test/videocodec_test_fixture_impl.cc",
"codecs/test/videocodec_test_fixture_impl.h",
]
deps = [
":codec_globals_headers",
":video_codec_interface",
":video_codecs_test_framework",
":video_coding_utility",
":videocodec_test_stats_impl",
":webrtc_vp9_helpers",
"../..:webrtc_common",
"../../api:array_view",
"../../api:videocodec_test_fixture_api",
"../../api/test/video:function_video_factory",
"../../api/video:video_bitrate_allocation",
"../../api/video_codecs:video_codecs_api",
"../../call:video_stream_api",
"../../common_video",
"../../media:rtc_audio_video",
"../../media:rtc_h264_profile_id",
"../../media:rtc_internal_video_codecs",
"../../media:rtc_media_base",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"../../rtc_base:rtc_base_tests_utils",
"../../rtc_base:task_queue_for_test",
"../../system_wrappers",
"../../test:fileutils",
"../../test:perf_test",
"../../test:test_support",
"../../test:video_test_common",
"../../test:video_test_support",
"//third_party/abseil-cpp/absl/types:optional",
]
}
rtc_library("videocodec_test_stats_impl") {
testonly = true
sources = [
"codecs/test/videocodec_test_stats_impl.cc",
"codecs/test/videocodec_test_stats_impl.h",
]
deps = [
"../../api:videocodec_test_fixture_api",
"../../rtc_base:checks",
"../../rtc_base:rtc_numerics",
"../../rtc_base:stringutils",
"../../test:test_common",
"../rtp_rtcp:rtp_rtcp_format",
]
}
rtc_library("video_coding_modules_tests") {
testonly = true
defines = []
sources = [
"codecs/h264/test/h264_impl_unittest.cc",
"codecs/multiplex/test/multiplex_adapter_unittest.cc",
"codecs/test/video_encoder_decoder_instantiation_tests.cc",
"codecs/test/videocodec_test_libvpx.cc",
"codecs/vp8/test/mock_libvpx_interface.h",
"codecs/vp8/test/vp8_impl_unittest.cc",
"codecs/vp9/test/vp9_impl_unittest.cc",
]
if (rtc_use_h264) {
sources += [ "codecs/test/videocodec_test_openh264.cc" ]
}
deps = [
":video_codec_interface",
":video_codecs_test_framework",
":video_coding_utility",
":videocodec_test_impl",
":webrtc_h264",
":webrtc_multiplex",
":webrtc_vp8",
":webrtc_vp9",
":webrtc_vp9_helpers",
"../..:webrtc_common",
"../../api:create_frame_generator",
"../../api:create_videocodec_test_fixture_api",
"../../api:frame_generator_api",
"../../api:mock_video_codec_factory",
"../../api:mock_video_decoder",
"../../api:mock_video_encoder",
"../../api:scoped_refptr",
"../../api:videocodec_test_fixture_api",
"../../api/test/video:function_video_factory",
"../../api/video:encoded_image",
"../../api/video:video_frame",
"../../api/video:video_frame_i420",
"../../api/video:video_rtp_headers",
"../../api/video_codecs:rtc_software_fallback_wrappers",
"../../api/video_codecs:video_codecs_api",
"../../common_video",
"../../common_video/test:utilities",
"../../media:rtc_h264_profile_id",
"../../media:rtc_internal_video_codecs",
"../../media:rtc_media_base",
"../../media:rtc_simulcast_encoder_adapter",
"../../media:rtc_vp9_profile",
"../../rtc_base",
"../../test:field_trial",
"../../test:fileutils",
"../../test:test_support",
"../../test:video_test_common",
"../rtp_rtcp:rtp_rtcp_format",
"//third_party/abseil-cpp/absl/types:optional",
"//third_party/libyuv",
]
data = video_coding_modules_tests_resources
if (is_android) {
sources += [ "codecs/test/videocodec_test_mediacodec.cc" ]
deps += [ ":android_codec_factory_helper" ]
}
if (is_ios || is_mac) {
sources += [ "codecs/test/videocodec_test_videotoolbox.cc" ]
deps += [ ":objc_codec_factory_helper" ]
if (num_video_coding_modules_tests_resources > 0) {
deps += [ ":video_coding_modules_tests_resources_bundle_data" ]
}
}
if (rtc_build_libvpx) {
deps += [ rtc_libvpx_dir ]
}
}
rtc_library("video_coding_unittests") {
testonly = true
sources = [
"codecs/test/videocodec_test_fixture_config_unittest.cc",
"codecs/test/videocodec_test_stats_impl_unittest.cc",
"codecs/test/videoprocessor_unittest.cc",
"codecs/vp8/default_temporal_layers_unittest.cc",
"codecs/vp8/libvpx_vp8_simulcast_test.cc",
"codecs/vp8/screenshare_layers_unittest.cc",
"codecs/vp9/svc_config_unittest.cc",
"codecs/vp9/svc_rate_allocator_unittest.cc",
"decoding_state_unittest.cc",
"fec_controller_unittest.cc",
"frame_buffer2_unittest.cc",
"frame_dependencies_calculator_unittest.cc",
"generic_decoder_unittest.cc",
"h264_sprop_parameter_sets_unittest.cc",
"h264_sps_pps_tracker_unittest.cc",
"histogram_unittest.cc",
"jitter_buffer_unittest.cc",
"jitter_estimator_tests.cc",
"loss_notification_controller_unittest.cc",
"nack_module_unittest.cc",
"packet_buffer_unittest.cc",
"receiver_unittest.cc",
"rtp_frame_reference_finder_unittest.cc",
"session_info_unittest.cc",
"test/stream_generator.cc",
"test/stream_generator.h",
"timing_unittest.cc",
"unique_timestamp_counter_unittest.cc",
"utility/decoded_frames_history_unittest.cc",
"utility/frame_dropper_unittest.cc",
"utility/framerate_controller_unittest.cc",
"utility/ivf_file_reader_unittest.cc",
"utility/ivf_file_writer_unittest.cc",
"utility/quality_scaler_unittest.cc",
"utility/simulcast_rate_allocator_unittest.cc",
"video_codec_initializer_unittest.cc",
"video_receiver_unittest.cc",
]
if (rtc_use_h264) {
sources += [
"codecs/h264/h264_encoder_impl_unittest.cc",
"codecs/h264/h264_simulcast_unittest.cc",
]
}
deps = [
":codec_globals_headers",
":encoded_frame",
":frame_dependencies_calculator",
":nack_module",
":simulcast_test_fixture_impl",
":video_codec_interface",
":video_codecs_test_framework",
":video_coding",
":video_coding_legacy",
":video_coding_utility",
":videocodec_test_impl",
":videocodec_test_stats_impl",
":webrtc_h264",
":webrtc_vp8",
":webrtc_vp8_temporal_layers",
":webrtc_vp9",
":webrtc_vp9_helpers",
"..:module_api",
"..:module_fec_api",
"../../api:array_view",
"../../api:create_simulcast_test_fixture_api",
"../../api:fec_controller_api",
"../../api:mock_fec_controller_override",
"../../api:mock_video_decoder",
"../../api:mock_video_encoder",
"../../api:scoped_refptr",
"../../api:simulcast_test_fixture_api",
"../../api:videocodec_test_fixture_api",
"../../api/task_queue:default_task_queue_factory",
"../../api/test/video:function_video_factory",
"../../api/video:builtin_video_bitrate_allocator_factory",
"../../api/video:video_adaptation",
"../../api/video:video_bitrate_allocation",
"../../api/video:video_bitrate_allocator",
"../../api/video:video_bitrate_allocator_factory",
"../../api/video:video_frame",
"../../api/video:video_frame_i420",
"../../api/video:video_frame_type",
"../../api/video:video_rtp_headers",
"../../api/video_codecs:video_codecs_api",
"../../api/video_codecs:vp8_temporal_layers_factory",
"../../common_video",
"../../common_video/generic_frame_descriptor",
"../../common_video/test:utilities",
"../../media:rtc_media_base",
"../../rtc_base",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"../../rtc_base:rtc_base_tests_utils",
"../../rtc_base:rtc_numerics",
"../../rtc_base:rtc_task_queue",
"../../rtc_base:task_queue_for_test",
"../../rtc_base/experiments:jitter_upper_bound_experiment",
"../../system_wrappers",
"../../system_wrappers:field_trial",
"../../system_wrappers:metrics",
"../../test:fake_video_codecs",
"../../test:field_trial",
"../../test:fileutils",
"../../test:test_common",
"../../test:test_support",
"../../test:video_test_common",
"../../test:video_test_support",
"../../test/time_controller:time_controller",
"../rtp_rtcp:rtp_rtcp_format",
"../rtp_rtcp:rtp_video_header",
"codecs/av1:video_coding_codecs_av1_tests",
"//third_party/abseil-cpp/absl/memory",
"//third_party/abseil-cpp/absl/types:optional",
"//third_party/abseil-cpp/absl/types:variant",
]
if (rtc_build_libvpx) {
deps += [ rtc_libvpx_dir ]
}
}
}

23
modules/video_coding/DEPS Normal file
View File

@ -0,0 +1,23 @@
include_rules = [
"+vpx",
"+call",
"+common_video",
"+sdk",
"+system_wrappers",
"+rtc_tools",
"+third_party/libyuv",
"+rtc_base/system/rtc_export.h",
]
specific_include_rules = {
"android_codec_factory_helper\.cc": [
"+base/android",
],
"multiplex_encoder_adapter\.cc": [
"+media/base",
],
".*test.*\.cc": [
"+media/base",
"+media/engine",
],
}

7
modules/video_coding/OWNERS.webrtc Normal file
View File

@ -0,0 +1,7 @@
asapersson@webrtc.org
brandtr@webrtc.org
ilnik@webrtc.org
marpan@webrtc.org
philipel@webrtc.org
sprang@webrtc.org
stefan@webrtc.org

59
modules/video_coding/codec_timer.cc Normal file
View File

@ -0,0 +1,59 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codec_timer.h"
#include <cstdint>
namespace webrtc {
namespace {
// The first kIgnoredSampleCount samples will be ignored.
const int kIgnoredSampleCount = 5;
// Return the |kPercentile| value in RequiredDecodeTimeMs().
const float kPercentile = 0.95f;
// The window size in ms.
const int64_t kTimeLimitMs = 10000;
} // anonymous namespace
VCMCodecTimer::VCMCodecTimer()
: ignored_sample_count_(0), filter_(kPercentile) {}
VCMCodecTimer::~VCMCodecTimer() = default;
void VCMCodecTimer::AddTiming(int64_t decode_time_ms, int64_t now_ms) {
// Ignore the first |kIgnoredSampleCount| samples.
if (ignored_sample_count_ < kIgnoredSampleCount) {
++ignored_sample_count_;
return;
}
// Insert new decode time value.
filter_.Insert(decode_time_ms);
history_.emplace(decode_time_ms, now_ms);
// Pop old decode time values.
while (!history_.empty() &&
now_ms - history_.front().sample_time_ms > kTimeLimitMs) {
filter_.Erase(history_.front().decode_time_ms);
history_.pop();
}
}
// Get the 95th percentile observed decode time within a time window.
int64_t VCMCodecTimer::RequiredDecodeTimeMs() const {
return filter_.GetPercentileValue();
}
VCMCodecTimer::Sample::Sample(int64_t decode_time_ms, int64_t sample_time_ms)
: decode_time_ms(decode_time_ms), sample_time_ms(sample_time_ms) {}
} // namespace webrtc

50
modules/video_coding/codec_timer.h Normal file
View File

@ -0,0 +1,50 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODEC_TIMER_H_
#define MODULES_VIDEO_CODING_CODEC_TIMER_H_
#include <queue>
#include "rtc_base/numerics/percentile_filter.h"
namespace webrtc {
class VCMCodecTimer {
public:
VCMCodecTimer();
~VCMCodecTimer();
// Add a new decode time to the filter.
void AddTiming(int64_t new_decode_time_ms, int64_t now_ms);
// Get the required decode time in ms. It is the 95th percentile observed
// decode time within a time window.
int64_t RequiredDecodeTimeMs() const;
private:
struct Sample {
Sample(int64_t decode_time_ms, int64_t sample_time_ms);
int64_t decode_time_ms;
int64_t sample_time_ms;
};
// The number of samples ignored so far.
int ignored_sample_count_;
// Queue with history of latest decode time values.
std::queue<Sample> history_;
// |filter_| contains the same values as |history_|, but in a data structure
// that allows efficient retrieval of the percentile value.
PercentileFilter<int64_t> filter_;
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODEC_TIMER_H_

88
modules/video_coding/codecs/av1/BUILD.gn Normal file
View File

@ -0,0 +1,88 @@
# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
import("//third_party/libaom/options.gni")
import("../../../../webrtc.gni")
rtc_library("libaom_av1_decoder") {
visibility = [ "*" ]
poisonous = [ "software_video_codecs" ]
public = [ "libaom_av1_decoder.h" ]
deps = [
"../../../../api/video_codecs:video_codecs_api",
"//third_party/abseil-cpp/absl/base:core_headers",
]
if (enable_libaom) {
sources = [ "libaom_av1_decoder.cc" ]
deps += [
"../..:video_codec_interface",
"../../../../api:scoped_refptr",
"../../../../api/video:encoded_image",
"../../../../api/video:video_frame_i420",
"../../../../common_video",
"../../../../rtc_base:logging",
"//third_party/abseil-cpp/absl/types:optional",
"//third_party/libaom",
"//third_party/libyuv",
]
} else {
sources = [ "libaom_av1_decoder_absent.cc" ]
}
}
rtc_library("libaom_av1_encoder") {
visibility = [ "*" ]
poisonous = [ "software_video_codecs" ]
public = [ "libaom_av1_encoder.h" ]
deps = [
"../../../../api/video_codecs:video_codecs_api",
"//third_party/abseil-cpp/absl/base:core_headers",
]
if (enable_libaom) {
sources = [ "libaom_av1_encoder.cc" ]
deps += [
"../..:video_codec_interface",
"../../../../api:scoped_refptr",
"../../../../api/video:encoded_image",
"../../../../api/video:video_frame",
"../../../../api/video:video_frame_i420",
"../../../../common_video",
"../../../../rtc_base:checks",
"../../../../rtc_base:logging",
"//third_party/abseil-cpp/absl/algorithm:container",
"//third_party/libaom",
]
} else {
sources = [ "libaom_av1_encoder_absent.cc" ]
}
}
if (rtc_include_tests) {
rtc_library("video_coding_codecs_av1_tests") {
testonly = true
if (enable_libaom) {
sources = [
"libaom_av1_encoder_unittest.cc",
"libaom_av1_unittest.cc",
]
deps = [
":libaom_av1_decoder",
":libaom_av1_encoder",
"../..:video_codec_interface",
"../../../../api:create_frame_generator",
"../../../../api:frame_generator_api",
"../../../../api/video_codecs:video_codecs_api",
"../../../../test:test_support",
"//third_party/abseil-cpp/absl/types:optional",
]
}
}
}

3
modules/video_coding/codecs/av1/DEPS Normal file
View File

@ -0,0 +1,3 @@
include_rules = [
"+third_party/libaom",
]

191
modules/video_coding/codecs/av1/libaom_av1_decoder.cc Normal file
View File

@ -0,0 +1,191 @@
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/av1/libaom_av1_decoder.h"
#include <stdint.h>
#include <memory>
#include "absl/types/optional.h"
#include "api/scoped_refptr.h"
#include "api/video/encoded_image.h"
#include "api/video/i420_buffer.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_decoder.h"
#include "common_video/include/i420_buffer_pool.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/logging.h"
#include "third_party/libaom/source/libaom/aom/aom_decoder.h"
#include "third_party/libaom/source/libaom/aom/aomdx.h"
#include "third_party/libyuv/include/libyuv/convert.h"
namespace webrtc {
namespace {
constexpr int kConfigLowBitDepth = 1; // 8-bits per luma/chroma sample.
constexpr int kDecFlags = 0; // 0 signals no post processing.
class LibaomAv1Decoder final : public VideoDecoder {
public:
LibaomAv1Decoder();
LibaomAv1Decoder(const LibaomAv1Decoder&) = delete;
LibaomAv1Decoder& operator=(const LibaomAv1Decoder&) = delete;
~LibaomAv1Decoder();
// Implements VideoDecoder.
int32_t InitDecode(const VideoCodec* codec_settings,
int number_of_cores) override;
// Decode an encoded video frame.
int32_t Decode(const EncodedImage& encoded_image,
bool missing_frames,
int64_t render_time_ms) override;
int32_t RegisterDecodeCompleteCallback(
DecodedImageCallback* callback) override;
int32_t Release() override;
private:
aom_codec_ctx_t context_;
bool inited_;
// Pool of memory buffers to store decoded image data for application access.
I420BufferPool buffer_pool_;
DecodedImageCallback* decode_complete_callback_;
};
LibaomAv1Decoder::LibaomAv1Decoder()
: context_(), // Force value initialization instead of default one.
inited_(false),
buffer_pool_(false, /*max_number_of_buffers=*/150),
decode_complete_callback_(nullptr) {}
LibaomAv1Decoder::~LibaomAv1Decoder() {
Release();
}
int32_t LibaomAv1Decoder::InitDecode(const VideoCodec* codec_settings,
int number_of_cores) {
aom_codec_dec_cfg_t config = {
static_cast<unsigned int>(number_of_cores), // Max # of threads.
0, // Frame width set after decode.
0, // Frame height set after decode.
kConfigLowBitDepth}; // Enable low-bit-depth code path.
aom_codec_err_t ret =
aom_codec_dec_init(&context_, aom_codec_av1_dx(), &config, kDecFlags);
if (ret != AOM_CODEC_OK) {
RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::InitDecode returned " << ret
<< " on aom_codec_dec_init.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
inited_ = true;
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t LibaomAv1Decoder::Decode(const EncodedImage& encoded_image,
bool missing_frames,
int64_t /*render_time_ms*/) {
if (!inited_) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
if (decode_complete_callback_ == nullptr) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
// Decode one video frame.
aom_codec_err_t ret =
aom_codec_decode(&context_, encoded_image.data(), encoded_image.size(),
/*user_priv=*/nullptr);
if (ret != AOM_CODEC_OK) {
RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::Decode returned " << ret
<< " on aom_codec_decode.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
// Get decoded frame data.
int corrupted_frame = 0;
aom_codec_iter_t iter = nullptr;
while (aom_image_t* decoded_image = aom_codec_get_frame(&context_, &iter)) {
if (aom_codec_control(&context_, AOMD_GET_FRAME_CORRUPTED,
&corrupted_frame)) {
RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::Decode "
"AOM_GET_FRAME_CORRUPTED.";
}
// Check that decoded image format is I420 and has 8-bit depth.
if (decoded_image->fmt != AOM_IMG_FMT_I420) {
RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::Decode invalid image format";
return WEBRTC_VIDEO_CODEC_ERROR;
}
// Return decoded frame data.
int qp;
ret = aom_codec_control_(&context_, AOMD_GET_LAST_QUANTIZER, &qp);
if (ret != AOM_CODEC_OK) {
RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::Decode returned " << ret
<< " on control AOME_GET_LAST_QUANTIZER.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
// Allocate memory for decoded frame.
rtc::scoped_refptr<I420Buffer> buffer =
buffer_pool_.CreateBuffer(decoded_image->d_w, decoded_image->d_h);
if (!buffer.get()) {
// Pool has too many pending frames.
RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::Decode returned due to lack of"
" space in decoded frame buffer pool.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
// Copy decoded_image to decoded_frame.
libyuv::I420Copy(
decoded_image->planes[AOM_PLANE_Y], decoded_image->stride[AOM_PLANE_Y],
decoded_image->planes[AOM_PLANE_U], decoded_image->stride[AOM_PLANE_U],
decoded_image->planes[AOM_PLANE_V], decoded_image->stride[AOM_PLANE_V],
buffer->MutableDataY(), buffer->StrideY(), buffer->MutableDataU(),
buffer->StrideU(), buffer->MutableDataV(), buffer->StrideV(),
decoded_image->d_w, decoded_image->d_h);
VideoFrame decoded_frame = VideoFrame::Builder()
.set_video_frame_buffer(buffer)
.set_timestamp_rtp(encoded_image.Timestamp())
.set_ntp_time_ms(encoded_image.ntp_time_ms_)
.set_color_space(encoded_image.ColorSpace())
.build();
decode_complete_callback_->Decoded(decoded_frame, absl::nullopt,
absl::nullopt);
}
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t LibaomAv1Decoder::RegisterDecodeCompleteCallback(
DecodedImageCallback* decode_complete_callback) {
decode_complete_callback_ = decode_complete_callback;
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t LibaomAv1Decoder::Release() {
if (aom_codec_destroy(&context_) != AOM_CODEC_OK) {
return WEBRTC_VIDEO_CODEC_MEMORY;
}
buffer_pool_.Release();
inited_ = false;
return WEBRTC_VIDEO_CODEC_OK;
}
} // namespace
const bool kIsLibaomAv1DecoderSupported = true;
std::unique_ptr<VideoDecoder> CreateLibaomAv1Decoder() {
return std::make_unique<LibaomAv1Decoder>();
}
} // namespace webrtc

26
modules/video_coding/codecs/av1/libaom_av1_decoder.h Normal file
View File

@ -0,0 +1,26 @@
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_AV1_LIBAOM_AV1_DECODER_H_
#define MODULES_VIDEO_CODING_CODECS_AV1_LIBAOM_AV1_DECODER_H_
#include <memory>
#include "absl/base/attributes.h"
#include "api/video_codecs/video_decoder.h"
namespace webrtc {
ABSL_CONST_INIT extern const bool kIsLibaomAv1DecoderSupported;
std::unique_ptr<VideoDecoder> CreateLibaomAv1Decoder();
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_AV1_LIBAOM_AV1_DECODER_H_

24
modules/video_coding/codecs/av1/libaom_av1_decoder_absent.cc Normal file
View File

@ -0,0 +1,24 @@
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/av1/libaom_av1_decoder.h"
#include <memory>
#include "api/video_codecs/video_decoder.h"
namespace webrtc {
const bool kIsLibaomAv1DecoderSupported = false;
std::unique_ptr<VideoDecoder> CreateLibaomAv1Decoder() {
return nullptr;
}
} // namespace webrtc

395
modules/video_coding/codecs/av1/libaom_av1_encoder.cc Normal file
View File

@ -0,0 +1,395 @@
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
#include <stddef.h>
#include <stdint.h>
#include <memory>
#include <vector>
#include "absl/algorithm/container.h"
#include "api/scoped_refptr.h"
#include "api/video/encoded_image.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_frame.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_encoder.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "third_party/libaom/source/libaom/aom/aom_codec.h"
#include "third_party/libaom/source/libaom/aom/aom_encoder.h"
#include "third_party/libaom/source/libaom/aom/aomcx.h"
namespace webrtc {
namespace {
// Encoder configuration parameters
constexpr int kQpMax = 56;
constexpr int kQpMin = 10;
constexpr int kDefaultEncSpeed = 7; // Use values 6, 7, or 8 for RTC.
constexpr int kUsageProfile = 1; // 0 = good quality; 1 = real-time.
constexpr int kMinQindex = 58; // Min qindex threshold for QP scaling.
constexpr int kMaxQindex = 180; // Max qindex threshold for QP scaling.
constexpr int kBitDepth = 8;
constexpr int kLagInFrames = 0; // No look ahead.
constexpr int kRtpTicksPerSecond = 90000;
constexpr float kMinimumFrameRate = 1.0;
class LibaomAv1Encoder final : public VideoEncoder {
public:
LibaomAv1Encoder();
~LibaomAv1Encoder();
int InitEncode(const VideoCodec* codec_settings,
const Settings& settings) override;
int32_t RegisterEncodeCompleteCallback(
EncodedImageCallback* encoded_image_callback) override;
int32_t Release() override;
int32_t Encode(const VideoFrame& frame,
const std::vector<VideoFrameType>* frame_types) override;
void SetRates(const RateControlParameters& parameters) override;
EncoderInfo GetEncoderInfo() const override;
private:
bool inited_;
bool keyframe_required_;
VideoCodec encoder_settings_;
aom_image_t* frame_for_encode_;
aom_codec_ctx_t ctx_;
aom_codec_enc_cfg_t cfg_;
EncodedImageCallback* encoded_image_callback_;
};
int32_t VerifyCodecSettings(const VideoCodec& codec_settings) {
if (codec_settings.width < 1) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (codec_settings.height < 1) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
// maxBitrate == 0 represents an unspecified maxBitRate.
if (codec_settings.maxBitrate > 0 &&
codec_settings.minBitrate > codec_settings.maxBitrate) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (codec_settings.maxBitrate > 0 &&
codec_settings.startBitrate > codec_settings.maxBitrate) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (codec_settings.startBitrate < codec_settings.minBitrate) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (codec_settings.maxFramerate < 1) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
return WEBRTC_VIDEO_CODEC_OK;
}
LibaomAv1Encoder::LibaomAv1Encoder()
: inited_(false),
keyframe_required_(true),
frame_for_encode_(nullptr),
encoded_image_callback_(nullptr) {}
LibaomAv1Encoder::~LibaomAv1Encoder() {
Release();
}
int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings,
const Settings& settings) {
if (codec_settings == nullptr) {
RTC_LOG(LS_WARNING) << "No codec settings provided to "
"LibaomAv1Encoder.";
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (settings.number_of_cores < 1) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (inited_) {
RTC_LOG(LS_WARNING) << "Initing LibaomAv1Encoder without first releasing.";
Release();
}
encoder_settings_ = *codec_settings;
// Sanity checks for encoder configuration.
const int32_t result = VerifyCodecSettings(encoder_settings_);
if (result < 0) {
RTC_LOG(LS_WARNING) << "Incorrect codec settings provided to "
"LibaomAv1Encoder.";
return result;
}
// Initialize encoder configuration structure with default values
aom_codec_err_t ret =
aom_codec_enc_config_default(aom_codec_av1_cx(), &cfg_, 0);
if (ret != AOM_CODEC_OK) {
RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
<< " on aom_codec_enc_config_default.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
// Overwrite default config with input encoder settings & RTC-relevant values.
cfg_.g_w = encoder_settings_.width;
cfg_.g_h = encoder_settings_.height;
cfg_.g_threads = settings.number_of_cores;
cfg_.g_timebase.num = 1;
cfg_.g_timebase.den = kRtpTicksPerSecond;
cfg_.rc_target_bitrate = encoder_settings_.maxBitrate; // kilobits/sec.
cfg_.g_input_bit_depth = kBitDepth;
cfg_.kf_mode = AOM_KF_DISABLED;
cfg_.rc_min_quantizer = kQpMin;
cfg_.rc_max_quantizer = kQpMax;
cfg_.g_usage = kUsageProfile;
// Low-latency settings.
cfg_.rc_end_usage = AOM_CBR; // Constant Bit Rate (CBR) mode
cfg_.g_pass = AOM_RC_ONE_PASS; // One-pass rate control
cfg_.g_lag_in_frames = kLagInFrames; // No look ahead when lag equals 0.
// Creating a wrapper to the image - setting image data to nullptr. Actual
// pointer will be set in encode. Setting align to 1, as it is meaningless
// (actual memory is not allocated).
frame_for_encode_ =
aom_img_alloc(nullptr, AOM_IMG_FMT_I420, cfg_.g_w, cfg_.g_h, 1);
// Flag options: AOM_CODEC_USE_PSNR and AOM_CODEC_USE_HIGHBITDEPTH
aom_codec_flags_t flags = 0;
// Initialize an encoder instance.
ret = aom_codec_enc_init(&ctx_, aom_codec_av1_cx(), &cfg_, flags);
if (ret != AOM_CODEC_OK) {
RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
<< " on aom_codec_enc_init.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
inited_ = true;
// Set control parameters
ret = aom_codec_control(&ctx_, AOME_SET_CPUUSED, kDefaultEncSpeed);
if (ret != AOM_CODEC_OK) {
RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
<< " on control AV1E_SET_CPUUSED.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_TPL_MODEL, 0);
if (ret != AOM_CODEC_OK) {
RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
<< " on control AV1E_SET_ENABLE_TPL_MODEL.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
ret = aom_codec_control(&ctx_, AV1E_SET_DELTAQ_MODE, 0);
if (ret != AOM_CODEC_OK) {
RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
<< " on control AV1E_SET_DELTAQ_MODE.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
ret = aom_codec_control(&ctx_, AV1E_SET_AQ_MODE, 3);
if (ret != AOM_CODEC_OK) {
RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
<< " on control AV1E_SET_AQ_MODE.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t LibaomAv1Encoder::RegisterEncodeCompleteCallback(
EncodedImageCallback* encoded_image_callback) {
encoded_image_callback_ = encoded_image_callback;
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t LibaomAv1Encoder::Release() {
if (frame_for_encode_ != nullptr) {
aom_img_free(frame_for_encode_);
frame_for_encode_ = nullptr;
}
if (inited_) {
if (aom_codec_destroy(&ctx_)) {
return WEBRTC_VIDEO_CODEC_MEMORY;
}
inited_ = false;
}
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t LibaomAv1Encoder::Encode(
const VideoFrame& frame,
const std::vector<VideoFrameType>* frame_types) {
if (!inited_ || encoded_image_callback_ == nullptr) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
keyframe_required_ =
frame_types != nullptr &&
absl::c_linear_search(*frame_types, VideoFrameType::kVideoFrameKey);
// Convert input frame to I420, if needed.
VideoFrame prepped_input_frame = frame;
if (prepped_input_frame.video_frame_buffer()->type() !=
VideoFrameBuffer::Type::kI420) {
rtc::scoped_refptr<I420BufferInterface> converted_buffer(
prepped_input_frame.video_frame_buffer()->ToI420());
prepped_input_frame = VideoFrame(converted_buffer, frame.timestamp(),
frame.render_time_ms(), frame.rotation());
}
// Set frame_for_encode_ data pointers and strides.
auto i420_buffer = prepped_input_frame.video_frame_buffer()->GetI420();
frame_for_encode_->planes[AOM_PLANE_Y] =
const_cast<unsigned char*>(i420_buffer->DataY());
frame_for_encode_->planes[AOM_PLANE_U] =
const_cast<unsigned char*>(i420_buffer->DataU());
frame_for_encode_->planes[AOM_PLANE_V] =
const_cast<unsigned char*>(i420_buffer->DataV());
frame_for_encode_->stride[AOM_PLANE_Y] = i420_buffer->StrideY();
frame_for_encode_->stride[AOM_PLANE_U] = i420_buffer->StrideU();
frame_for_encode_->stride[AOM_PLANE_V] = i420_buffer->StrideV();
const uint32_t duration =
kRtpTicksPerSecond / static_cast<float>(encoder_settings_.maxFramerate);
aom_enc_frame_flags_t flags = (keyframe_required_) ? AOM_EFLAG_FORCE_KF : 0;
// Encode a frame.
aom_codec_err_t ret = aom_codec_encode(&ctx_, frame_for_encode_,
frame.timestamp(), duration, flags);
if (ret != AOM_CODEC_OK) {
RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret
<< " on aom_codec_encode.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
// Get encoded image data.
EncodedImage encoded_image;
encoded_image._completeFrame = true;
aom_codec_iter_t iter = nullptr;
int data_pkt_count = 0;
while (const aom_codec_cx_pkt_t* pkt = aom_codec_get_cx_data(&ctx_, &iter)) {
if (pkt->kind == AOM_CODEC_CX_FRAME_PKT && pkt->data.frame.sz > 0) {
if (data_pkt_count > 0) {
RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encoder returned more than "
"one data packet for an input video frame.";
Release();
}
// TODO(bugs.webrtc.org/11174): Remove this hack when
// webrtc_pc_e2e::SingleProcessEncodedImageDataInjector not used or fixed
// not to assume that encoded image transfered as is.
const uint8_t* data = static_cast<const uint8_t*>(pkt->data.frame.buf);
size_t size = pkt->data.frame.sz;
if (size > 2 && data[0] == 0b0'0010'010 && data[1] == 0) {
// Typically frame starts with a Temporal Delimter OBU of size 0 that is
// not need by any component in webrtc and discarded during rtp
// packetization. Before discarded it confuses test framework that
// assumes received encoded frame is exactly same as sent frame.
data += 2;
size -= 2;
}
encoded_image.SetEncodedData(EncodedImageBuffer::Create(data, size));
bool is_key_frame = ((pkt->data.frame.flags & AOM_EFLAG_FORCE_KF) != 0);
encoded_image._frameType = is_key_frame
? VideoFrameType::kVideoFrameKey
: VideoFrameType::kVideoFrameDelta;
encoded_image.SetTimestamp(frame.timestamp());
encoded_image.capture_time_ms_ = frame.render_time_ms();
encoded_image.rotation_ = frame.rotation();
encoded_image.content_type_ = VideoContentType::UNSPECIFIED;
// If encoded image width/height info are added to aom_codec_cx_pkt_t,
// use those values in lieu of the values in frame.
encoded_image._encodedHeight = frame.height();
encoded_image._encodedWidth = frame.width();
encoded_image.timing_.flags = VideoSendTiming::kInvalid;
int qp = -1;
ret = aom_codec_control(&ctx_, AOME_GET_LAST_QUANTIZER, &qp);
if (ret != AOM_CODEC_OK) {
RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret
<< " on control AOME_GET_LAST_QUANTIZER.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
encoded_image.qp_ = qp;
encoded_image.SetColorSpace(frame.color_space());
++data_pkt_count;
}
}
// Deliver encoded image data.
if (encoded_image.size() > 0) {
CodecSpecificInfo codec_specific_info;
encoded_image_callback_->OnEncodedImage(encoded_image, &codec_specific_info,
nullptr);
}
return WEBRTC_VIDEO_CODEC_OK;
}
void LibaomAv1Encoder::SetRates(const RateControlParameters& parameters) {
if (!inited_) {
RTC_LOG(LS_WARNING) << "SetRates() while encoder is not initialized";
return;
}
if (parameters.framerate_fps < kMinimumFrameRate) {
RTC_LOG(LS_WARNING) << "Unsupported framerate (must be >= "
<< kMinimumFrameRate
<< " ): " << parameters.framerate_fps;
return;
}
if (parameters.bitrate.get_sum_bps() == 0) {
RTC_LOG(LS_WARNING) << "Attempt to set target bit rate to zero";
return;
}
// Check input target bit rate value.
uint32_t rc_target_bitrate_kbps = parameters.bitrate.get_sum_kbps();
if (encoder_settings_.maxBitrate > 0)
RTC_DCHECK_LE(rc_target_bitrate_kbps, encoder_settings_.maxBitrate);
RTC_DCHECK_GE(rc_target_bitrate_kbps, encoder_settings_.minBitrate);
// Set target bit rate.
cfg_.rc_target_bitrate = rc_target_bitrate_kbps;
// Set frame rate to closest integer value.
encoder_settings_.maxFramerate =
static_cast<uint32_t>(parameters.framerate_fps + 0.5);
// Update encoder context.
aom_codec_err_t error_code = aom_codec_enc_config_set(&ctx_, &cfg_);
if (error_code != AOM_CODEC_OK) {
RTC_LOG(LS_WARNING) << "Error configuring encoder, error code: "
<< error_code;
}
}
VideoEncoder::EncoderInfo LibaomAv1Encoder::GetEncoderInfo() const {
EncoderInfo info;
info.supports_native_handle = false;
info.implementation_name = "libaom";
info.has_trusted_rate_controller = true;
info.is_hardware_accelerated = false;
info.scaling_settings = VideoEncoder::ScalingSettings(kMinQindex, kMaxQindex);
return info;
}
} // namespace
const bool kIsLibaomAv1EncoderSupported = true;
std::unique_ptr<VideoEncoder> CreateLibaomAv1Encoder() {
return std::make_unique<LibaomAv1Encoder>();
}
} // namespace webrtc

26
modules/video_coding/codecs/av1/libaom_av1_encoder.h Normal file
View File

@ -0,0 +1,26 @@
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_AV1_LIBAOM_AV1_ENCODER_H_
#define MODULES_VIDEO_CODING_CODECS_AV1_LIBAOM_AV1_ENCODER_H_
#include <memory>
#include "absl/base/attributes.h"
#include "api/video_codecs/video_encoder.h"
namespace webrtc {
ABSL_CONST_INIT extern const bool kIsLibaomAv1EncoderSupported;
std::unique_ptr<VideoEncoder> CreateLibaomAv1Encoder();
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_AV1_LIBAOM_AV1_ENCODER_H_

24
modules/video_coding/codecs/av1/libaom_av1_encoder_absent.cc Normal file
View File

@ -0,0 +1,24 @@
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
#include <memory>
#include "api/video_codecs/video_encoder.h"
namespace webrtc {
const bool kIsLibaomAv1EncoderSupported = false;
std::unique_ptr<VideoEncoder> CreateLibaomAv1Encoder() {
return nullptr;
}
} // namespace webrtc

44
modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
#include <memory>
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_encoder.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "test/gtest.h"
namespace webrtc {
namespace {
TEST(LibaomAv1EncoderTest, CanCreate) {
std::unique_ptr<VideoEncoder> encoder = CreateLibaomAv1Encoder();
EXPECT_TRUE(encoder);
}
TEST(LibaomAv1EncoderTest, InitAndRelease) {
std::unique_ptr<VideoEncoder> encoder = CreateLibaomAv1Encoder();
ASSERT_TRUE(encoder);
VideoCodec codec_settings;
codec_settings.width = 1280;
codec_settings.height = 720;
codec_settings.maxFramerate = 30;
VideoEncoder::Capabilities capabilities(/*loss_notification=*/false);
VideoEncoder::Settings encoder_settings(capabilities, /*number_of_cores=*/1,
/*max_payload_size=*/1200);
EXPECT_EQ(encoder->InitEncode(&codec_settings, encoder_settings),
WEBRTC_VIDEO_CODEC_OK);
EXPECT_EQ(encoder->Release(), WEBRTC_VIDEO_CODEC_OK);
}
} // namespace
} // namespace webrtc

206
modules/video_coding/codecs/av1/libaom_av1_unittest.cc Normal file
View File

@ -0,0 +1,206 @@
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <stddef.h>
#include <stdint.h>
#include <memory>
#include <vector>
#include "absl/types/optional.h"
#include "api/test/create_frame_generator.h"
#include "api/test/frame_generator_interface.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_encoder.h"
#include "modules/video_coding/codecs/av1/libaom_av1_decoder.h"
#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "test/gmock.h"
#include "test/gtest.h"
namespace webrtc {
namespace {
using ::testing::ElementsAreArray;
using ::testing::IsEmpty;
using ::testing::Not;
using ::testing::NotNull;
// Use small resolution for this test to make it faster.
constexpr int kWidth = 320;
constexpr int kHeight = 180;
constexpr int kFramerate = 30;
constexpr int kRtpTicksPerSecond = 90000;
class TestAv1Encoder {
public:
struct Encoded {
EncodedImage encoded_image;
CodecSpecificInfo codec_specific_info;
};
TestAv1Encoder() : encoder_(CreateLibaomAv1Encoder()) {
RTC_CHECK(encoder_);
VideoCodec codec_settings;
codec_settings.width = kWidth;
codec_settings.height = kHeight;
codec_settings.maxFramerate = kFramerate;
VideoEncoder::Settings encoder_settings(
VideoEncoder::Capabilities(/*loss_notification=*/false),
/*number_of_cores=*/1, /*max_payload_size=*/1200);
EXPECT_EQ(encoder_->InitEncode(&codec_settings, encoder_settings),
WEBRTC_VIDEO_CODEC_OK);
EXPECT_EQ(encoder_->RegisterEncodeCompleteCallback(&callback_),
WEBRTC_VIDEO_CODEC_OK);
}
// This class requires pointer stability and thus not copyable nor movable.
TestAv1Encoder(const TestAv1Encoder&) = delete;
TestAv1Encoder& operator=(const TestAv1Encoder&) = delete;
void EncodeAndAppend(const VideoFrame& frame, std::vector<Encoded>* encoded) {
callback_.SetEncodeStorage(encoded);
std::vector<VideoFrameType> frame_types = {
VideoFrameType::kVideoFrameDelta};
EXPECT_EQ(encoder_->Encode(frame, &frame_types), WEBRTC_VIDEO_CODEC_OK);
// Prefer to crash checking nullptr rather than writing to random memory.
callback_.SetEncodeStorage(nullptr);
}
private:
class EncoderCallback : public EncodedImageCallback {
public:
void SetEncodeStorage(std::vector<Encoded>* storage) { storage_ = storage; }
private:
Result OnEncodedImage(
const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific_info,
const RTPFragmentationHeader* /*fragmentation*/) override {
RTC_CHECK(storage_);
storage_->push_back({encoded_image, *codec_specific_info});
return Result(Result::Error::OK);
}
std::vector<Encoded>* storage_ = nullptr;
};
EncoderCallback callback_;
std::unique_ptr<VideoEncoder> encoder_;
};
class TestAv1Decoder {
public:
TestAv1Decoder() {
decoder_ = CreateLibaomAv1Decoder();
if (decoder_ == nullptr) {
ADD_FAILURE() << "Failed to create a decoder";
return;
}
EXPECT_EQ(decoder_->InitDecode(/*codec_settings=*/nullptr,
/*number_of_cores=*/1),
WEBRTC_VIDEO_CODEC_OK);
EXPECT_EQ(decoder_->RegisterDecodeCompleteCallback(&callback_),
WEBRTC_VIDEO_CODEC_OK);
}
// This class requires pointer stability and thus not copyable nor movable.
TestAv1Decoder(const TestAv1Decoder&) = delete;
TestAv1Decoder& operator=(const TestAv1Decoder&) = delete;
void Decode(int64_t frame_id, const EncodedImage& image) {
ASSERT_THAT(decoder_, NotNull());
requested_ids_.push_back(frame_id);
int32_t error = decoder_->Decode(image, /*missing_frames=*/false,
/*render_time_ms=*/image.capture_time_ms_);
if (error != WEBRTC_VIDEO_CODEC_OK) {
ADD_FAILURE() << "Failed to decode frame id " << frame_id
<< " with error code " << error;
return;
}
decoded_ids_.push_back(frame_id);
}
const std::vector<int64_t>& requested_frame_ids() const {
return requested_ids_;
}
const std::vector<int64_t>& decoded_frame_ids() const { return decoded_ids_; }
size_t num_output_frames() const { return callback_.num_called(); }
private:
// Decoder callback that only counts how many times it was called.
// While it is tempting to replace it with a simple mock, that one requires
// to set expectation on number of calls in advance. Tests below unsure about
// expected number of calls until after calls are done.
class DecoderCallback : public DecodedImageCallback {
public:
size_t num_called() const { return num_called_; }
private:
int32_t Decoded(VideoFrame& /*decoded_image*/) override {
++num_called_;
return 0;
}
void Decoded(VideoFrame& /*decoded_image*/,
absl::optional<int32_t> /*decode_time_ms*/,
absl::optional<uint8_t> /*qp*/) override {
++num_called_;
}
int num_called_ = 0;
};
std::vector<int64_t> requested_ids_;
std::vector<int64_t> decoded_ids_;
DecoderCallback callback_;
std::unique_ptr<VideoDecoder> decoder_;
};
std::vector<VideoFrame> GenerateFrames(size_t num_frames) {
std::vector<VideoFrame> frames;
frames.reserve(num_frames);
auto input_frame_generator = test::CreateSquareFrameGenerator(
kWidth, kHeight, test::FrameGeneratorInterface::OutputType::kI420,
absl::nullopt);
uint32_t timestamp = 1000;
for (size_t i = 0; i < num_frames; ++i) {
frames.push_back(
VideoFrame::Builder()
.set_video_frame_buffer(input_frame_generator->NextFrame().buffer)
.set_timestamp_rtp(timestamp += kRtpTicksPerSecond / kFramerate)
.build());
}
return frames;
}
TEST(LibaomAv1Test, EncodeDecode) {
TestAv1Decoder decoder;
TestAv1Encoder encoder;
std::vector<TestAv1Encoder::Encoded> encoded_frames;
for (const VideoFrame& frame : GenerateFrames(/*num_frames=*/4)) {
encoder.EncodeAndAppend(frame, &encoded_frames);
}
for (size_t frame_idx = 0; frame_idx < encoded_frames.size(); ++frame_idx) {
decoder.Decode(static_cast<int64_t>(frame_idx),
encoded_frames[frame_idx].encoded_image);
}
// Check encoder produced some frames for decoder to decode.
ASSERT_THAT(encoded_frames, Not(IsEmpty()));
// Check decoder found all of them valid.
EXPECT_THAT(decoder.decoded_frame_ids(),
ElementsAreArray(decoder.requested_frame_ids()));
// Check each of them produced an output frame.
EXPECT_EQ(decoder.num_output_frames(), decoder.decoded_frame_ids().size());
}
} // namespace
} // namespace webrtc

5
modules/video_coding/codecs/h264/DEPS Normal file
View File

@ -0,0 +1,5 @@
include_rules = [
"+third_party/ffmpeg",
"+third_party/openh264",
"+media/base",
]

2
modules/video_coding/codecs/h264/OWNERS.webrtc Normal file
View File

@ -0,0 +1,2 @@
sprang@webrtc.org
ssilkin@webrtc.org

120
modules/video_coding/codecs/h264/h264.cc Normal file
View File

@ -0,0 +1,120 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#include "modules/video_coding/codecs/h264/include/h264.h"
#include <memory>
#include <string>
#include "absl/types/optional.h"
#include "api/video_codecs/sdp_video_format.h"
#include "media/base/media_constants.h"
#if defined(WEBRTC_USE_H264)
#include "modules/video_coding/codecs/h264/h264_decoder_impl.h"
#include "modules/video_coding/codecs/h264/h264_encoder_impl.h"
#endif
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
namespace webrtc {
namespace {
#if defined(WEBRTC_USE_H264)
bool g_rtc_use_h264 = true;
#endif
// If H.264 OpenH264/FFmpeg codec is supported.
bool IsH264CodecSupported() {
#if defined(WEBRTC_USE_H264)
return g_rtc_use_h264;
#else
return false;
#endif
}
} // namespace
SdpVideoFormat CreateH264Format(H264::Profile profile,
H264::Level level,
const std::string& packetization_mode) {
const absl::optional<std::string> profile_string =
H264::ProfileLevelIdToString(H264::ProfileLevelId(profile, level));
RTC_CHECK(profile_string);
return SdpVideoFormat(
cricket::kH264CodecName,
{{cricket::kH264FmtpProfileLevelId, *profile_string},
{cricket::kH264FmtpLevelAsymmetryAllowed, "1"},
{cricket::kH264FmtpPacketizationMode, packetization_mode}});
}
void DisableRtcUseH264() {
#if defined(WEBRTC_USE_H264)
g_rtc_use_h264 = false;
#endif
}
std::vector<SdpVideoFormat> SupportedH264Codecs() {
if (!IsH264CodecSupported())
return std::vector<SdpVideoFormat>();
// We only support encoding Constrained Baseline Profile (CBP), but the
// decoder supports more profiles. We can list all profiles here that are
// supported by the decoder and that are also supersets of CBP, i.e. the
// decoder for that profile is required to be able to decode CBP. This means
// we can encode and send CBP even though we negotiated a potentially
// higher profile. See the H264 spec for more information.
//
// We support both packetization modes 0 (mandatory) and 1 (optional,
// preferred).
return {
CreateH264Format(H264::kProfileBaseline, H264::kLevel3_1, "1"),
CreateH264Format(H264::kProfileBaseline, H264::kLevel3_1, "0"),
CreateH264Format(H264::kProfileConstrainedBaseline, H264::kLevel3_1, "1"),
CreateH264Format(H264::kProfileConstrainedBaseline, H264::kLevel3_1,
"0")};
}
std::unique_ptr<H264Encoder> H264Encoder::Create(
const cricket::VideoCodec& codec) {
RTC_DCHECK(H264Encoder::IsSupported());
#if defined(WEBRTC_USE_H264)
RTC_CHECK(g_rtc_use_h264);
RTC_LOG(LS_INFO) << "Creating H264EncoderImpl.";
return std::make_unique<H264EncoderImpl>(codec);
#else
RTC_NOTREACHED();
return nullptr;
#endif
}
bool H264Encoder::IsSupported() {
return IsH264CodecSupported();
}
std::unique_ptr<H264Decoder> H264Decoder::Create() {
RTC_DCHECK(H264Decoder::IsSupported());
#if defined(WEBRTC_USE_H264)
RTC_CHECK(g_rtc_use_h264);
RTC_LOG(LS_INFO) << "Creating H264DecoderImpl.";
return std::make_unique<H264DecoderImpl>();
#else
RTC_NOTREACHED();
return nullptr;
#endif
}
bool H264Decoder::IsSupported() {
return IsH264CodecSupported();
}
} // namespace webrtc

178
modules/video_coding/codecs/h264/h264_color_space.cc Normal file
View File

@ -0,0 +1,178 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
// Everything declared/defined in this header is only required when WebRTC is
// build with H264 support, please do not move anything out of the
// #ifdef unless needed and tested.
#ifdef WEBRTC_USE_H264
#include "modules/video_coding/codecs/h264/h264_color_space.h"
namespace webrtc {
ColorSpace ExtractH264ColorSpace(AVCodecContext* codec) {
ColorSpace::PrimaryID primaries = ColorSpace::PrimaryID::kUnspecified;
switch (codec->color_primaries) {
case AVCOL_PRI_BT709:
primaries = ColorSpace::PrimaryID::kBT709;
break;
case AVCOL_PRI_BT470M:
primaries = ColorSpace::PrimaryID::kBT470M;
break;
case AVCOL_PRI_BT470BG:
primaries = ColorSpace::PrimaryID::kBT470BG;
break;
case AVCOL_PRI_SMPTE170M:
primaries = ColorSpace::PrimaryID::kSMPTE170M;
break;
case AVCOL_PRI_SMPTE240M:
primaries = ColorSpace::PrimaryID::kSMPTE240M;
break;
case AVCOL_PRI_FILM:
primaries = ColorSpace::PrimaryID::kFILM;
break;
case AVCOL_PRI_BT2020:
primaries = ColorSpace::PrimaryID::kBT2020;
break;
case AVCOL_PRI_SMPTE428:
primaries = ColorSpace::PrimaryID::kSMPTEST428;
break;
case AVCOL_PRI_SMPTE431:
primaries = ColorSpace::PrimaryID::kSMPTEST431;
break;
case AVCOL_PRI_SMPTE432:
primaries = ColorSpace::PrimaryID::kSMPTEST432;
break;
case AVCOL_PRI_JEDEC_P22:
primaries = ColorSpace::PrimaryID::kJEDECP22;
break;
case AVCOL_PRI_RESERVED0:
case AVCOL_PRI_UNSPECIFIED:
case AVCOL_PRI_RESERVED:
default:
break;
}
ColorSpace::TransferID transfer = ColorSpace::TransferID::kUnspecified;
switch (codec->color_trc) {
case AVCOL_TRC_BT709:
transfer = ColorSpace::TransferID::kBT709;
break;
case AVCOL_TRC_GAMMA22:
transfer = ColorSpace::TransferID::kGAMMA22;
break;
case AVCOL_TRC_GAMMA28:
transfer = ColorSpace::TransferID::kGAMMA28;
break;
case AVCOL_TRC_SMPTE170M:
transfer = ColorSpace::TransferID::kSMPTE170M;
break;
case AVCOL_TRC_SMPTE240M:
transfer = ColorSpace::TransferID::kSMPTE240M;
break;
case AVCOL_TRC_LINEAR:
transfer = ColorSpace::TransferID::kLINEAR;
break;
case AVCOL_TRC_LOG:
transfer = ColorSpace::TransferID::kLOG;
break;
case AVCOL_TRC_LOG_SQRT:
transfer = ColorSpace::TransferID::kLOG_SQRT;
break;
case AVCOL_TRC_IEC61966_2_4:
transfer = ColorSpace::TransferID::kIEC61966_2_4;
break;
case AVCOL_TRC_BT1361_ECG:
transfer = ColorSpace::TransferID::kBT1361_ECG;
break;
case AVCOL_TRC_IEC61966_2_1:
transfer = ColorSpace::TransferID::kIEC61966_2_1;
break;
case AVCOL_TRC_BT2020_10:
transfer = ColorSpace::TransferID::kBT2020_10;
break;
case AVCOL_TRC_BT2020_12:
transfer = ColorSpace::TransferID::kBT2020_12;
break;
case AVCOL_TRC_SMPTE2084:
transfer = ColorSpace::TransferID::kSMPTEST2084;
break;
case AVCOL_TRC_SMPTE428:
transfer = ColorSpace::TransferID::kSMPTEST428;
break;
case AVCOL_TRC_ARIB_STD_B67:
transfer = ColorSpace::TransferID::kARIB_STD_B67;
break;
case AVCOL_TRC_RESERVED0:
case AVCOL_TRC_UNSPECIFIED:
case AVCOL_TRC_RESERVED:
default:
break;
}
ColorSpace::MatrixID matrix = ColorSpace::MatrixID::kUnspecified;
switch (codec->colorspace) {
case AVCOL_SPC_RGB:
matrix = ColorSpace::MatrixID::kRGB;
break;
case AVCOL_SPC_BT709:
matrix = ColorSpace::MatrixID::kBT709;
break;
case AVCOL_SPC_FCC:
matrix = ColorSpace::MatrixID::kFCC;
break;
case AVCOL_SPC_BT470BG:
matrix = ColorSpace::MatrixID::kBT470BG;
break;
case AVCOL_SPC_SMPTE170M:
matrix = ColorSpace::MatrixID::kSMPTE170M;
break;
case AVCOL_SPC_SMPTE240M:
matrix = ColorSpace::MatrixID::kSMPTE240M;
break;
case AVCOL_SPC_YCGCO:
matrix = ColorSpace::MatrixID::kYCOCG;
break;
case AVCOL_SPC_BT2020_NCL:
matrix = ColorSpace::MatrixID::kBT2020_NCL;
break;
case AVCOL_SPC_BT2020_CL:
matrix = ColorSpace::MatrixID::kBT2020_CL;
break;
case AVCOL_SPC_SMPTE2085:
matrix = ColorSpace::MatrixID::kSMPTE2085;
break;
case AVCOL_SPC_CHROMA_DERIVED_NCL:
case AVCOL_SPC_CHROMA_DERIVED_CL:
case AVCOL_SPC_ICTCP:
case AVCOL_SPC_UNSPECIFIED:
case AVCOL_SPC_RESERVED:
default:
break;
}
ColorSpace::RangeID range = ColorSpace::RangeID::kInvalid;
switch (codec->color_range) {
case AVCOL_RANGE_MPEG:
range = ColorSpace::RangeID::kLimited;
break;
case AVCOL_RANGE_JPEG:
range = ColorSpace::RangeID::kFull;
break;
case AVCOL_RANGE_UNSPECIFIED:
default:
break;
}
return ColorSpace(primaries, transfer, matrix, range);
}
} // namespace webrtc
#endif // WEBRTC_USE_H264

38
modules/video_coding/codecs/h264/h264_color_space.h Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_H264_H264_COLOR_SPACE_H_
#define MODULES_VIDEO_CODING_CODECS_H264_H264_COLOR_SPACE_H_
// Everything declared in this header is only required when WebRTC is
// build with H264 support, please do not move anything out of the
// #ifdef unless needed and tested.
#ifdef WEBRTC_USE_H264
#if defined(WEBRTC_WIN) && !defined(__clang__)
#error "See: bugs.webrtc.org/9213#c13."
#endif
#include "api/video/color_space.h"
extern "C" {
#include "third_party/ffmpeg/libavcodec/avcodec.h"
} // extern "C"
namespace webrtc {
// Helper class for extracting color space information from H264 stream.
ColorSpace ExtractH264ColorSpace(AVCodecContext* codec);
} // namespace webrtc
#endif // WEBRTC_USE_H264
#endif // MODULES_VIDEO_CODING_CODECS_H264_H264_COLOR_SPACE_H_

384
modules/video_coding/codecs/h264/h264_decoder_impl.cc Normal file
View File

@ -0,0 +1,384 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
// Everything declared/defined in this header is only required when WebRTC is
// build with H264 support, please do not move anything out of the
// #ifdef unless needed and tested.
#ifdef WEBRTC_USE_H264
#include "modules/video_coding/codecs/h264/h264_decoder_impl.h"
#include <algorithm>
#include <limits>
#include <memory>
extern "C" {
#include "third_party/ffmpeg/libavcodec/avcodec.h"
#include "third_party/ffmpeg/libavformat/avformat.h"
#include "third_party/ffmpeg/libavutil/imgutils.h"
} // extern "C"
#include "api/video/color_space.h"
#include "api/video/i010_buffer.h"
#include "api/video/i420_buffer.h"
#include "common_video/include/video_frame_buffer.h"
#include "modules/video_coding/codecs/h264/h264_color_space.h"
#include "rtc_base/checks.h"
#include "rtc_base/critical_section.h"
#include "rtc_base/keep_ref_until_done.h"
#include "rtc_base/logging.h"
#include "system_wrappers/include/field_trial.h"
#include "system_wrappers/include/metrics.h"
namespace webrtc {
namespace {
const AVPixelFormat kPixelFormatDefault = AV_PIX_FMT_YUV420P;
const AVPixelFormat kPixelFormatFullRange = AV_PIX_FMT_YUVJ420P;
const size_t kYPlaneIndex = 0;
const size_t kUPlaneIndex = 1;
const size_t kVPlaneIndex = 2;
// Used by histograms. Values of entries should not be changed.
enum H264DecoderImplEvent {
kH264DecoderEventInit = 0,
kH264DecoderEventError = 1,
kH264DecoderEventMax = 16,
};
} // namespace
int H264DecoderImpl::AVGetBuffer2(AVCodecContext* context,
AVFrame* av_frame,
int flags) {
// Set in |InitDecode|.
H264DecoderImpl* decoder = static_cast<H264DecoderImpl*>(context->opaque);
// DCHECK values set in |InitDecode|.
RTC_DCHECK(decoder);
// Necessary capability to be allowed to provide our own buffers.
RTC_DCHECK(context->codec->capabilities | AV_CODEC_CAP_DR1);
// Limited or full range YUV420 is expected.
RTC_CHECK(context->pix_fmt == kPixelFormatDefault ||
context->pix_fmt == kPixelFormatFullRange);
// |av_frame->width| and |av_frame->height| are set by FFmpeg. These are the
// actual image's dimensions and may be different from |context->width| and
// |context->coded_width| due to reordering.
int width = av_frame->width;
int height = av_frame->height;
// See |lowres|, if used the decoder scales the image by 1/2^(lowres). This
// has implications on which resolutions are valid, but we don't use it.
RTC_CHECK_EQ(context->lowres, 0);
// Adjust the |width| and |height| to values acceptable by the decoder.
// Without this, FFmpeg may overflow the buffer. If modified, |width| and/or
// |height| are larger than the actual image and the image has to be cropped
// (top-left corner) after decoding to avoid visible borders to the right and
// bottom of the actual image.
avcodec_align_dimensions(context, &width, &height);
RTC_CHECK_GE(width, 0);
RTC_CHECK_GE(height, 0);
int ret = av_image_check_size(static_cast<unsigned int>(width),
static_cast<unsigned int>(height), 0, nullptr);
if (ret < 0) {
RTC_LOG(LS_ERROR) << "Invalid picture size " << width << "x" << height;
decoder->ReportError();
return ret;
}
// The video frame is stored in |frame_buffer|. |av_frame| is FFmpeg's version
// of a video frame and will be set up to reference |frame_buffer|'s data.
// FFmpeg expects the initial allocation to be zero-initialized according to
// http://crbug.com/390941. Our pool is set up to zero-initialize new buffers.
// TODO(nisse): Delete that feature from the video pool, instead add
// an explicit call to InitializeData here.
rtc::scoped_refptr<I420Buffer> frame_buffer =
decoder->pool_.CreateBuffer(width, height);
int y_size = width * height;
int uv_size = frame_buffer->ChromaWidth() * frame_buffer->ChromaHeight();
// DCHECK that we have a continuous buffer as is required.
RTC_DCHECK_EQ(frame_buffer->DataU(), frame_buffer->DataY() + y_size);
RTC_DCHECK_EQ(frame_buffer->DataV(), frame_buffer->DataU() + uv_size);
int total_size = y_size + 2 * uv_size;
av_frame->format = context->pix_fmt;
av_frame->reordered_opaque = context->reordered_opaque;
// Set |av_frame| members as required by FFmpeg.
av_frame->data[kYPlaneIndex] = frame_buffer->MutableDataY();
av_frame->linesize[kYPlaneIndex] = frame_buffer->StrideY();
av_frame->data[kUPlaneIndex] = frame_buffer->MutableDataU();
av_frame->linesize[kUPlaneIndex] = frame_buffer->StrideU();
av_frame->data[kVPlaneIndex] = frame_buffer->MutableDataV();
av_frame->linesize[kVPlaneIndex] = frame_buffer->StrideV();
RTC_DCHECK_EQ(av_frame->extended_data, av_frame->data);
// Create a VideoFrame object, to keep a reference to the buffer.
// TODO(nisse): The VideoFrame's timestamp and rotation info is not used.
// Refactor to do not use a VideoFrame object at all.
av_frame->buf[0] = av_buffer_create(
av_frame->data[kYPlaneIndex], total_size, AVFreeBuffer2,
static_cast<void*>(
std::make_unique<VideoFrame>(VideoFrame::Builder()
.set_video_frame_buffer(frame_buffer)
.set_rotation(kVideoRotation_0)
.set_timestamp_us(0)
.build())
.release()),
0);
RTC_CHECK(av_frame->buf[0]);
return 0;
}
void H264DecoderImpl::AVFreeBuffer2(void* opaque, uint8_t* data) {
// The buffer pool recycles the buffer used by |video_frame| when there are no
// more references to it. |video_frame| is a thin buffer holder and is not
// recycled.
VideoFrame* video_frame = static_cast<VideoFrame*>(opaque);
delete video_frame;
}
H264DecoderImpl::H264DecoderImpl()
: pool_(true),
decoded_image_callback_(nullptr),
has_reported_init_(false),
has_reported_error_(false) {}
H264DecoderImpl::~H264DecoderImpl() {
Release();
}
int32_t H264DecoderImpl::InitDecode(const VideoCodec* codec_settings,
int32_t number_of_cores) {
ReportInit();
if (codec_settings && codec_settings->codecType != kVideoCodecH264) {
ReportError();
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
// Release necessary in case of re-initializing.
int32_t ret = Release();
if (ret != WEBRTC_VIDEO_CODEC_OK) {
ReportError();
return ret;
}
RTC_DCHECK(!av_context_);
// Initialize AVCodecContext.
av_context_.reset(avcodec_alloc_context3(nullptr));
av_context_->codec_type = AVMEDIA_TYPE_VIDEO;
av_context_->codec_id = AV_CODEC_ID_H264;
if (codec_settings) {
av_context_->coded_width = codec_settings->width;
av_context_->coded_height = codec_settings->height;
}
av_context_->pix_fmt = kPixelFormatDefault;
av_context_->extradata = nullptr;
av_context_->extradata_size = 0;
// If this is ever increased, look at |av_context_->thread_safe_callbacks| and
// make it possible to disable the thread checker in the frame buffer pool.
av_context_->thread_count = 1;
av_context_->thread_type = FF_THREAD_SLICE;
// Function used by FFmpeg to get buffers to store decoded frames in.
av_context_->get_buffer2 = AVGetBuffer2;
// |get_buffer2| is called with the context, there |opaque| can be used to get
// a pointer |this|.
av_context_->opaque = this;
AVCodec* codec = avcodec_find_decoder(av_context_->codec_id);
if (!codec) {
// This is an indication that FFmpeg has not been initialized or it has not
// been compiled/initialized with the correct set of codecs.
RTC_LOG(LS_ERROR) << "FFmpeg H.264 decoder not found.";
Release();
ReportError();
return WEBRTC_VIDEO_CODEC_ERROR;
}
int res = avcodec_open2(av_context_.get(), codec, nullptr);
if (res < 0) {
RTC_LOG(LS_ERROR) << "avcodec_open2 error: " << res;
Release();
ReportError();
return WEBRTC_VIDEO_CODEC_ERROR;
}
av_frame_.reset(av_frame_alloc());
if (codec_settings && codec_settings->buffer_pool_size) {
if (!pool_.Resize(*codec_settings->buffer_pool_size)) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
}
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t H264DecoderImpl::Release() {
av_context_.reset();
av_frame_.reset();
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t H264DecoderImpl::RegisterDecodeCompleteCallback(
DecodedImageCallback* callback) {
decoded_image_callback_ = callback;
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t H264DecoderImpl::Decode(const EncodedImage& input_image,
bool /*missing_frames*/,
int64_t /*render_time_ms*/) {
if (!IsInitialized()) {
ReportError();
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
if (!decoded_image_callback_) {
RTC_LOG(LS_WARNING)
<< "InitDecode() has been called, but a callback function "
"has not been set with RegisterDecodeCompleteCallback()";
ReportError();
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
if (!input_image.data() || !input_image.size()) {
ReportError();
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
AVPacket packet;
av_init_packet(&packet);
// packet.data has a non-const type, but isn't modified by
// avcodec_send_packet.
packet.data = const_cast<uint8_t*>(input_image.data());
if (input_image.size() >
static_cast<size_t>(std::numeric_limits<int>::max())) {
ReportError();
return WEBRTC_VIDEO_CODEC_ERROR;
}
packet.size = static_cast<int>(input_image.size());
int64_t frame_timestamp_us = input_image.ntp_time_ms_ * 1000; // ms -> μs
av_context_->reordered_opaque = frame_timestamp_us;
int result = avcodec_send_packet(av_context_.get(), &packet);
if (result < 0) {
RTC_LOG(LS_ERROR) << "avcodec_send_packet error: " << result;
ReportError();
return WEBRTC_VIDEO_CODEC_ERROR;
}
result = avcodec_receive_frame(av_context_.get(), av_frame_.get());
if (result < 0) {
RTC_LOG(LS_ERROR) << "avcodec_receive_frame error: " << result;
ReportError();
return WEBRTC_VIDEO_CODEC_ERROR;
}
// We don't expect reordering. Decoded frame tamestamp should match
// the input one.
RTC_DCHECK_EQ(av_frame_->reordered_opaque, frame_timestamp_us);
absl::optional<uint8_t> qp;
// TODO(sakal): Maybe it is possible to get QP directly from FFmpeg.
h264_bitstream_parser_.ParseBitstream(input_image.data(), input_image.size());
int qp_int;
if (h264_bitstream_parser_.GetLastSliceQp(&qp_int)) {
qp.emplace(qp_int);
}
// Obtain the |video_frame| containing the decoded image.
VideoFrame* input_frame =
static_cast<VideoFrame*>(av_buffer_get_opaque(av_frame_->buf[0]));
RTC_DCHECK(input_frame);
const webrtc::I420BufferInterface* i420_buffer =
input_frame->video_frame_buffer()->GetI420();
// When needed, FFmpeg applies cropping by moving plane pointers and adjusting
// frame width/height. Ensure that cropped buffers lie within the allocated
// memory.
RTC_DCHECK_LE(av_frame_->width, i420_buffer->width());
RTC_DCHECK_LE(av_frame_->height, i420_buffer->height());
RTC_DCHECK_GE(av_frame_->data[kYPlaneIndex], i420_buffer->DataY());
RTC_DCHECK_LE(
av_frame_->data[kYPlaneIndex] +
av_frame_->linesize[kYPlaneIndex] * av_frame_->height,
i420_buffer->DataY() + i420_buffer->StrideY() * i420_buffer->height());
RTC_DCHECK_GE(av_frame_->data[kUPlaneIndex], i420_buffer->DataU());
RTC_DCHECK_LE(av_frame_->data[kUPlaneIndex] +
av_frame_->linesize[kUPlaneIndex] * av_frame_->height / 2,
i420_buffer->DataU() +
i420_buffer->StrideU() * i420_buffer->height() / 2);
RTC_DCHECK_GE(av_frame_->data[kVPlaneIndex], i420_buffer->DataV());
RTC_DCHECK_LE(av_frame_->data[kVPlaneIndex] +
av_frame_->linesize[kVPlaneIndex] * av_frame_->height / 2,
i420_buffer->DataV() +
i420_buffer->StrideV() * i420_buffer->height() / 2);
auto cropped_buffer = WrapI420Buffer(
av_frame_->width, av_frame_->height, av_frame_->data[kYPlaneIndex],
av_frame_->linesize[kYPlaneIndex], av_frame_->data[kUPlaneIndex],
av_frame_->linesize[kUPlaneIndex], av_frame_->data[kVPlaneIndex],
av_frame_->linesize[kVPlaneIndex], rtc::KeepRefUntilDone(i420_buffer));
// Pass on color space from input frame if explicitly specified.
const ColorSpace& color_space =
input_image.ColorSpace() ? *input_image.ColorSpace()
: ExtractH264ColorSpace(av_context_.get());
VideoFrame decoded_frame = VideoFrame::Builder()
.set_video_frame_buffer(cropped_buffer)
.set_timestamp_rtp(input_image.Timestamp())
.set_color_space(color_space)
.build();
// Return decoded frame.
// TODO(nisse): Timestamp and rotation are all zero here. Change decoder
// interface to pass a VideoFrameBuffer instead of a VideoFrame?
decoded_image_callback_->Decoded(decoded_frame, absl::nullopt, qp);
// Stop referencing it, possibly freeing |input_frame|.
av_frame_unref(av_frame_.get());
input_frame = nullptr;
return WEBRTC_VIDEO_CODEC_OK;
}
const char* H264DecoderImpl::ImplementationName() const {
return "FFmpeg";
}
bool H264DecoderImpl::IsInitialized() const {
return av_context_ != nullptr;
}
void H264DecoderImpl::ReportInit() {
if (has_reported_init_)
return;
RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264DecoderImpl.Event",
kH264DecoderEventInit, kH264DecoderEventMax);
has_reported_init_ = true;
}
void H264DecoderImpl::ReportError() {
if (has_reported_error_)
return;
RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264DecoderImpl.Event",
kH264DecoderEventError, kH264DecoderEventMax);
has_reported_error_ = true;
}
} // namespace webrtc
#endif // WEBRTC_USE_H264

111
modules/video_coding/codecs/h264/h264_decoder_impl.h Normal file
View File

@ -0,0 +1,111 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#ifndef MODULES_VIDEO_CODING_CODECS_H264_H264_DECODER_IMPL_H_
#define MODULES_VIDEO_CODING_CODECS_H264_H264_DECODER_IMPL_H_
// Everything declared in this header is only required when WebRTC is
// build with H264 support, please do not move anything out of the
// #ifdef unless needed and tested.
#ifdef WEBRTC_USE_H264
#if defined(WEBRTC_WIN) && !defined(__clang__)
#error "See: bugs.webrtc.org/9213#c13."
#endif
#include <memory>
#include "modules/video_coding/codecs/h264/include/h264.h"
// CAVEAT: According to ffmpeg docs for avcodec_send_packet, ffmpeg requires a
// few extra padding bytes after the end of input. And in addition, docs for
// AV_INPUT_BUFFER_PADDING_SIZE says "If the first 23 bits of the additional
// bytes are not 0, then damaged MPEG bitstreams could cause overread and
// segfault."
//
// WebRTC doesn't ensure any such padding, and REQUIRES ffmpeg to be compiled
// with CONFIG_SAFE_BITSTREAM_READER, which is intended to eliminate
// out-of-bounds reads. ffmpeg docs doesn't say explicitly what effects this
// flag has on the h.264 decoder or avcodec_send_packet, though, so this is in
// some way depending on undocumented behavior. If any problems turn up, we may
// have to add an extra copy operation, to enforce padding before buffers are
// passed to ffmpeg.
extern "C" {
#include "third_party/ffmpeg/libavcodec/avcodec.h"
} // extern "C"
#include "common_video/h264/h264_bitstream_parser.h"
#include "common_video/include/i420_buffer_pool.h"
namespace webrtc {
struct AVCodecContextDeleter {
void operator()(AVCodecContext* ptr) const { avcodec_free_context(&ptr); }
};
struct AVFrameDeleter {
void operator()(AVFrame* ptr) const { av_frame_free(&ptr); }
};
class H264DecoderImpl : public H264Decoder {
public:
H264DecoderImpl();
~H264DecoderImpl() override;
// If |codec_settings| is NULL it is ignored. If it is not NULL,
// |codec_settings->codecType| must be |kVideoCodecH264|.
int32_t InitDecode(const VideoCodec* codec_settings,
int32_t number_of_cores) override;
int32_t Release() override;
int32_t RegisterDecodeCompleteCallback(
DecodedImageCallback* callback) override;
// |missing_frames|, |fragmentation| and |render_time_ms| are ignored.
int32_t Decode(const EncodedImage& input_image,
bool /*missing_frames*/,
int64_t render_time_ms = -1) override;
const char* ImplementationName() const override;
private:
// Called by FFmpeg when it needs a frame buffer to store decoded frames in.
// The |VideoFrame| returned by FFmpeg at |Decode| originate from here. Their
// buffers are reference counted and freed by FFmpeg using |AVFreeBuffer2|.
static int AVGetBuffer2(AVCodecContext* context,
AVFrame* av_frame,
int flags);
// Called by FFmpeg when it is done with a video frame, see |AVGetBuffer2|.
static void AVFreeBuffer2(void* opaque, uint8_t* data);
bool IsInitialized() const;
// Reports statistics with histograms.
void ReportInit();
void ReportError();
I420BufferPool pool_;
std::unique_ptr<AVCodecContext, AVCodecContextDeleter> av_context_;
std::unique_ptr<AVFrame, AVFrameDeleter> av_frame_;
DecodedImageCallback* decoded_image_callback_;
bool has_reported_init_;
bool has_reported_error_;
webrtc::H264BitstreamParser h264_bitstream_parser_;
};
} // namespace webrtc
#endif // WEBRTC_USE_H264
#endif // MODULES_VIDEO_CODING_CODECS_H264_H264_DECODER_IMPL_H_

641
modules/video_coding/codecs/h264/h264_encoder_impl.cc Normal file
View File

@ -0,0 +1,641 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
// Everything declared/defined in this header is only required when WebRTC is
// build with H264 support, please do not move anything out of the
// #ifdef unless needed and tested.
#ifdef WEBRTC_USE_H264
#include "modules/video_coding/codecs/h264/h264_encoder_impl.h"
#include <limits>
#include <string>
#include "absl/strings/match.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "modules/video_coding/utility/simulcast_rate_allocator.h"
#include "modules/video_coding/utility/simulcast_utility.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/time_utils.h"
#include "system_wrappers/include/metrics.h"
#include "third_party/libyuv/include/libyuv/convert.h"
#include "third_party/libyuv/include/libyuv/scale.h"
#include "third_party/openh264/src/codec/api/svc/codec_api.h"
#include "third_party/openh264/src/codec/api/svc/codec_app_def.h"
#include "third_party/openh264/src/codec/api/svc/codec_def.h"
#include "third_party/openh264/src/codec/api/svc/codec_ver.h"
namespace webrtc {
namespace {
const bool kOpenH264EncoderDetailedLogging = false;
// QP scaling thresholds.
static const int kLowH264QpThreshold = 24;
static const int kHighH264QpThreshold = 37;
// Used by histograms. Values of entries should not be changed.
enum H264EncoderImplEvent {
kH264EncoderEventInit = 0,
kH264EncoderEventError = 1,
kH264EncoderEventMax = 16,
};
int NumberOfThreads(int width, int height, int number_of_cores) {
// TODO(hbos): In Chromium, multiple threads do not work with sandbox on Mac,
// see crbug.com/583348. Until further investigated, only use one thread.
// if (width * height >= 1920 * 1080 && number_of_cores > 8) {
// return 8; // 8 threads for 1080p on high perf machines.
// } else if (width * height > 1280 * 960 && number_of_cores >= 6) {
// return 3; // 3 threads for 1080p.
// } else if (width * height > 640 * 480 && number_of_cores >= 3) {
// return 2; // 2 threads for qHD/HD.
// } else {
// return 1; // 1 thread for VGA or less.
// }
// TODO(sprang): Also check sSliceArgument.uiSliceNum om GetEncoderPrams(),
// before enabling multithreading here.
return 1;
}
VideoFrameType ConvertToVideoFrameType(EVideoFrameType type) {
switch (type) {
case videoFrameTypeIDR:
return VideoFrameType::kVideoFrameKey;
case videoFrameTypeSkip:
case videoFrameTypeI:
case videoFrameTypeP:
case videoFrameTypeIPMixed:
return VideoFrameType::kVideoFrameDelta;
case videoFrameTypeInvalid:
break;
}
RTC_NOTREACHED() << "Unexpected/invalid frame type: " << type;
return VideoFrameType::kEmptyFrame;
}
} // namespace
// Helper method used by H264EncoderImpl::Encode.
// Copies the encoded bytes from |info| to |encoded_image| and updates the
// fragmentation information of |frag_header|. The |encoded_image->_buffer| may
// be deleted and reallocated if a bigger buffer is required.
//
// After OpenH264 encoding, the encoded bytes are stored in |info| spread out
// over a number of layers and "NAL units". Each NAL unit is a fragment starting
// with the four-byte start code {0,0,0,1}. All of this data (including the
// start codes) is copied to the |encoded_image->_buffer| and the |frag_header|
// is updated to point to each fragment, with offsets and lengths set as to
// exclude the start codes.
static void RtpFragmentize(EncodedImage* encoded_image,
SFrameBSInfo* info,
RTPFragmentationHeader* frag_header) {
// Calculate minimum buffer size required to hold encoded data.
size_t required_capacity = 0;
size_t fragments_count = 0;
for (int layer = 0; layer < info->iLayerNum; ++layer) {
const SLayerBSInfo& layerInfo = info->sLayerInfo[layer];
for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++fragments_count) {
RTC_CHECK_GE(layerInfo.pNalLengthInByte[nal], 0);
// Ensure |required_capacity| will not overflow.
RTC_CHECK_LE(layerInfo.pNalLengthInByte[nal],
std::numeric_limits<size_t>::max() - required_capacity);
required_capacity += layerInfo.pNalLengthInByte[nal];
}
}
// TODO(nisse): Use a cache or buffer pool to avoid allocation?
encoded_image->SetEncodedData(EncodedImageBuffer::Create(required_capacity));
// Iterate layers and NAL units, note each NAL unit as a fragment and copy
// the data to |encoded_image->_buffer|.
const uint8_t start_code[4] = {0, 0, 0, 1};
frag_header->VerifyAndAllocateFragmentationHeader(fragments_count);
size_t frag = 0;
encoded_image->set_size(0);
for (int layer = 0; layer < info->iLayerNum; ++layer) {
const SLayerBSInfo& layerInfo = info->sLayerInfo[layer];
// Iterate NAL units making up this layer, noting fragments.
size_t layer_len = 0;
for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++frag) {
// Because the sum of all layer lengths, |required_capacity|, fits in a
// |size_t|, we know that any indices in-between will not overflow.
RTC_DCHECK_GE(layerInfo.pNalLengthInByte[nal], 4);
RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 0], start_code[0]);
RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 1], start_code[1]);
RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 2], start_code[2]);
RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 3], start_code[3]);
frag_header->fragmentationOffset[frag] =
encoded_image->size() + layer_len + sizeof(start_code);
frag_header->fragmentationLength[frag] =
layerInfo.pNalLengthInByte[nal] - sizeof(start_code);
layer_len += layerInfo.pNalLengthInByte[nal];
}
// Copy the entire layer's data (including start codes).
memcpy(encoded_image->data() + encoded_image->size(), layerInfo.pBsBuf,
layer_len);
encoded_image->set_size(encoded_image->size() + layer_len);
}
}
H264EncoderImpl::H264EncoderImpl(const cricket::VideoCodec& codec)
: packetization_mode_(H264PacketizationMode::SingleNalUnit),
max_payload_size_(0),
number_of_cores_(0),
encoded_image_callback_(nullptr),
has_reported_init_(false),
has_reported_error_(false) {
RTC_CHECK(absl::EqualsIgnoreCase(codec.name, cricket::kH264CodecName));
std::string packetization_mode_string;
if (codec.GetParam(cricket::kH264FmtpPacketizationMode,
&packetization_mode_string) &&
packetization_mode_string == "1") {
packetization_mode_ = H264PacketizationMode::NonInterleaved;
}
downscaled_buffers_.reserve(kMaxSimulcastStreams - 1);
encoded_images_.reserve(kMaxSimulcastStreams);
encoders_.reserve(kMaxSimulcastStreams);
configurations_.reserve(kMaxSimulcastStreams);
tl0sync_limit_.reserve(kMaxSimulcastStreams);
}
H264EncoderImpl::~H264EncoderImpl() {
Release();
}
int32_t H264EncoderImpl::InitEncode(const VideoCodec* inst,
const VideoEncoder::Settings& settings) {
ReportInit();
if (!inst || inst->codecType != kVideoCodecH264) {
ReportError();
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (inst->maxFramerate == 0) {
ReportError();
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (inst->width < 1 || inst->height < 1) {
ReportError();
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
int32_t release_ret = Release();
if (release_ret != WEBRTC_VIDEO_CODEC_OK) {
ReportError();
return release_ret;
}
int number_of_streams = SimulcastUtility::NumberOfSimulcastStreams(*inst);
bool doing_simulcast = (number_of_streams > 1);
if (doing_simulcast &&
!SimulcastUtility::ValidSimulcastParameters(*inst, number_of_streams)) {
return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED;
}
downscaled_buffers_.resize(number_of_streams - 1);
encoded_images_.resize(number_of_streams);
encoders_.resize(number_of_streams);
pictures_.resize(number_of_streams);
configurations_.resize(number_of_streams);
tl0sync_limit_.resize(number_of_streams);
number_of_cores_ = settings.number_of_cores;
max_payload_size_ = settings.max_payload_size;
codec_ = *inst;
// Code expects simulcastStream resolutions to be correct, make sure they are
// filled even when there are no simulcast layers.
if (codec_.numberOfSimulcastStreams == 0) {
codec_.simulcastStream[0].width = codec_.width;
codec_.simulcastStream[0].height = codec_.height;
}
for (int i = 0, idx = number_of_streams - 1; i < number_of_streams;
++i, --idx) {
ISVCEncoder* openh264_encoder;
// Create encoder.
if (WelsCreateSVCEncoder(&openh264_encoder) != 0) {
// Failed to create encoder.
RTC_LOG(LS_ERROR) << "Failed to create OpenH264 encoder";
RTC_DCHECK(!openh264_encoder);
Release();
ReportError();
return WEBRTC_VIDEO_CODEC_ERROR;
}
RTC_DCHECK(openh264_encoder);
if (kOpenH264EncoderDetailedLogging) {
int trace_level = WELS_LOG_DETAIL;
openh264_encoder->SetOption(ENCODER_OPTION_TRACE_LEVEL, &trace_level);
}
// else WELS_LOG_DEFAULT is used by default.
// Store h264 encoder.
encoders_[i] = openh264_encoder;
// Set internal settings from codec_settings
configurations_[i].simulcast_idx = idx;
configurations_[i].sending = false;
configurations_[i].width = codec_.simulcastStream[idx].width;
configurations_[i].height = codec_.simulcastStream[idx].height;
configurations_[i].max_frame_rate = static_cast<float>(codec_.maxFramerate);
configurations_[i].frame_dropping_on = codec_.H264()->frameDroppingOn;
configurations_[i].key_frame_interval = codec_.H264()->keyFrameInterval;
configurations_[i].num_temporal_layers =
codec_.simulcastStream[idx].numberOfTemporalLayers;
// Create downscaled image buffers.
if (i > 0) {
downscaled_buffers_[i - 1] = I420Buffer::Create(
configurations_[i].width, configurations_[i].height,
configurations_[i].width, configurations_[i].width / 2,
configurations_[i].width / 2);
}
// Codec_settings uses kbits/second; encoder uses bits/second.
configurations_[i].max_bps = codec_.maxBitrate * 1000;
configurations_[i].target_bps = codec_.startBitrate * 1000;
// Create encoder parameters based on the layer configuration.
SEncParamExt encoder_params = CreateEncoderParams(i);
// Initialize.
if (openh264_encoder->InitializeExt(&encoder_params) != 0) {
RTC_LOG(LS_ERROR) << "Failed to initialize OpenH264 encoder";
Release();
ReportError();
return WEBRTC_VIDEO_CODEC_ERROR;
}
// TODO(pbos): Base init params on these values before submitting.
int video_format = EVideoFormatType::videoFormatI420;
openh264_encoder->SetOption(ENCODER_OPTION_DATAFORMAT, &video_format);
// Initialize encoded image. Default buffer size: size of unencoded data.
const size_t new_capacity =
CalcBufferSize(VideoType::kI420, codec_.simulcastStream[idx].width,
codec_.simulcastStream[idx].height);
encoded_images_[i].SetEncodedData(EncodedImageBuffer::Create(new_capacity));
encoded_images_[i]._completeFrame = true;
encoded_images_[i]._encodedWidth = codec_.simulcastStream[idx].width;
encoded_images_[i]._encodedHeight = codec_.simulcastStream[idx].height;
encoded_images_[i].set_size(0);
tl0sync_limit_[i] = configurations_[i].num_temporal_layers;
}
SimulcastRateAllocator init_allocator(codec_);
VideoBitrateAllocation allocation =
init_allocator.Allocate(VideoBitrateAllocationParameters(
DataRate::KilobitsPerSec(codec_.startBitrate), codec_.maxFramerate));
SetRates(RateControlParameters(allocation, codec_.maxFramerate));
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t H264EncoderImpl::Release() {
while (!encoders_.empty()) {
ISVCEncoder* openh264_encoder = encoders_.back();
if (openh264_encoder) {
RTC_CHECK_EQ(0, openh264_encoder->Uninitialize());
WelsDestroySVCEncoder(openh264_encoder);
}
encoders_.pop_back();
}
downscaled_buffers_.clear();
configurations_.clear();
encoded_images_.clear();
pictures_.clear();
tl0sync_limit_.clear();
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t H264EncoderImpl::RegisterEncodeCompleteCallback(
EncodedImageCallback* callback) {
encoded_image_callback_ = callback;
return WEBRTC_VIDEO_CODEC_OK;
}
void H264EncoderImpl::SetRates(const RateControlParameters& parameters) {
if (encoders_.empty()) {
RTC_LOG(LS_WARNING) << "SetRates() while uninitialized.";
return;
}
if (parameters.framerate_fps < 1.0) {
RTC_LOG(LS_WARNING) << "Invalid frame rate: " << parameters.framerate_fps;
return;
}
if (parameters.bitrate.get_sum_bps() == 0) {
// Encoder paused, turn off all encoding.
for (size_t i = 0; i < configurations_.size(); ++i) {
configurations_[i].SetStreamState(false);
}
return;
}
codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps);
size_t stream_idx = encoders_.size() - 1;
for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) {
// Update layer config.
configurations_[i].target_bps =
parameters.bitrate.GetSpatialLayerSum(stream_idx);
configurations_[i].max_frame_rate = parameters.framerate_fps;
if (configurations_[i].target_bps) {
configurations_[i].SetStreamState(true);
// Update h264 encoder.
SBitrateInfo target_bitrate;
memset(&target_bitrate, 0, sizeof(SBitrateInfo));
target_bitrate.iLayer = SPATIAL_LAYER_ALL,
target_bitrate.iBitrate = configurations_[i].target_bps;
encoders_[i]->SetOption(ENCODER_OPTION_BITRATE, &target_bitrate);
encoders_[i]->SetOption(ENCODER_OPTION_FRAME_RATE,
&configurations_[i].max_frame_rate);
} else {
configurations_[i].SetStreamState(false);
}
}
}
int32_t H264EncoderImpl::Encode(
const VideoFrame& input_frame,
const std::vector<VideoFrameType>* frame_types) {
if (encoders_.empty()) {
ReportError();
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
if (!encoded_image_callback_) {
RTC_LOG(LS_WARNING)
<< "InitEncode() has been called, but a callback function "
"has not been set with RegisterEncodeCompleteCallback()";
ReportError();
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
rtc::scoped_refptr<const I420BufferInterface> frame_buffer =
input_frame.video_frame_buffer()->ToI420();
bool send_key_frame = false;
for (size_t i = 0; i < configurations_.size(); ++i) {
if (configurations_[i].key_frame_request && configurations_[i].sending) {
send_key_frame = true;
break;
}
}
if (!send_key_frame && frame_types) {
for (size_t i = 0; i < configurations_.size(); ++i) {
const size_t simulcast_idx =
static_cast<size_t>(configurations_[i].simulcast_idx);
if (configurations_[i].sending && simulcast_idx < frame_types->size() &&
(*frame_types)[simulcast_idx] == VideoFrameType::kVideoFrameKey) {
send_key_frame = true;
break;
}
}
}
RTC_DCHECK_EQ(configurations_[0].width, frame_buffer->width());
RTC_DCHECK_EQ(configurations_[0].height, frame_buffer->height());
// Encode image for each layer.
for (size_t i = 0; i < encoders_.size(); ++i) {
// EncodeFrame input.
pictures_[i] = {0};
pictures_[i].iPicWidth = configurations_[i].width;
pictures_[i].iPicHeight = configurations_[i].height;
pictures_[i].iColorFormat = EVideoFormatType::videoFormatI420;
pictures_[i].uiTimeStamp = input_frame.ntp_time_ms();
// Downscale images on second and ongoing layers.
if (i == 0) {
pictures_[i].iStride[0] = frame_buffer->StrideY();
pictures_[i].iStride[1] = frame_buffer->StrideU();
pictures_[i].iStride[2] = frame_buffer->StrideV();
pictures_[i].pData[0] = const_cast<uint8_t*>(frame_buffer->DataY());
pictures_[i].pData[1] = const_cast<uint8_t*>(frame_buffer->DataU());
pictures_[i].pData[2] = const_cast<uint8_t*>(frame_buffer->DataV());
} else {
pictures_[i].iStride[0] = downscaled_buffers_[i - 1]->StrideY();
pictures_[i].iStride[1] = downscaled_buffers_[i - 1]->StrideU();
pictures_[i].iStride[2] = downscaled_buffers_[i - 1]->StrideV();
pictures_[i].pData[0] =
const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataY());
pictures_[i].pData[1] =
const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataU());
pictures_[i].pData[2] =
const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataV());
// Scale the image down a number of times by downsampling factor.
libyuv::I420Scale(pictures_[i - 1].pData[0], pictures_[i - 1].iStride[0],
pictures_[i - 1].pData[1], pictures_[i - 1].iStride[1],
pictures_[i - 1].pData[2], pictures_[i - 1].iStride[2],
configurations_[i - 1].width,
configurations_[i - 1].height, pictures_[i].pData[0],
pictures_[i].iStride[0], pictures_[i].pData[1],
pictures_[i].iStride[1], pictures_[i].pData[2],
pictures_[i].iStride[2], configurations_[i].width,
configurations_[i].height, libyuv::kFilterBilinear);
}
if (!configurations_[i].sending) {
continue;
}
if (frame_types != nullptr) {
// Skip frame?
if ((*frame_types)[i] == VideoFrameType::kEmptyFrame) {
continue;
}
}
if (send_key_frame) {
// API doc says ForceIntraFrame(false) does nothing, but calling this
// function forces a key frame regardless of the |bIDR| argument's value.
// (If every frame is a key frame we get lag/delays.)
encoders_[i]->ForceIntraFrame(true);
configurations_[i].key_frame_request = false;
}
// EncodeFrame output.
SFrameBSInfo info;
memset(&info, 0, sizeof(SFrameBSInfo));
// Encode!
int enc_ret = encoders_[i]->EncodeFrame(&pictures_[i], &info);
if (enc_ret != 0) {
RTC_LOG(LS_ERROR)
<< "OpenH264 frame encoding failed, EncodeFrame returned " << enc_ret
<< ".";
ReportError();
return WEBRTC_VIDEO_CODEC_ERROR;
}
encoded_images_[i]._encodedWidth = configurations_[i].width;
encoded_images_[i]._encodedHeight = configurations_[i].height;
encoded_images_[i].SetTimestamp(input_frame.timestamp());
encoded_images_[i]._frameType = ConvertToVideoFrameType(info.eFrameType);
encoded_images_[i].SetSpatialIndex(configurations_[i].simulcast_idx);
// Split encoded image up into fragments. This also updates
// |encoded_image_|.
RTPFragmentationHeader frag_header;
RtpFragmentize(&encoded_images_[i], &info, &frag_header);
// Encoder can skip frames to save bandwidth in which case
// |encoded_images_[i]._length| == 0.
if (encoded_images_[i].size() > 0) {
// Parse QP.
h264_bitstream_parser_.ParseBitstream(encoded_images_[i].data(),
encoded_images_[i].size());
h264_bitstream_parser_.GetLastSliceQp(&encoded_images_[i].qp_);
// Deliver encoded image.
CodecSpecificInfo codec_specific;
codec_specific.codecType = kVideoCodecH264;
codec_specific.codecSpecific.H264.packetization_mode =
packetization_mode_;
codec_specific.codecSpecific.H264.temporal_idx = kNoTemporalIdx;
codec_specific.codecSpecific.H264.idr_frame =
info.eFrameType == videoFrameTypeIDR;
codec_specific.codecSpecific.H264.base_layer_sync = false;
if (configurations_[i].num_temporal_layers > 1) {
const uint8_t tid = info.sLayerInfo[0].uiTemporalId;
codec_specific.codecSpecific.H264.temporal_idx = tid;
codec_specific.codecSpecific.H264.base_layer_sync =
tid > 0 && tid < tl0sync_limit_[i];
if (codec_specific.codecSpecific.H264.base_layer_sync) {
tl0sync_limit_[i] = tid;
}
if (tid == 0) {
tl0sync_limit_[i] = configurations_[i].num_temporal_layers;
}
}
encoded_image_callback_->OnEncodedImage(encoded_images_[i],
&codec_specific, &frag_header);
}
}
return WEBRTC_VIDEO_CODEC_OK;
}
// Initialization parameters.
// There are two ways to initialize. There is SEncParamBase (cleared with
// memset(&p, 0, sizeof(SEncParamBase)) used in Initialize, and SEncParamExt
// which is a superset of SEncParamBase (cleared with GetDefaultParams) used
// in InitializeExt.
SEncParamExt H264EncoderImpl::CreateEncoderParams(size_t i) const {
SEncParamExt encoder_params;
encoders_[i]->GetDefaultParams(&encoder_params);
if (codec_.mode == VideoCodecMode::kRealtimeVideo) {
encoder_params.iUsageType = CAMERA_VIDEO_REAL_TIME;
} else if (codec_.mode == VideoCodecMode::kScreensharing) {
encoder_params.iUsageType = SCREEN_CONTENT_REAL_TIME;
} else {
RTC_NOTREACHED();
}
encoder_params.iPicWidth = configurations_[i].width;
encoder_params.iPicHeight = configurations_[i].height;
encoder_params.iTargetBitrate = configurations_[i].target_bps;
// Keep unspecified. WebRTC's max codec bitrate is not the same setting
// as OpenH264's iMaxBitrate. More details in https://crbug.com/webrtc/11543
encoder_params.iMaxBitrate = UNSPECIFIED_BIT_RATE;
// Rate Control mode
encoder_params.iRCMode = RC_BITRATE_MODE;
encoder_params.fMaxFrameRate = configurations_[i].max_frame_rate;
// The following parameters are extension parameters (they're in SEncParamExt,
// not in SEncParamBase).
encoder_params.bEnableFrameSkip = configurations_[i].frame_dropping_on;
// |uiIntraPeriod| - multiple of GOP size
// |keyFrameInterval| - number of frames
encoder_params.uiIntraPeriod = configurations_[i].key_frame_interval;
encoder_params.uiMaxNalSize = 0;
// Threading model: use auto.
// 0: auto (dynamic imp. internal encoder)
// 1: single thread (default value)
// >1: number of threads
encoder_params.iMultipleThreadIdc = NumberOfThreads(
encoder_params.iPicWidth, encoder_params.iPicHeight, number_of_cores_);
// The base spatial layer 0 is the only one we use.
encoder_params.sSpatialLayers[0].iVideoWidth = encoder_params.iPicWidth;
encoder_params.sSpatialLayers[0].iVideoHeight = encoder_params.iPicHeight;
encoder_params.sSpatialLayers[0].fFrameRate = encoder_params.fMaxFrameRate;
encoder_params.sSpatialLayers[0].iSpatialBitrate =
encoder_params.iTargetBitrate;
encoder_params.sSpatialLayers[0].iMaxSpatialBitrate =
encoder_params.iMaxBitrate;
encoder_params.iTemporalLayerNum = configurations_[i].num_temporal_layers;
if (encoder_params.iTemporalLayerNum > 1) {
encoder_params.iNumRefFrame = 1;
}
RTC_LOG(INFO) << "OpenH264 version is " << OPENH264_MAJOR << "."
<< OPENH264_MINOR;
switch (packetization_mode_) {
case H264PacketizationMode::SingleNalUnit:
// Limit the size of the packets produced.
encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1;
encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode =
SM_SIZELIMITED_SLICE;
encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceSizeConstraint =
static_cast<unsigned int>(max_payload_size_);
RTC_LOG(INFO) << "Encoder is configured with NALU constraint: "
<< max_payload_size_ << " bytes";
break;
case H264PacketizationMode::NonInterleaved:
// When uiSliceMode = SM_FIXEDSLCNUM_SLICE, uiSliceNum = 0 means auto
// design it with cpu core number.
// TODO(sprang): Set to 0 when we understand why the rate controller borks
// when uiSliceNum > 1.
encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1;
encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode =
SM_FIXEDSLCNUM_SLICE;
break;
}
return encoder_params;
}
void H264EncoderImpl::ReportInit() {
if (has_reported_init_)
return;
RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264EncoderImpl.Event",
kH264EncoderEventInit, kH264EncoderEventMax);
has_reported_init_ = true;
}
void H264EncoderImpl::ReportError() {
if (has_reported_error_)
return;
RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264EncoderImpl.Event",
kH264EncoderEventError, kH264EncoderEventMax);
has_reported_error_ = true;
}
VideoEncoder::EncoderInfo H264EncoderImpl::GetEncoderInfo() const {
EncoderInfo info;
info.supports_native_handle = false;
info.implementation_name = "OpenH264";
info.scaling_settings =
VideoEncoder::ScalingSettings(kLowH264QpThreshold, kHighH264QpThreshold);
info.is_hardware_accelerated = false;
info.has_internal_source = false;
info.supports_simulcast = true;
return info;
}
void H264EncoderImpl::LayerConfig::SetStreamState(bool send_stream) {
if (send_stream && !sending) {
// Need a key frame if we have not sent this stream before.
key_frame_request = true;
}
sending = send_stream;
}
} // namespace webrtc
#endif // WEBRTC_USE_H264

117
modules/video_coding/codecs/h264/h264_encoder_impl.h Normal file
View File

@ -0,0 +1,117 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#ifndef MODULES_VIDEO_CODING_CODECS_H264_H264_ENCODER_IMPL_H_
#define MODULES_VIDEO_CODING_CODECS_H264_H264_ENCODER_IMPL_H_
// Everything declared in this header is only required when WebRTC is
// build with H264 support, please do not move anything out of the
// #ifdef unless needed and tested.
#ifdef WEBRTC_USE_H264
#if defined(WEBRTC_WIN) && !defined(__clang__)
#error "See: bugs.webrtc.org/9213#c13."
#endif
#include <memory>
#include <vector>
#include "api/video/i420_buffer.h"
#include "api/video_codecs/video_encoder.h"
#include "common_video/h264/h264_bitstream_parser.h"
#include "modules/video_coding/codecs/h264/include/h264.h"
#include "modules/video_coding/utility/quality_scaler.h"
#include "third_party/openh264/src/codec/api/svc/codec_app_def.h"
class ISVCEncoder;
namespace webrtc {
class H264EncoderImpl : public H264Encoder {
public:
struct LayerConfig {
int simulcast_idx = 0;
int width = -1;
int height = -1;
bool sending = true;
bool key_frame_request = false;
float max_frame_rate = 0;
uint32_t target_bps = 0;
uint32_t max_bps = 0;
bool frame_dropping_on = false;
int key_frame_interval = 0;
int num_temporal_layers = 1;
void SetStreamState(bool send_stream);
};
public:
explicit H264EncoderImpl(const cricket::VideoCodec& codec);
~H264EncoderImpl() override;
// |settings.max_payload_size| is ignored.
// The following members of |codec_settings| are used. The rest are ignored.
// - codecType (must be kVideoCodecH264)
// - targetBitrate
// - maxFramerate
// - width
// - height
int32_t InitEncode(const VideoCodec* codec_settings,
const VideoEncoder::Settings& settings) override;
int32_t Release() override;
int32_t RegisterEncodeCompleteCallback(
EncodedImageCallback* callback) override;
void SetRates(const RateControlParameters& parameters) override;
// The result of encoding - an EncodedImage and RTPFragmentationHeader - are
// passed to the encode complete callback.
int32_t Encode(const VideoFrame& frame,
const std::vector<VideoFrameType>* frame_types) override;
EncoderInfo GetEncoderInfo() const override;
// Exposed for testing.
H264PacketizationMode PacketizationModeForTesting() const {
return packetization_mode_;
}
private:
SEncParamExt CreateEncoderParams(size_t i) const;
webrtc::H264BitstreamParser h264_bitstream_parser_;
// Reports statistics with histograms.
void ReportInit();
void ReportError();
std::vector<ISVCEncoder*> encoders_;
std::vector<SSourcePicture> pictures_;
std::vector<rtc::scoped_refptr<I420Buffer>> downscaled_buffers_;
std::vector<LayerConfig> configurations_;
std::vector<EncodedImage> encoded_images_;
VideoCodec codec_;
H264PacketizationMode packetization_mode_;
size_t max_payload_size_;
int32_t number_of_cores_;
EncodedImageCallback* encoded_image_callback_;
bool has_reported_init_;
bool has_reported_error_;
std::vector<uint8_t> tl0sync_limit_;
};
} // namespace webrtc
#endif // WEBRTC_USE_H264
#endif // MODULES_VIDEO_CODING_CODECS_H264_H264_ENCODER_IMPL_H_

89
modules/video_coding/codecs/h264/h264_encoder_impl_unittest.cc Normal file
View File

@ -0,0 +1,89 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#include "modules/video_coding/codecs/h264/h264_encoder_impl.h"
#include "api/video_codecs/video_encoder.h"
#include "test/gtest.h"
namespace webrtc {
namespace {
const int kMaxPayloadSize = 1024;
const int kNumCores = 1;
const VideoEncoder::Capabilities kCapabilities(false);
const VideoEncoder::Settings kSettings(kCapabilities,
kNumCores,
kMaxPayloadSize);
void SetDefaultSettings(VideoCodec* codec_settings) {
codec_settings->codecType = kVideoCodecH264;
codec_settings->maxFramerate = 60;
codec_settings->width = 640;
codec_settings->height = 480;
// If frame dropping is false, we get a warning that bitrate can't
// be controlled for RC_QUALITY_MODE; RC_BITRATE_MODE and RC_TIMESTAMP_MODE
codec_settings->H264()->frameDroppingOn = true;
codec_settings->startBitrate = 2000;
codec_settings->maxBitrate = 4000;
}
TEST(H264EncoderImplTest, CanInitializeWithDefaultParameters) {
H264EncoderImpl encoder(cricket::VideoCodec("H264"));
VideoCodec codec_settings;
SetDefaultSettings(&codec_settings);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder.InitEncode(&codec_settings, kSettings));
EXPECT_EQ(H264PacketizationMode::NonInterleaved,
encoder.PacketizationModeForTesting());
}
TEST(H264EncoderImplTest, CanInitializeWithNonInterleavedModeExplicitly) {
cricket::VideoCodec codec("H264");
codec.SetParam(cricket::kH264FmtpPacketizationMode, "1");
H264EncoderImpl encoder(codec);
VideoCodec codec_settings;
SetDefaultSettings(&codec_settings);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder.InitEncode(&codec_settings, kSettings));
EXPECT_EQ(H264PacketizationMode::NonInterleaved,
encoder.PacketizationModeForTesting());
}
TEST(H264EncoderImplTest, CanInitializeWithSingleNalUnitModeExplicitly) {
cricket::VideoCodec codec("H264");
codec.SetParam(cricket::kH264FmtpPacketizationMode, "0");
H264EncoderImpl encoder(codec);
VideoCodec codec_settings;
SetDefaultSettings(&codec_settings);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder.InitEncode(&codec_settings, kSettings));
EXPECT_EQ(H264PacketizationMode::SingleNalUnit,
encoder.PacketizationModeForTesting());
}
TEST(H264EncoderImplTest, CanInitializeWithRemovedParameter) {
cricket::VideoCodec codec("H264");
codec.RemoveParam(cricket::kH264FmtpPacketizationMode);
H264EncoderImpl encoder(codec);
VideoCodec codec_settings;
SetDefaultSettings(&codec_settings);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder.InitEncode(&codec_settings, kSettings));
EXPECT_EQ(H264PacketizationMode::SingleNalUnit,
encoder.PacketizationModeForTesting());
}
} // anonymous namespace
} // namespace webrtc

103
modules/video_coding/codecs/h264/h264_simulcast_unittest.cc Normal file
View File

@ -0,0 +1,103 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <memory>
#include "api/test/create_simulcast_test_fixture.h"
#include "api/test/simulcast_test_fixture.h"
#include "api/test/video/function_video_decoder_factory.h"
#include "api/test/video/function_video_encoder_factory.h"
#include "modules/video_coding/codecs/h264/include/h264.h"
#include "test/gtest.h"
namespace webrtc {
namespace test {
namespace {
std::unique_ptr<SimulcastTestFixture> CreateSpecificSimulcastTestFixture() {
std::unique_ptr<VideoEncoderFactory> encoder_factory =
std::make_unique<FunctionVideoEncoderFactory>(
[]() { return H264Encoder::Create(cricket::VideoCodec("H264")); });
std::unique_ptr<VideoDecoderFactory> decoder_factory =
std::make_unique<FunctionVideoDecoderFactory>(
[]() { return H264Decoder::Create(); });
return CreateSimulcastTestFixture(std::move(encoder_factory),
std::move(decoder_factory),
SdpVideoFormat("H264"));
}
} // namespace
TEST(TestH264Simulcast, TestKeyFrameRequestsOnAllStreams) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestKeyFrameRequestsOnAllStreams();
}
TEST(TestH264Simulcast, TestPaddingAllStreams) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestPaddingAllStreams();
}
TEST(TestH264Simulcast, TestPaddingTwoStreams) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestPaddingTwoStreams();
}
TEST(TestH264Simulcast, TestPaddingTwoStreamsOneMaxedOut) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestPaddingTwoStreamsOneMaxedOut();
}
TEST(TestH264Simulcast, TestPaddingOneStream) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestPaddingOneStream();
}
TEST(TestH264Simulcast, TestPaddingOneStreamTwoMaxedOut) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestPaddingOneStreamTwoMaxedOut();
}
TEST(TestH264Simulcast, TestSendAllStreams) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestSendAllStreams();
}
TEST(TestH264Simulcast, TestDisablingStreams) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestDisablingStreams();
}
TEST(TestH264Simulcast, TestActiveStreams) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestActiveStreams();
}
TEST(TestH264Simulcast, TestSwitchingToOneStream) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestSwitchingToOneStream();
}
TEST(TestH264Simulcast, TestSwitchingToOneOddStream) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestSwitchingToOneOddStream();
}
TEST(TestH264Simulcast, TestStrideEncodeDecode) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestStrideEncodeDecode();
}
TEST(TestH264Simulcast, TestSpatioTemporalLayers333PatternEncoder) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestSpatioTemporalLayers333PatternEncoder();
}
} // namespace test
} // namespace webrtc

63
modules/video_coding/codecs/h264/include/h264.h Normal file
View File

@ -0,0 +1,63 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#ifndef MODULES_VIDEO_CODING_CODECS_H264_INCLUDE_H264_H_
#define MODULES_VIDEO_CODING_CODECS_H264_INCLUDE_H264_H_
#include <memory>
#include <string>
#include <vector>
#include "media/base/codec.h"
#include "media/base/h264_profile_level_id.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "rtc_base/system/rtc_export.h"
namespace webrtc {
struct SdpVideoFormat;
// Creates an H264 SdpVideoFormat entry with specified paramters.
RTC_EXPORT SdpVideoFormat
CreateH264Format(H264::Profile profile,
H264::Level level,
const std::string& packetization_mode);
// Set to disable the H.264 encoder/decoder implementations that are provided if
// |rtc_use_h264| build flag is true (if false, this function does nothing).
// This function should only be called before or during WebRTC initialization
// and is not thread-safe.
RTC_EXPORT void DisableRtcUseH264();
// Returns a vector with all supported internal H264 profiles that we can
// negotiate in SDP, in order of preference.
std::vector<SdpVideoFormat> SupportedH264Codecs();
class RTC_EXPORT H264Encoder : public VideoEncoder {
public:
static std::unique_ptr<H264Encoder> Create(const cricket::VideoCodec& codec);
// If H.264 is supported (any implementation).
static bool IsSupported();
~H264Encoder() override {}
};
class RTC_EXPORT H264Decoder : public VideoDecoder {
public:
static std::unique_ptr<H264Decoder> Create();
static bool IsSupported();
~H264Decoder() override {}
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_H264_INCLUDE_H264_H_

85
modules/video_coding/codecs/h264/include/h264_globals.h Normal file
View File

@ -0,0 +1,85 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
// This file contains codec dependent definitions that are needed in
// order to compile the WebRTC codebase, even if this codec is not used.
#ifndef MODULES_VIDEO_CODING_CODECS_H264_INCLUDE_H264_GLOBALS_H_
#define MODULES_VIDEO_CODING_CODECS_H264_INCLUDE_H264_GLOBALS_H_
#include <string>
#include "modules/video_coding/codecs/interface/common_constants.h"
#include "rtc_base/checks.h"
namespace webrtc {
// The packetization types that we support: single, aggregated, and fragmented.
enum H264PacketizationTypes {
kH264SingleNalu, // This packet contains a single NAL unit.
kH264StapA, // This packet contains STAP-A (single time
// aggregation) packets. If this packet has an
// associated NAL unit type, it'll be for the
// first such aggregated packet.
kH264FuA, // This packet contains a FU-A (fragmentation
// unit) packet, meaning it is a part of a frame
// that was too large to fit into a single packet.
};
// Packetization modes are defined in RFC 6184 section 6
// Due to the structure containing this being initialized with zeroes
// in some places, and mode 1 being default, mode 1 needs to have the value
// zero. https://crbug.com/webrtc/6803
enum class H264PacketizationMode {
NonInterleaved = 0, // Mode 1 - STAP-A, FU-A is allowed
SingleNalUnit // Mode 0 - only single NALU allowed
};
// This function is declared inline because it is not clear which
// .cc file it should belong to.
// TODO(hta): Refactor. https://bugs.webrtc.org/6842
// TODO(jonasolsson): Use absl::string_view instead when that's available.
inline std::string ToString(H264PacketizationMode mode) {
if (mode == H264PacketizationMode::NonInterleaved) {
return "NonInterleaved";
} else if (mode == H264PacketizationMode::SingleNalUnit) {
return "SingleNalUnit";
}
RTC_NOTREACHED();
return "";
}
struct NaluInfo {
uint8_t type;
int sps_id;
int pps_id;
};
const size_t kMaxNalusPerPacket = 10;
struct RTPVideoHeaderH264 {
// The NAL unit type. If this is a header for a
// fragmented packet, it's the NAL unit type of
// the original data. If this is the header for an
// aggregated packet, it's the NAL unit type of
// the first NAL unit in the packet.
uint8_t nalu_type;
// The packetization type of this buffer - single, aggregated or fragmented.
H264PacketizationTypes packetization_type;
NaluInfo nalus[kMaxNalusPerPacket];
size_t nalus_length;
// The packetization mode of this transport. Packetization mode
// determines which packetization types are allowed when packetizing.
H264PacketizationMode packetization_mode;
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_H264_INCLUDE_H264_GLOBALS_H_

99
modules/video_coding/codecs/h264/test/h264_impl_unittest.cc Normal file
View File

@ -0,0 +1,99 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <stdint.h>
#include <memory>
#include "absl/types/optional.h"
#include "api/video/color_space.h"
#include "api/video/encoded_image.h"
#include "api/video/video_frame.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/video_encoder.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "media/base/codec.h"
#include "media/base/media_constants.h"
#include "modules/video_coding/codecs/h264/include/h264.h"
#include "modules/video_coding/codecs/test/video_codec_unittest.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "test/gtest.h"
#include "test/video_codec_settings.h"
namespace webrtc {
class TestH264Impl : public VideoCodecUnitTest {
protected:
std::unique_ptr<VideoEncoder> CreateEncoder() override {
return H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
}
std::unique_ptr<VideoDecoder> CreateDecoder() override {
return H264Decoder::Create();
}
void ModifyCodecSettings(VideoCodec* codec_settings) override {
webrtc::test::CodecSettings(kVideoCodecH264, codec_settings);
}
};
#ifdef WEBRTC_USE_H264
#define MAYBE_EncodeDecode EncodeDecode
#define MAYBE_DecodedQpEqualsEncodedQp DecodedQpEqualsEncodedQp
#else
#define MAYBE_EncodeDecode DISABLED_EncodeDecode
#define MAYBE_DecodedQpEqualsEncodedQp DISABLED_DecodedQpEqualsEncodedQp
#endif
TEST_F(TestH264Impl, MAYBE_EncodeDecode) {
VideoFrame input_frame = NextInputFrame();
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(input_frame, nullptr));
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
// First frame should be a key frame.
encoded_frame._frameType = VideoFrameType::kVideoFrameKey;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, false, 0));
std::unique_ptr<VideoFrame> decoded_frame;
absl::optional<uint8_t> decoded_qp;
ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
ASSERT_TRUE(decoded_frame);
EXPECT_GT(I420PSNR(&input_frame, decoded_frame.get()), 36);
const ColorSpace color_space = *decoded_frame->color_space();
EXPECT_EQ(ColorSpace::PrimaryID::kUnspecified, color_space.primaries());
EXPECT_EQ(ColorSpace::TransferID::kUnspecified, color_space.transfer());
EXPECT_EQ(ColorSpace::MatrixID::kUnspecified, color_space.matrix());
EXPECT_EQ(ColorSpace::RangeID::kInvalid, color_space.range());
EXPECT_EQ(ColorSpace::ChromaSiting::kUnspecified,
color_space.chroma_siting_horizontal());
EXPECT_EQ(ColorSpace::ChromaSiting::kUnspecified,
color_space.chroma_siting_vertical());
}
TEST_F(TestH264Impl, MAYBE_DecodedQpEqualsEncodedQp) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
// First frame should be a key frame.
encoded_frame._frameType = VideoFrameType::kVideoFrameKey;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, false, 0));
std::unique_ptr<VideoFrame> decoded_frame;
absl::optional<uint8_t> decoded_qp;
ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
ASSERT_TRUE(decoded_frame);
ASSERT_TRUE(decoded_qp);
EXPECT_EQ(encoded_frame.qp_, *decoded_qp);
}
} // namespace webrtc

28
modules/video_coding/codecs/interface/common_constants.h Normal file
View File

@ -0,0 +1,28 @@
/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
// This file contains constants that are used by multiple global
// codec definitions (modules/video_coding/codecs/*/include/*_globals.h)
#ifndef MODULES_VIDEO_CODING_CODECS_INTERFACE_COMMON_CONSTANTS_H_
#define MODULES_VIDEO_CODING_CODECS_INTERFACE_COMMON_CONSTANTS_H_
#include <stdint.h>
namespace webrtc {
const int16_t kNoPictureId = -1;
const int16_t kNoTl0PicIdx = -1;
const uint8_t kNoTemporalIdx = 0xFF;
const int kNoKeyIdx = -1;
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_INTERFACE_COMMON_CONSTANTS_H_

57
modules/video_coding/codecs/multiplex/augmented_video_frame_buffer.cc Normal file
View File

@ -0,0 +1,57 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/multiplex/include/augmented_video_frame_buffer.h"
#include <stdint.h>
#include <utility>
#include "api/video/video_frame_buffer.h"
namespace webrtc {
AugmentedVideoFrameBuffer::AugmentedVideoFrameBuffer(
const rtc::scoped_refptr<VideoFrameBuffer>& video_frame_buffer,
std::unique_ptr<uint8_t[]> augmenting_data,
uint16_t augmenting_data_size)
: augmenting_data_size_(augmenting_data_size),
augmenting_data_(std::move(augmenting_data)),
video_frame_buffer_(video_frame_buffer) {}
rtc::scoped_refptr<VideoFrameBuffer>
AugmentedVideoFrameBuffer::GetVideoFrameBuffer() const {
return video_frame_buffer_;
}
uint8_t* AugmentedVideoFrameBuffer::GetAugmentingData() const {
return augmenting_data_.get();
}
uint16_t AugmentedVideoFrameBuffer::GetAugmentingDataSize() const {
return augmenting_data_size_;
}
VideoFrameBuffer::Type AugmentedVideoFrameBuffer::type() const {
return video_frame_buffer_->type();
}
int AugmentedVideoFrameBuffer::width() const {
return video_frame_buffer_->width();
}
int AugmentedVideoFrameBuffer::height() const {
return video_frame_buffer_->height();
}
rtc::scoped_refptr<I420BufferInterface> AugmentedVideoFrameBuffer::ToI420() {
return video_frame_buffer_->ToI420();
}
} // namespace webrtc

56
modules/video_coding/codecs/multiplex/include/augmented_video_frame_buffer.h Normal file
View File

@ -0,0 +1,56 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_AUGMENTED_VIDEO_FRAME_BUFFER_H_
#define MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_AUGMENTED_VIDEO_FRAME_BUFFER_H_
#include <cstdint>
#include <memory>
#include "api/scoped_refptr.h"
#include "api/video/video_frame_buffer.h"
namespace webrtc {
class AugmentedVideoFrameBuffer : public VideoFrameBuffer {
public:
AugmentedVideoFrameBuffer(
const rtc::scoped_refptr<VideoFrameBuffer>& video_frame_buffer,
std::unique_ptr<uint8_t[]> augmenting_data,
uint16_t augmenting_data_size);
// Retrieves the underlying VideoFrameBuffer without the augmented data
rtc::scoped_refptr<VideoFrameBuffer> GetVideoFrameBuffer() const;
// Gets a pointer to the augmenting data and moves ownership to the caller
uint8_t* GetAugmentingData() const;
// Get the size of the augmenting data
uint16_t GetAugmentingDataSize() const;
// Returns the type of the underlying VideoFrameBuffer
Type type() const final;
// Returns the width of the underlying VideoFrameBuffer
int width() const final;
// Returns the height of the underlying VideoFrameBuffer
int height() const final;
// Get the I140 Buffer from the underlying frame buffer
rtc::scoped_refptr<I420BufferInterface> ToI420() final;
private:
uint16_t augmenting_data_size_;
std::unique_ptr<uint8_t[]> augmenting_data_;
rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_;
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_AUGMENTED_VIDEO_FRAME_BUFFER_H_

81
modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h Normal file
View File

@ -0,0 +1,81 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_MULTIPLEX_DECODER_ADAPTER_H_
#define MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_MULTIPLEX_DECODER_ADAPTER_H_
#include <map>
#include <memory>
#include <vector>
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/video_decoder_factory.h"
#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
namespace webrtc {
class MultiplexDecoderAdapter : public VideoDecoder {
public:
// |factory| is not owned and expected to outlive this class.
MultiplexDecoderAdapter(VideoDecoderFactory* factory,
const SdpVideoFormat& associated_format,
bool supports_augmenting_data = false);
virtual ~MultiplexDecoderAdapter();
// Implements VideoDecoder
int32_t InitDecode(const VideoCodec* codec_settings,
int32_t number_of_cores) override;
int32_t Decode(const EncodedImage& input_image,
bool missing_frames,
int64_t render_time_ms) override;
int32_t RegisterDecodeCompleteCallback(
DecodedImageCallback* callback) override;
int32_t Release() override;
void Decoded(AlphaCodecStream stream_idx,
VideoFrame* decoded_image,
absl::optional<int32_t> decode_time_ms,
absl::optional<uint8_t> qp);
private:
// Wrapper class that redirects Decoded() calls.
class AdapterDecodedImageCallback;
// Holds the decoded image output of a frame.
struct DecodedImageData;
// Holds the augmenting data of an image
struct AugmentingData;
void MergeAlphaImages(VideoFrame* decoded_image,
const absl::optional<int32_t>& decode_time_ms,
const absl::optional<uint8_t>& qp,
VideoFrame* multiplex_decoded_image,
const absl::optional<int32_t>& multiplex_decode_time_ms,
const absl::optional<uint8_t>& multiplex_qp,
std::unique_ptr<uint8_t[]> augmenting_data,
uint16_t augmenting_data_length);
VideoDecoderFactory* const factory_;
const SdpVideoFormat associated_format_;
std::vector<std::unique_ptr<VideoDecoder>> decoders_;
std::vector<std::unique_ptr<AdapterDecodedImageCallback>> adapter_callbacks_;
DecodedImageCallback* decoded_complete_callback_;
// Holds YUV or AXX decode output of a frame that is identified by timestamp.
std::map<uint32_t /* timestamp */, DecodedImageData> decoded_data_;
std::map<uint32_t /* timestamp */, AugmentingData> decoded_augmenting_data_;
const bool supports_augmenting_data_;
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_MULTIPLEX_DECODER_ADAPTER_H_

92
modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h Normal file
View File

@ -0,0 +1,92 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_MULTIPLEX_ENCODER_ADAPTER_H_
#define MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_MULTIPLEX_ENCODER_ADAPTER_H_
#include <map>
#include <memory>
#include <vector>
#include "api/fec_controller_override.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "modules/video_coding/codecs/multiplex/multiplex_encoded_image_packer.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "rtc_base/critical_section.h"
namespace webrtc {
enum AlphaCodecStream {
kYUVStream = 0,
kAXXStream = 1,
kAlphaCodecStreams = 2,
};
class MultiplexEncoderAdapter : public VideoEncoder {
public:
// |factory| is not owned and expected to outlive this class.
MultiplexEncoderAdapter(VideoEncoderFactory* factory,
const SdpVideoFormat& associated_format,
bool supports_augmenting_data = false);
virtual ~MultiplexEncoderAdapter();
// Implements VideoEncoder
void SetFecControllerOverride(
FecControllerOverride* fec_controller_override) override;
int InitEncode(const VideoCodec* inst,
const VideoEncoder::Settings& settings) override;
int Encode(const VideoFrame& input_image,
const std::vector<VideoFrameType>* frame_types) override;
int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override;
void SetRates(const RateControlParameters& parameters) override;
void OnPacketLossRateUpdate(float packet_loss_rate) override;
void OnRttUpdate(int64_t rtt_ms) override;
void OnLossNotification(const LossNotification& loss_notification) override;
int Release() override;
EncoderInfo GetEncoderInfo() const override;
EncodedImageCallback::Result OnEncodedImage(
AlphaCodecStream stream_idx,
const EncodedImage& encodedImage,
const CodecSpecificInfo* codecSpecificInfo,
const RTPFragmentationHeader* fragmentation);
private:
// Wrapper class that redirects OnEncodedImage() calls.
class AdapterEncodedImageCallback;
VideoEncoderFactory* const factory_;
const SdpVideoFormat associated_format_;
std::vector<std::unique_ptr<VideoEncoder>> encoders_;
std::vector<std::unique_ptr<AdapterEncodedImageCallback>> adapter_callbacks_;
EncodedImageCallback* encoded_complete_callback_;
std::map<uint32_t /* timestamp */, MultiplexImage> stashed_images_
RTC_GUARDED_BY(crit_);
uint16_t picture_index_ = 0;
std::vector<uint8_t> multiplex_dummy_planes_;
int key_frame_interval_;
EncodedImage combined_image_;
rtc::CriticalSection crit_;
const bool supports_augmented_data_;
int augmenting_data_size_ = 0;
EncoderInfo encoder_info_;
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_MULTIPLEX_ENCODER_ADAPTER_H_

269
modules/video_coding/codecs/multiplex/multiplex_decoder_adapter.cc Normal file
View File

@ -0,0 +1,269 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h"
#include "api/video/encoded_image.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_frame_buffer.h"
#include "common_video/include/video_frame_buffer.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "modules/video_coding/codecs/multiplex/include/augmented_video_frame_buffer.h"
#include "modules/video_coding/codecs/multiplex/multiplex_encoded_image_packer.h"
#include "rtc_base/keep_ref_until_done.h"
#include "rtc_base/logging.h"
namespace {
void KeepBufferRefs(rtc::scoped_refptr<webrtc::VideoFrameBuffer>,
rtc::scoped_refptr<webrtc::VideoFrameBuffer>) {}
} // anonymous namespace
namespace webrtc {
class MultiplexDecoderAdapter::AdapterDecodedImageCallback
: public webrtc::DecodedImageCallback {
public:
AdapterDecodedImageCallback(webrtc::MultiplexDecoderAdapter* adapter,
AlphaCodecStream stream_idx)
: adapter_(adapter), stream_idx_(stream_idx) {}
void Decoded(VideoFrame& decoded_image,
absl::optional<int32_t> decode_time_ms,
absl::optional<uint8_t> qp) override {
if (!adapter_)
return;
adapter_->Decoded(stream_idx_, &decoded_image, decode_time_ms, qp);
}
int32_t Decoded(VideoFrame& decoded_image) override {
RTC_NOTREACHED();
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t Decoded(VideoFrame& decoded_image, int64_t decode_time_ms) override {
RTC_NOTREACHED();
return WEBRTC_VIDEO_CODEC_OK;
}
private:
MultiplexDecoderAdapter* adapter_;
const AlphaCodecStream stream_idx_;
};
struct MultiplexDecoderAdapter::DecodedImageData {
explicit DecodedImageData(AlphaCodecStream stream_idx)
: stream_idx_(stream_idx),
decoded_image_(
VideoFrame::Builder()
.set_video_frame_buffer(
I420Buffer::Create(1 /* width */, 1 /* height */))
.set_timestamp_rtp(0)
.set_timestamp_us(0)
.set_rotation(kVideoRotation_0)
.build()) {
RTC_DCHECK_EQ(kAXXStream, stream_idx);
}
DecodedImageData(AlphaCodecStream stream_idx,
const VideoFrame& decoded_image,
const absl::optional<int32_t>& decode_time_ms,
const absl::optional<uint8_t>& qp)
: stream_idx_(stream_idx),
decoded_image_(decoded_image),
decode_time_ms_(decode_time_ms),
qp_(qp) {}
const AlphaCodecStream stream_idx_;
VideoFrame decoded_image_;
const absl::optional<int32_t> decode_time_ms_;
const absl::optional<uint8_t> qp_;
private:
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(DecodedImageData);
};
struct MultiplexDecoderAdapter::AugmentingData {
AugmentingData(std::unique_ptr<uint8_t[]> augmenting_data, uint16_t data_size)
: data_(std::move(augmenting_data)), size_(data_size) {}
std::unique_ptr<uint8_t[]> data_;
const uint16_t size_;
private:
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AugmentingData);
};
MultiplexDecoderAdapter::MultiplexDecoderAdapter(
VideoDecoderFactory* factory,
const SdpVideoFormat& associated_format,
bool supports_augmenting_data)
: factory_(factory),
associated_format_(associated_format),
supports_augmenting_data_(supports_augmenting_data) {}
MultiplexDecoderAdapter::~MultiplexDecoderAdapter() {
Release();
}
int32_t MultiplexDecoderAdapter::InitDecode(const VideoCodec* codec_settings,
int32_t number_of_cores) {
RTC_DCHECK_EQ(kVideoCodecMultiplex, codec_settings->codecType);
VideoCodec settings = *codec_settings;
settings.codecType = PayloadStringToCodecType(associated_format_.name);
for (size_t i = 0; i < kAlphaCodecStreams; ++i) {
std::unique_ptr<VideoDecoder> decoder =
factory_->CreateVideoDecoder(associated_format_);
const int32_t rv = decoder->InitDecode(&settings, number_of_cores);
if (rv)
return rv;
adapter_callbacks_.emplace_back(
new MultiplexDecoderAdapter::AdapterDecodedImageCallback(
this, static_cast<AlphaCodecStream>(i)));
decoder->RegisterDecodeCompleteCallback(adapter_callbacks_.back().get());
decoders_.emplace_back(std::move(decoder));
}
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t MultiplexDecoderAdapter::Decode(const EncodedImage& input_image,
bool missing_frames,
int64_t render_time_ms) {
MultiplexImage image = MultiplexEncodedImagePacker::Unpack(input_image);
if (supports_augmenting_data_) {
RTC_DCHECK(decoded_augmenting_data_.find(input_image.Timestamp()) ==
decoded_augmenting_data_.end());
decoded_augmenting_data_.emplace(
std::piecewise_construct,
std::forward_as_tuple(input_image.Timestamp()),
std::forward_as_tuple(std::move(image.augmenting_data),
image.augmenting_data_size));
}
if (image.component_count == 1) {
RTC_DCHECK(decoded_data_.find(input_image.Timestamp()) ==
decoded_data_.end());
decoded_data_.emplace(std::piecewise_construct,
std::forward_as_tuple(input_image.Timestamp()),
std::forward_as_tuple(kAXXStream));
}
int32_t rv = 0;
for (size_t i = 0; i < image.image_components.size(); i++) {
rv = decoders_[image.image_components[i].component_index]->Decode(
image.image_components[i].encoded_image, missing_frames,
render_time_ms);
if (rv != WEBRTC_VIDEO_CODEC_OK)
return rv;
}
return rv;
}
int32_t MultiplexDecoderAdapter::RegisterDecodeCompleteCallback(
DecodedImageCallback* callback) {
decoded_complete_callback_ = callback;
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t MultiplexDecoderAdapter::Release() {
for (auto& decoder : decoders_) {
const int32_t rv = decoder->Release();
if (rv)
return rv;
}
decoders_.clear();
adapter_callbacks_.clear();
return WEBRTC_VIDEO_CODEC_OK;
}
void MultiplexDecoderAdapter::Decoded(AlphaCodecStream stream_idx,
VideoFrame* decoded_image,
absl::optional<int32_t> decode_time_ms,
absl::optional<uint8_t> qp) {
const auto& other_decoded_data_it =
decoded_data_.find(decoded_image->timestamp());
const auto& augmenting_data_it =
decoded_augmenting_data_.find(decoded_image->timestamp());
const bool has_augmenting_data =
augmenting_data_it != decoded_augmenting_data_.end();
if (other_decoded_data_it != decoded_data_.end()) {
uint16_t augmenting_data_size =
has_augmenting_data ? augmenting_data_it->second.size_ : 0;
std::unique_ptr<uint8_t[]> augmenting_data =
has_augmenting_data ? std::move(augmenting_data_it->second.data_)
: nullptr;
auto& other_image_data = other_decoded_data_it->second;
if (stream_idx == kYUVStream) {
RTC_DCHECK_EQ(kAXXStream, other_image_data.stream_idx_);
MergeAlphaImages(decoded_image, decode_time_ms, qp,
&other_image_data.decoded_image_,
other_image_data.decode_time_ms_, other_image_data.qp_,
std::move(augmenting_data), augmenting_data_size);
} else {
RTC_DCHECK_EQ(kYUVStream, other_image_data.stream_idx_);
RTC_DCHECK_EQ(kAXXStream, stream_idx);
MergeAlphaImages(&other_image_data.decoded_image_,
other_image_data.decode_time_ms_, other_image_data.qp_,
decoded_image, decode_time_ms, qp,
std::move(augmenting_data), augmenting_data_size);
}
decoded_data_.erase(decoded_data_.begin(), other_decoded_data_it);
if (has_augmenting_data) {
decoded_augmenting_data_.erase(decoded_augmenting_data_.begin(),
augmenting_data_it);
}
return;
}
RTC_DCHECK(decoded_data_.find(decoded_image->timestamp()) ==
decoded_data_.end());
decoded_data_.emplace(
std::piecewise_construct,
std::forward_as_tuple(decoded_image->timestamp()),
std::forward_as_tuple(stream_idx, *decoded_image, decode_time_ms, qp));
}
void MultiplexDecoderAdapter::MergeAlphaImages(
VideoFrame* decoded_image,
const absl::optional<int32_t>& decode_time_ms,
const absl::optional<uint8_t>& qp,
VideoFrame* alpha_decoded_image,
const absl::optional<int32_t>& alpha_decode_time_ms,
const absl::optional<uint8_t>& alpha_qp,
std::unique_ptr<uint8_t[]> augmenting_data,
uint16_t augmenting_data_length) {
rtc::scoped_refptr<VideoFrameBuffer> merged_buffer;
if (!alpha_decoded_image->timestamp()) {
merged_buffer = decoded_image->video_frame_buffer();
} else {
rtc::scoped_refptr<webrtc::I420BufferInterface> yuv_buffer =
decoded_image->video_frame_buffer()->ToI420();
rtc::scoped_refptr<webrtc::I420BufferInterface> alpha_buffer =
alpha_decoded_image->video_frame_buffer()->ToI420();
RTC_DCHECK_EQ(yuv_buffer->width(), alpha_buffer->width());
RTC_DCHECK_EQ(yuv_buffer->height(), alpha_buffer->height());
merged_buffer = WrapI420ABuffer(
yuv_buffer->width(), yuv_buffer->height(), yuv_buffer->DataY(),
yuv_buffer->StrideY(), yuv_buffer->DataU(), yuv_buffer->StrideU(),
yuv_buffer->DataV(), yuv_buffer->StrideV(), alpha_buffer->DataY(),
alpha_buffer->StrideY(),
rtc::Bind(&KeepBufferRefs, yuv_buffer, alpha_buffer));
}
if (supports_augmenting_data_) {
merged_buffer = rtc::scoped_refptr<webrtc::AugmentedVideoFrameBuffer>(
new rtc::RefCountedObject<AugmentedVideoFrameBuffer>(
merged_buffer, std::move(augmenting_data), augmenting_data_length));
}
VideoFrame merged_image = VideoFrame::Builder()
.set_video_frame_buffer(merged_buffer)
.set_timestamp_rtp(decoded_image->timestamp())
.set_timestamp_us(0)
.set_rotation(decoded_image->rotation())
.set_id(decoded_image->id())
.set_packet_infos(decoded_image->packet_infos())
.build();
decoded_complete_callback_->Decoded(merged_image, decode_time_ms, qp);
}
} // namespace webrtc

277
modules/video_coding/codecs/multiplex/multiplex_encoded_image_packer.cc Normal file
View File

@ -0,0 +1,277 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/multiplex/multiplex_encoded_image_packer.h"
#include <cstring>
#include <utility>
#include "modules/rtp_rtcp/source/byte_io.h"
#include "rtc_base/checks.h"
namespace webrtc {
int PackHeader(uint8_t* buffer, MultiplexImageHeader header) {
int offset = 0;
ByteWriter<uint8_t>::WriteBigEndian(buffer + offset, header.component_count);
offset += sizeof(uint8_t);
ByteWriter<uint16_t>::WriteBigEndian(buffer + offset, header.image_index);
offset += sizeof(uint16_t);
ByteWriter<uint16_t>::WriteBigEndian(buffer + offset,
header.augmenting_data_size);
offset += sizeof(uint16_t);
ByteWriter<uint32_t>::WriteBigEndian(buffer + offset,
header.augmenting_data_offset);
offset += sizeof(uint32_t);
ByteWriter<uint32_t>::WriteBigEndian(buffer + offset,
header.first_component_header_offset);
offset += sizeof(uint32_t);
RTC_DCHECK_EQ(offset, kMultiplexImageHeaderSize);
return offset;
}
MultiplexImageHeader UnpackHeader(const uint8_t* buffer) {
MultiplexImageHeader header;
int offset = 0;
header.component_count = ByteReader<uint8_t>::ReadBigEndian(buffer + offset);
offset += sizeof(uint8_t);
header.image_index = ByteReader<uint16_t>::ReadBigEndian(buffer + offset);
offset += sizeof(uint16_t);
header.augmenting_data_size =
ByteReader<uint16_t>::ReadBigEndian(buffer + offset);
offset += sizeof(uint16_t);
header.augmenting_data_offset =
ByteReader<uint32_t>::ReadBigEndian(buffer + offset);
offset += sizeof(uint32_t);
header.first_component_header_offset =
ByteReader<uint32_t>::ReadBigEndian(buffer + offset);
offset += sizeof(uint32_t);
RTC_DCHECK_EQ(offset, kMultiplexImageHeaderSize);
return header;
}
int PackFrameHeader(uint8_t* buffer,
MultiplexImageComponentHeader frame_header) {
int offset = 0;
ByteWriter<uint32_t>::WriteBigEndian(
buffer + offset, frame_header.next_component_header_offset);
offset += sizeof(uint32_t);
ByteWriter<uint8_t>::WriteBigEndian(buffer + offset,
frame_header.component_index);
offset += sizeof(uint8_t);
ByteWriter<uint32_t>::WriteBigEndian(buffer + offset,
frame_header.bitstream_offset);
offset += sizeof(uint32_t);
ByteWriter<uint32_t>::WriteBigEndian(buffer + offset,
frame_header.bitstream_length);
offset += sizeof(uint32_t);
ByteWriter<uint8_t>::WriteBigEndian(buffer + offset, frame_header.codec_type);
offset += sizeof(uint8_t);
ByteWriter<uint8_t>::WriteBigEndian(
buffer + offset, static_cast<uint8_t>(frame_header.frame_type));
offset += sizeof(uint8_t);
RTC_DCHECK_EQ(offset, kMultiplexImageComponentHeaderSize);
return offset;
}
MultiplexImageComponentHeader UnpackFrameHeader(const uint8_t* buffer) {
MultiplexImageComponentHeader frame_header;
int offset = 0;
frame_header.next_component_header_offset =
ByteReader<uint32_t>::ReadBigEndian(buffer + offset);
offset += sizeof(uint32_t);
frame_header.component_index =
ByteReader<uint8_t>::ReadBigEndian(buffer + offset);
offset += sizeof(uint8_t);
frame_header.bitstream_offset =
ByteReader<uint32_t>::ReadBigEndian(buffer + offset);
offset += sizeof(uint32_t);
frame_header.bitstream_length =
ByteReader<uint32_t>::ReadBigEndian(buffer + offset);
offset += sizeof(uint32_t);
// TODO(nisse): This makes the wire format depend on the numeric values of the
// VideoCodecType and VideoFrameType enum constants.
frame_header.codec_type = static_cast<VideoCodecType>(
ByteReader<uint8_t>::ReadBigEndian(buffer + offset));
offset += sizeof(uint8_t);
frame_header.frame_type = static_cast<VideoFrameType>(
ByteReader<uint8_t>::ReadBigEndian(buffer + offset));
offset += sizeof(uint8_t);
RTC_DCHECK_EQ(offset, kMultiplexImageComponentHeaderSize);
return frame_header;
}
void PackBitstream(uint8_t* buffer, MultiplexImageComponent image) {
memcpy(buffer, image.encoded_image.data(), image.encoded_image.size());
}
MultiplexImage::MultiplexImage(uint16_t picture_index,
uint8_t frame_count,
std::unique_ptr<uint8_t[]> augmenting_data,
uint16_t augmenting_data_size)
: image_index(picture_index),
component_count(frame_count),
augmenting_data_size(augmenting_data_size),
augmenting_data(std::move(augmenting_data)) {}
EncodedImage MultiplexEncodedImagePacker::PackAndRelease(
const MultiplexImage& multiplex_image) {
MultiplexImageHeader header;
std::vector<MultiplexImageComponentHeader> frame_headers;
header.component_count = multiplex_image.component_count;
header.image_index = multiplex_image.image_index;
int header_offset = kMultiplexImageHeaderSize;
header.first_component_header_offset = header_offset;
header.augmenting_data_offset =
header_offset +
kMultiplexImageComponentHeaderSize * header.component_count;
header.augmenting_data_size = multiplex_image.augmenting_data_size;
int bitstream_offset =
header.augmenting_data_offset + header.augmenting_data_size;
const std::vector<MultiplexImageComponent>& images =
multiplex_image.image_components;
EncodedImage combined_image = images[0].encoded_image;
for (size_t i = 0; i < images.size(); i++) {
MultiplexImageComponentHeader frame_header;
header_offset += kMultiplexImageComponentHeaderSize;
frame_header.next_component_header_offset =
(i == images.size() - 1) ? 0 : header_offset;
frame_header.component_index = images[i].component_index;
frame_header.bitstream_offset = bitstream_offset;
frame_header.bitstream_length =
static_cast<uint32_t>(images[i].encoded_image.size());
bitstream_offset += frame_header.bitstream_length;
frame_header.codec_type = images[i].codec_type;
frame_header.frame_type = images[i].encoded_image._frameType;
// As long as one component is delta frame, we have to mark the combined
// frame as delta frame, because it is necessary for all components to be
// key frame so as to decode the whole image without previous frame data.
// Thus only when all components are key frames, we can mark the combined
// frame as key frame.
if (frame_header.frame_type == VideoFrameType::kVideoFrameDelta) {
combined_image._frameType = VideoFrameType::kVideoFrameDelta;
}
frame_headers.push_back(frame_header);
}
auto buffer = EncodedImageBuffer::Create(bitstream_offset);
combined_image.SetEncodedData(buffer);
// header
header_offset = PackHeader(buffer->data(), header);
RTC_DCHECK_EQ(header.first_component_header_offset,
kMultiplexImageHeaderSize);
// Frame Header
for (size_t i = 0; i < images.size(); i++) {
int relative_offset =
PackFrameHeader(buffer->data() + header_offset, frame_headers[i]);
RTC_DCHECK_EQ(relative_offset, kMultiplexImageComponentHeaderSize);
header_offset = frame_headers[i].next_component_header_offset;
RTC_DCHECK_EQ(header_offset,
(i == images.size() - 1)
? 0
: (kMultiplexImageHeaderSize +
kMultiplexImageComponentHeaderSize * (i + 1)));
}
// Augmenting Data
if (multiplex_image.augmenting_data_size != 0) {
memcpy(buffer->data() + header.augmenting_data_offset,
multiplex_image.augmenting_data.get(),
multiplex_image.augmenting_data_size);
}
// Bitstreams
for (size_t i = 0; i < images.size(); i++) {
PackBitstream(buffer->data() + frame_headers[i].bitstream_offset,
images[i]);
}
return combined_image;
}
MultiplexImage MultiplexEncodedImagePacker::Unpack(
const EncodedImage& combined_image) {
const MultiplexImageHeader& header = UnpackHeader(combined_image.data());
std::vector<MultiplexImageComponentHeader> frame_headers;
int header_offset = header.first_component_header_offset;
while (header_offset > 0) {
frame_headers.push_back(
UnpackFrameHeader(combined_image.data() + header_offset));
header_offset = frame_headers.back().next_component_header_offset;
}
RTC_DCHECK_LE(frame_headers.size(), header.component_count);
std::unique_ptr<uint8_t[]> augmenting_data = nullptr;
if (header.augmenting_data_size != 0) {
augmenting_data =
std::unique_ptr<uint8_t[]>(new uint8_t[header.augmenting_data_size]);
memcpy(augmenting_data.get(),
combined_image.data() + header.augmenting_data_offset,
header.augmenting_data_size);
}
MultiplexImage multiplex_image(header.image_index, header.component_count,
std::move(augmenting_data),
header.augmenting_data_size);
for (size_t i = 0; i < frame_headers.size(); i++) {
MultiplexImageComponent image_component;
image_component.component_index = frame_headers[i].component_index;
image_component.codec_type = frame_headers[i].codec_type;
EncodedImage encoded_image = combined_image;
encoded_image.SetTimestamp(combined_image.Timestamp());
encoded_image._frameType = frame_headers[i].frame_type;
encoded_image.SetEncodedData(EncodedImageBuffer::Create(
combined_image.data() + frame_headers[i].bitstream_offset,
frame_headers[i].bitstream_length));
image_component.encoded_image = encoded_image;
multiplex_image.image_components.push_back(image_component);
}
return multiplex_image;
}
} // namespace webrtc

119
modules/video_coding/codecs/multiplex/multiplex_encoded_image_packer.h Normal file
View File

@ -0,0 +1,119 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_MULTIPLEX_MULTIPLEX_ENCODED_IMAGE_PACKER_H_
#define MODULES_VIDEO_CODING_CODECS_MULTIPLEX_MULTIPLEX_ENCODED_IMAGE_PACKER_H_
#include <cstdint>
#include <memory>
#include <vector>
#include "api/video/encoded_image.h"
namespace webrtc {
// Struct describing the whole bundle of multiple frames of an image.
// This struct is expected to be the set in the beginning of a picture's
// bitstream data.
struct MultiplexImageHeader {
// The number of frame components making up the complete picture data.
// For example, |frame_count| = 2 for the case of YUV frame with Alpha frame.
uint8_t component_count;
// The increasing image ID given by the encoder. For different components
// of a single picture, they have the same |picture_index|.
uint16_t image_index;
// The location of the first MultiplexImageComponentHeader in the bitstream,
// in terms of byte from the beginning of the bitstream.
uint32_t first_component_header_offset;
// The location of the augmenting data in the bitstream, in terms of bytes
// from the beginning of the bitstream
uint32_t augmenting_data_offset;
// The size of the augmenting data in the bitstream it terms of byte
uint16_t augmenting_data_size;
};
const int kMultiplexImageHeaderSize =
sizeof(uint8_t) + 2 * sizeof(uint16_t) + 2 * sizeof(uint32_t);
// Struct describing the individual image component's content.
struct MultiplexImageComponentHeader {
// The location of the next MultiplexImageComponentHeader in the bitstream,
// in terms of the byte from the beginning of the bitstream;
uint32_t next_component_header_offset;
// Identifies which component this frame represent, i.e. YUV frame vs Alpha
// frame.
uint8_t component_index;
// The location of the real encoded image data of the frame in the bitstream,
// in terms of byte from the beginning of the bitstream.
uint32_t bitstream_offset;
// Indicates the number of bytes of the encoded image data.
uint32_t bitstream_length;
// Indicated the underlying VideoCodecType of the frame, i.e. VP9 or VP8 etc.
VideoCodecType codec_type;
// Indicated the underlying frame is a key frame or delta frame.
VideoFrameType frame_type;
};
const int kMultiplexImageComponentHeaderSize =
sizeof(uint32_t) + sizeof(uint8_t) + sizeof(uint32_t) + sizeof(uint32_t) +
sizeof(uint8_t) + sizeof(uint8_t);
// Struct holding the encoded image for one component.
struct MultiplexImageComponent {
// Indicated the underlying VideoCodecType of the frame, i.e. VP9 or VP8 etc.
VideoCodecType codec_type;
// Identifies which component this frame represent, i.e. YUV frame vs Alpha
// frame.
uint8_t component_index;
// Stores the actual frame data of the encoded image.
EncodedImage encoded_image;
};
// Struct holding the whole frame bundle of components of an image.
struct MultiplexImage {
uint16_t image_index;
uint8_t component_count;
uint16_t augmenting_data_size;
std::unique_ptr<uint8_t[]> augmenting_data;
std::vector<MultiplexImageComponent> image_components;
MultiplexImage(uint16_t picture_index,
uint8_t component_count,
std::unique_ptr<uint8_t[]> augmenting_data,
uint16_t augmenting_data_size);
};
// A utility class providing conversion between two representations of a
// multiplex image frame:
// 1. Packed version is just one encoded image, we pack all necessary metadata
// in the bitstream as headers.
// 2. Unpacked version is essentially a list of encoded images, one for one
// component.
class MultiplexEncodedImagePacker {
public:
// Note: It is caller responsibility to release the buffer of the result.
static EncodedImage PackAndRelease(const MultiplexImage& image);
// Note: The image components just share the memory with |combined_image|.
static MultiplexImage Unpack(const EncodedImage& combined_image);
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_MULTIPLEX_MULTIPLEX_ENCODED_IMAGE_PACKER_H_

336
modules/video_coding/codecs/multiplex/multiplex_encoder_adapter.cc Normal file
View File

@ -0,0 +1,336 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
#include <cstring>
#include "api/video/encoded_image.h"
#include "api/video_codecs/video_encoder.h"
#include "common_video/include/video_frame_buffer.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "media/base/video_common.h"
#include "modules/include/module_common_types.h"
#include "modules/video_coding/codecs/multiplex/include/augmented_video_frame_buffer.h"
#include "rtc_base/keep_ref_until_done.h"
#include "rtc_base/logging.h"
namespace webrtc {
// Callback wrapper that helps distinguish returned results from |encoders_|
// instances.
class MultiplexEncoderAdapter::AdapterEncodedImageCallback
: public webrtc::EncodedImageCallback {
public:
AdapterEncodedImageCallback(webrtc::MultiplexEncoderAdapter* adapter,
AlphaCodecStream stream_idx)
: adapter_(adapter), stream_idx_(stream_idx) {}
EncodedImageCallback::Result OnEncodedImage(
const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific_info,
const RTPFragmentationHeader* fragmentation) override {
if (!adapter_)
return Result(Result::OK);
return adapter_->OnEncodedImage(stream_idx_, encoded_image,
codec_specific_info, fragmentation);
}
private:
MultiplexEncoderAdapter* adapter_;
const AlphaCodecStream stream_idx_;
};
MultiplexEncoderAdapter::MultiplexEncoderAdapter(
VideoEncoderFactory* factory,
const SdpVideoFormat& associated_format,
bool supports_augmented_data)
: factory_(factory),
associated_format_(associated_format),
encoded_complete_callback_(nullptr),
key_frame_interval_(0),
supports_augmented_data_(supports_augmented_data) {}
MultiplexEncoderAdapter::~MultiplexEncoderAdapter() {
Release();
}
void MultiplexEncoderAdapter::SetFecControllerOverride(
FecControllerOverride* fec_controller_override) {
// Ignored.
}
int MultiplexEncoderAdapter::InitEncode(
const VideoCodec* inst,
const VideoEncoder::Settings& settings) {
const size_t buffer_size =
CalcBufferSize(VideoType::kI420, inst->width, inst->height);
multiplex_dummy_planes_.resize(buffer_size);
// It is more expensive to encode 0x00, so use 0x80 instead.
std::fill(multiplex_dummy_planes_.begin(), multiplex_dummy_planes_.end(),
0x80);
RTC_DCHECK_EQ(kVideoCodecMultiplex, inst->codecType);
VideoCodec video_codec = *inst;
video_codec.codecType = PayloadStringToCodecType(associated_format_.name);
// Take over the key frame interval at adapter level, because we have to
// sync the key frames for both sub-encoders.
switch (video_codec.codecType) {
case kVideoCodecVP8:
key_frame_interval_ = video_codec.VP8()->keyFrameInterval;
video_codec.VP8()->keyFrameInterval = 0;
break;
case kVideoCodecVP9:
key_frame_interval_ = video_codec.VP9()->keyFrameInterval;
video_codec.VP9()->keyFrameInterval = 0;
break;
case kVideoCodecH264:
key_frame_interval_ = video_codec.H264()->keyFrameInterval;
video_codec.H264()->keyFrameInterval = 0;
break;
default:
break;
}
encoder_info_ = EncoderInfo();
encoder_info_.implementation_name = "MultiplexEncoderAdapter (";
encoder_info_.requested_resolution_alignment = 1;
// This needs to be false so that we can do the split in Encode().
encoder_info_.supports_native_handle = false;
for (size_t i = 0; i < kAlphaCodecStreams; ++i) {
std::unique_ptr<VideoEncoder> encoder =
factory_->CreateVideoEncoder(associated_format_);
const int rv = encoder->InitEncode(&video_codec, settings);
if (rv) {
RTC_LOG(LS_ERROR) << "Failed to create multiplex codec index " << i;
return rv;
}
adapter_callbacks_.emplace_back(new AdapterEncodedImageCallback(
this, static_cast<AlphaCodecStream>(i)));
encoder->RegisterEncodeCompleteCallback(adapter_callbacks_.back().get());
const EncoderInfo& encoder_impl_info = encoder->GetEncoderInfo();
encoder_info_.implementation_name += encoder_impl_info.implementation_name;
if (i != kAlphaCodecStreams - 1) {
encoder_info_.implementation_name += ", ";
}
// Uses hardware support if any of the encoders uses it.
// For example, if we are having issues with down-scaling due to
// pipelining delay in HW encoders we need higher encoder usage
// thresholds in CPU adaptation.
if (i == 0) {
encoder_info_.is_hardware_accelerated =
encoder_impl_info.is_hardware_accelerated;
} else {
encoder_info_.is_hardware_accelerated |=
encoder_impl_info.is_hardware_accelerated;
}
encoder_info_.requested_resolution_alignment = cricket::LeastCommonMultiple(
encoder_info_.requested_resolution_alignment,
encoder_impl_info.requested_resolution_alignment);
encoder_info_.has_internal_source = false;
encoders_.emplace_back(std::move(encoder));
}
encoder_info_.implementation_name += ")";
return WEBRTC_VIDEO_CODEC_OK;
}
int MultiplexEncoderAdapter::Encode(
const VideoFrame& input_image,
const std::vector<VideoFrameType>* frame_types) {
if (!encoded_complete_callback_) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
std::vector<VideoFrameType> adjusted_frame_types;
if (key_frame_interval_ > 0 && picture_index_ % key_frame_interval_ == 0) {
adjusted_frame_types.push_back(VideoFrameType::kVideoFrameKey);
} else {
adjusted_frame_types.push_back(VideoFrameType::kVideoFrameDelta);
}
const bool has_alpha = input_image.video_frame_buffer()->type() ==
VideoFrameBuffer::Type::kI420A;
std::unique_ptr<uint8_t[]> augmenting_data = nullptr;
uint16_t augmenting_data_length = 0;
AugmentedVideoFrameBuffer* augmented_video_frame_buffer = nullptr;
if (supports_augmented_data_) {
augmented_video_frame_buffer = static_cast<AugmentedVideoFrameBuffer*>(
input_image.video_frame_buffer().get());
augmenting_data_length =
augmented_video_frame_buffer->GetAugmentingDataSize();
augmenting_data =
std::unique_ptr<uint8_t[]>(new uint8_t[augmenting_data_length]);
memcpy(augmenting_data.get(),
augmented_video_frame_buffer->GetAugmentingData(),
augmenting_data_length);
augmenting_data_size_ = augmenting_data_length;
}
{
rtc::CritScope cs(&crit_);
stashed_images_.emplace(
std::piecewise_construct,
std::forward_as_tuple(input_image.timestamp()),
std::forward_as_tuple(
picture_index_, has_alpha ? kAlphaCodecStreams : 1,
std::move(augmenting_data), augmenting_data_length));
}
++picture_index_;
// Encode YUV
int rv = encoders_[kYUVStream]->Encode(input_image, &adjusted_frame_types);
// If we do not receive an alpha frame, we send a single frame for this
// |picture_index_|. The receiver will receive |frame_count| as 1 which
// specifies this case.
if (rv || !has_alpha)
return rv;
// Encode AXX
const I420ABufferInterface* yuva_buffer =
supports_augmented_data_
? augmented_video_frame_buffer->GetVideoFrameBuffer()->GetI420A()
: input_image.video_frame_buffer()->GetI420A();
rtc::scoped_refptr<I420BufferInterface> alpha_buffer =
WrapI420Buffer(input_image.width(), input_image.height(),
yuva_buffer->DataA(), yuva_buffer->StrideA(),
multiplex_dummy_planes_.data(), yuva_buffer->StrideU(),
multiplex_dummy_planes_.data(), yuva_buffer->StrideV(),
rtc::KeepRefUntilDone(input_image.video_frame_buffer()));
VideoFrame alpha_image = VideoFrame::Builder()
.set_video_frame_buffer(alpha_buffer)
.set_timestamp_rtp(input_image.timestamp())
.set_timestamp_ms(input_image.render_time_ms())
.set_rotation(input_image.rotation())
.set_id(input_image.id())
.set_packet_infos(input_image.packet_infos())
.build();
rv = encoders_[kAXXStream]->Encode(alpha_image, &adjusted_frame_types);
return rv;
}
int MultiplexEncoderAdapter::RegisterEncodeCompleteCallback(
EncodedImageCallback* callback) {
encoded_complete_callback_ = callback;
return WEBRTC_VIDEO_CODEC_OK;
}
void MultiplexEncoderAdapter::SetRates(
const RateControlParameters& parameters) {
VideoBitrateAllocation bitrate_allocation(parameters.bitrate);
bitrate_allocation.SetBitrate(
0, 0, parameters.bitrate.GetBitrate(0, 0) - augmenting_data_size_);
for (auto& encoder : encoders_) {
// TODO(emircan): |framerate| is used to calculate duration in encoder
// instances. We report the total frame rate to keep real time for now.
// Remove this after refactoring duration logic.
encoder->SetRates(RateControlParameters(
bitrate_allocation,
static_cast<uint32_t>(encoders_.size() * parameters.framerate_fps),
parameters.bandwidth_allocation -
DataRate::BitsPerSec(augmenting_data_size_)));
}
}
void MultiplexEncoderAdapter::OnPacketLossRateUpdate(float packet_loss_rate) {
for (auto& encoder : encoders_) {
encoder->OnPacketLossRateUpdate(packet_loss_rate);
}
}
void MultiplexEncoderAdapter::OnRttUpdate(int64_t rtt_ms) {
for (auto& encoder : encoders_) {
encoder->OnRttUpdate(rtt_ms);
}
}
void MultiplexEncoderAdapter::OnLossNotification(
const LossNotification& loss_notification) {
for (auto& encoder : encoders_) {
encoder->OnLossNotification(loss_notification);
}
}
int MultiplexEncoderAdapter::Release() {
for (auto& encoder : encoders_) {
const int rv = encoder->Release();
if (rv)
return rv;
}
encoders_.clear();
adapter_callbacks_.clear();
rtc::CritScope cs(&crit_);
stashed_images_.clear();
return WEBRTC_VIDEO_CODEC_OK;
}
VideoEncoder::EncoderInfo MultiplexEncoderAdapter::GetEncoderInfo() const {
return encoder_info_;
}
EncodedImageCallback::Result MultiplexEncoderAdapter::OnEncodedImage(
AlphaCodecStream stream_idx,
const EncodedImage& encodedImage,
const CodecSpecificInfo* codecSpecificInfo,
const RTPFragmentationHeader* fragmentation) {
// Save the image
MultiplexImageComponent image_component;
image_component.component_index = stream_idx;
image_component.codec_type =
PayloadStringToCodecType(associated_format_.name);
image_component.encoded_image = encodedImage;
// If we don't already own the buffer, make a copy.
image_component.encoded_image.Retain();
rtc::CritScope cs(&crit_);
const auto& stashed_image_itr =
stashed_images_.find(encodedImage.Timestamp());
const auto& stashed_image_next_itr = std::next(stashed_image_itr, 1);
RTC_DCHECK(stashed_image_itr != stashed_images_.end());
MultiplexImage& stashed_image = stashed_image_itr->second;
const uint8_t frame_count = stashed_image.component_count;
stashed_image.image_components.push_back(image_component);
if (stashed_image.image_components.size() == frame_count) {
// Complete case
for (auto iter = stashed_images_.begin();
iter != stashed_images_.end() && iter != stashed_image_next_itr;
iter++) {
// No image at all, skip.
if (iter->second.image_components.size() == 0)
continue;
// We have to send out those stashed frames, otherwise the delta frame
// dependency chain is broken.
combined_image_ =
MultiplexEncodedImagePacker::PackAndRelease(iter->second);
CodecSpecificInfo codec_info = *codecSpecificInfo;
codec_info.codecType = kVideoCodecMultiplex;
encoded_complete_callback_->OnEncodedImage(combined_image_, &codec_info,
fragmentation);
}
stashed_images_.erase(stashed_images_.begin(), stashed_image_next_itr);
}
return EncodedImageCallback::Result(EncodedImageCallback::Result::OK);
}
} // namespace webrtc

320
modules/video_coding/codecs/multiplex/test/multiplex_adapter_unittest.cc Normal file
View File

@ -0,0 +1,320 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <stddef.h>
#include <cstdint>
#include <memory>
#include <utility>
#include <vector>
#include "absl/types/optional.h"
#include "api/scoped_refptr.h"
#include "api/test/mock_video_decoder_factory.h"
#include "api/test/mock_video_encoder_factory.h"
#include "api/video/encoded_image.h"
#include "api/video/video_frame.h"
#include "api/video/video_frame_buffer.h"
#include "api/video/video_rotation.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/video_encoder.h"
#include "common_video/include/video_frame_buffer.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "media/base/media_constants.h"
#include "modules/video_coding/codecs/multiplex/include/augmented_video_frame_buffer.h"
#include "modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h"
#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
#include "modules/video_coding/codecs/multiplex/multiplex_encoded_image_packer.h"
#include "modules/video_coding/codecs/test/video_codec_unittest.h"
#include "modules/video_coding/codecs/vp9/include/vp9.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/keep_ref_until_done.h"
#include "rtc_base/ref_counted_object.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "test/video_codec_settings.h"
using ::testing::_;
using ::testing::Return;
namespace webrtc {
constexpr const char* kMultiplexAssociatedCodecName = cricket::kVp9CodecName;
const VideoCodecType kMultiplexAssociatedCodecType =
PayloadStringToCodecType(kMultiplexAssociatedCodecName);
class TestMultiplexAdapter : public VideoCodecUnitTest,
public ::testing::WithParamInterface<
bool /* supports_augmenting_data */> {
public:
TestMultiplexAdapter()
: decoder_factory_(new webrtc::MockVideoDecoderFactory),
encoder_factory_(new webrtc::MockVideoEncoderFactory),
supports_augmenting_data_(GetParam()) {}
protected:
std::unique_ptr<VideoDecoder> CreateDecoder() override {
return std::make_unique<MultiplexDecoderAdapter>(
decoder_factory_.get(), SdpVideoFormat(kMultiplexAssociatedCodecName),
supports_augmenting_data_);
}
std::unique_ptr<VideoEncoder> CreateEncoder() override {
return std::make_unique<MultiplexEncoderAdapter>(
encoder_factory_.get(), SdpVideoFormat(kMultiplexAssociatedCodecName),
supports_augmenting_data_);
}
void ModifyCodecSettings(VideoCodec* codec_settings) override {
webrtc::test::CodecSettings(kMultiplexAssociatedCodecType, codec_settings);
codec_settings->VP9()->numberOfTemporalLayers = 1;
codec_settings->VP9()->numberOfSpatialLayers = 1;
codec_settings->codecType = webrtc::kVideoCodecMultiplex;
}
std::unique_ptr<VideoFrame> CreateDataAugmentedInputFrame(
VideoFrame* video_frame) {
rtc::scoped_refptr<VideoFrameBuffer> video_buffer =
video_frame->video_frame_buffer();
std::unique_ptr<uint8_t[]> data =
std::unique_ptr<uint8_t[]>(new uint8_t[16]);
for (int i = 0; i < 16; i++) {
data[i] = i;
}
rtc::scoped_refptr<AugmentedVideoFrameBuffer> augmented_video_frame_buffer =
new rtc::RefCountedObject<AugmentedVideoFrameBuffer>(
video_buffer, std::move(data), 16);
return std::make_unique<VideoFrame>(
VideoFrame::Builder()
.set_video_frame_buffer(augmented_video_frame_buffer)
.set_timestamp_rtp(video_frame->timestamp())
.set_timestamp_ms(video_frame->render_time_ms())
.set_rotation(video_frame->rotation())
.set_id(video_frame->id())
.build());
}
std::unique_ptr<VideoFrame> CreateI420AInputFrame() {
VideoFrame input_frame = NextInputFrame();
rtc::scoped_refptr<webrtc::I420BufferInterface> yuv_buffer =
input_frame.video_frame_buffer()->ToI420();
rtc::scoped_refptr<I420ABufferInterface> yuva_buffer = WrapI420ABuffer(
yuv_buffer->width(), yuv_buffer->height(), yuv_buffer->DataY(),
yuv_buffer->StrideY(), yuv_buffer->DataU(), yuv_buffer->StrideU(),
yuv_buffer->DataV(), yuv_buffer->StrideV(), yuv_buffer->DataY(),
yuv_buffer->StrideY(), rtc::KeepRefUntilDone(yuv_buffer));
return std::make_unique<VideoFrame>(VideoFrame::Builder()
.set_video_frame_buffer(yuva_buffer)
.set_timestamp_rtp(123)
.set_timestamp_ms(345)
.set_rotation(kVideoRotation_0)
.build());
}
std::unique_ptr<VideoFrame> CreateInputFrame(bool contains_alpha) {
std::unique_ptr<VideoFrame> video_frame;
if (contains_alpha) {
video_frame = CreateI420AInputFrame();
} else {
VideoFrame next_frame = NextInputFrame();
video_frame = std::make_unique<VideoFrame>(
VideoFrame::Builder()
.set_video_frame_buffer(next_frame.video_frame_buffer())
.set_timestamp_rtp(next_frame.timestamp())
.set_timestamp_ms(next_frame.render_time_ms())
.set_rotation(next_frame.rotation())
.set_id(next_frame.id())
.build());
}
if (supports_augmenting_data_) {
video_frame = CreateDataAugmentedInputFrame(video_frame.get());
}
return video_frame;
}
void CheckData(rtc::scoped_refptr<VideoFrameBuffer> video_frame_buffer) {
if (!supports_augmenting_data_) {
return;
}
AugmentedVideoFrameBuffer* augmented_buffer =
static_cast<AugmentedVideoFrameBuffer*>(video_frame_buffer.get());
EXPECT_EQ(augmented_buffer->GetAugmentingDataSize(), 16);
uint8_t* data = augmented_buffer->GetAugmentingData();
for (int i = 0; i < 16; i++) {
EXPECT_EQ(data[i], i);
}
}
std::unique_ptr<VideoFrame> ExtractAXXFrame(const VideoFrame& video_frame) {
rtc::scoped_refptr<VideoFrameBuffer> video_frame_buffer =
video_frame.video_frame_buffer();
if (supports_augmenting_data_) {
AugmentedVideoFrameBuffer* augmentedBuffer =
static_cast<AugmentedVideoFrameBuffer*>(video_frame_buffer.get());
video_frame_buffer = augmentedBuffer->GetVideoFrameBuffer();
}
const I420ABufferInterface* yuva_buffer = video_frame_buffer->GetI420A();
rtc::scoped_refptr<I420BufferInterface> axx_buffer = WrapI420Buffer(
yuva_buffer->width(), yuva_buffer->height(), yuva_buffer->DataA(),
yuva_buffer->StrideA(), yuva_buffer->DataU(), yuva_buffer->StrideU(),
yuva_buffer->DataV(), yuva_buffer->StrideV(),
rtc::KeepRefUntilDone(video_frame_buffer));
return std::make_unique<VideoFrame>(VideoFrame::Builder()
.set_video_frame_buffer(axx_buffer)
.set_timestamp_rtp(123)
.set_timestamp_ms(345)
.set_rotation(kVideoRotation_0)
.build());
}
private:
void SetUp() override {
EXPECT_CALL(*decoder_factory_, Die);
// The decoders/encoders will be owned by the caller of
// CreateVideoDecoder()/CreateVideoEncoder().
EXPECT_CALL(*decoder_factory_, CreateVideoDecoder)
.Times(2)
.WillRepeatedly([] { return VP9Decoder::Create(); });
EXPECT_CALL(*encoder_factory_, Die);
EXPECT_CALL(*encoder_factory_, CreateVideoEncoder)
.Times(2)
.WillRepeatedly([] { return VP9Encoder::Create(); });
VideoCodecUnitTest::SetUp();
}
const std::unique_ptr<webrtc::MockVideoDecoderFactory> decoder_factory_;
const std::unique_ptr<webrtc::MockVideoEncoderFactory> encoder_factory_;
const bool supports_augmenting_data_;
};
// TODO(emircan): Currently VideoCodecUnitTest tests do a complete setup
// step that goes beyond constructing |decoder_|. Simplify these tests to do
// less.
TEST_P(TestMultiplexAdapter, ConstructAndDestructDecoder) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release());
}
TEST_P(TestMultiplexAdapter, ConstructAndDestructEncoder) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
}
TEST_P(TestMultiplexAdapter, EncodeDecodeI420Frame) {
std::unique_ptr<VideoFrame> input_frame = CreateInputFrame(false);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(*input_frame, nullptr));
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
EXPECT_EQ(kVideoCodecMultiplex, codec_specific_info.codecType);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, false, -1));
std::unique_ptr<VideoFrame> decoded_frame;
absl::optional<uint8_t> decoded_qp;
ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
ASSERT_TRUE(decoded_frame);
EXPECT_GT(I420PSNR(input_frame.get(), decoded_frame.get()), 36);
CheckData(decoded_frame->video_frame_buffer());
}
TEST_P(TestMultiplexAdapter, EncodeDecodeI420AFrame) {
std::unique_ptr<VideoFrame> yuva_frame = CreateInputFrame(true);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(*yuva_frame, nullptr));
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
EXPECT_EQ(kVideoCodecMultiplex, codec_specific_info.codecType);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, false, 0));
std::unique_ptr<VideoFrame> decoded_frame;
absl::optional<uint8_t> decoded_qp;
ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
ASSERT_TRUE(decoded_frame);
EXPECT_GT(I420PSNR(yuva_frame.get(), decoded_frame.get()), 36);
// Find PSNR for AXX bits.
std::unique_ptr<VideoFrame> input_axx_frame = ExtractAXXFrame(*yuva_frame);
std::unique_ptr<VideoFrame> output_axx_frame =
ExtractAXXFrame(*decoded_frame);
EXPECT_GT(I420PSNR(input_axx_frame.get(), output_axx_frame.get()), 47);
CheckData(decoded_frame->video_frame_buffer());
}
TEST_P(TestMultiplexAdapter, CheckSingleFrameEncodedBitstream) {
std::unique_ptr<VideoFrame> input_frame = CreateInputFrame(false);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(*input_frame, nullptr));
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
EXPECT_EQ(kVideoCodecMultiplex, codec_specific_info.codecType);
EXPECT_FALSE(encoded_frame.SpatialIndex());
const MultiplexImage& unpacked_frame =
MultiplexEncodedImagePacker::Unpack(encoded_frame);
EXPECT_EQ(0, unpacked_frame.image_index);
EXPECT_EQ(1, unpacked_frame.component_count);
const MultiplexImageComponent& component = unpacked_frame.image_components[0];
EXPECT_EQ(0, component.component_index);
EXPECT_NE(nullptr, component.encoded_image.data());
EXPECT_EQ(VideoFrameType::kVideoFrameKey, component.encoded_image._frameType);
}
TEST_P(TestMultiplexAdapter, CheckDoubleFramesEncodedBitstream) {
std::unique_ptr<VideoFrame> yuva_frame = CreateInputFrame(true);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(*yuva_frame, nullptr));
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
EXPECT_EQ(kVideoCodecMultiplex, codec_specific_info.codecType);
EXPECT_FALSE(encoded_frame.SpatialIndex());
const MultiplexImage& unpacked_frame =
MultiplexEncodedImagePacker::Unpack(encoded_frame);
EXPECT_EQ(0, unpacked_frame.image_index);
EXPECT_EQ(2, unpacked_frame.component_count);
EXPECT_EQ(unpacked_frame.image_components.size(),
unpacked_frame.component_count);
for (int i = 0; i < unpacked_frame.component_count; ++i) {
const MultiplexImageComponent& component =
unpacked_frame.image_components[i];
EXPECT_EQ(i, component.component_index);
EXPECT_NE(nullptr, component.encoded_image.data());
EXPECT_EQ(VideoFrameType::kVideoFrameKey,
component.encoded_image._frameType);
}
}
TEST_P(TestMultiplexAdapter, ImageIndexIncreases) {
std::unique_ptr<VideoFrame> yuva_frame = CreateInputFrame(true);
const size_t expected_num_encoded_frames = 3;
for (size_t i = 0; i < expected_num_encoded_frames; ++i) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(*yuva_frame, nullptr));
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
const MultiplexImage& unpacked_frame =
MultiplexEncodedImagePacker::Unpack(encoded_frame);
EXPECT_EQ(i, unpacked_frame.image_index);
EXPECT_EQ(
i ? VideoFrameType::kVideoFrameDelta : VideoFrameType::kVideoFrameKey,
encoded_frame._frameType);
}
}
INSTANTIATE_TEST_SUITE_P(TestMultiplexAdapter,
TestMultiplexAdapter,
::testing::Bool());
} // namespace webrtc

85
modules/video_coding/codecs/test/android_codec_factory_helper.cc Normal file
View File

@ -0,0 +1,85 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/test/android_codec_factory_helper.h"
#include <pthread.h>
#include <memory>
#include "rtc_base/checks.h"
#include "rtc_base/ignore_wundef.h"
#include "sdk/android/native_api/base/init.h"
#include "sdk/android/native_api/codecs/wrapper.h"
#include "sdk/android/native_api/jni/class_loader.h"
#include "sdk/android/native_api/jni/jvm.h"
#include "sdk/android/native_api/jni/scoped_java_ref.h"
// Note: this dependency is dangerous since it reaches into Chromium's base.
// There's a risk of e.g. macro clashes. This file may only be used in tests.
// Since we use Chrome's build system for creating the gtest binary, this should
// be fine.
RTC_PUSH_IGNORING_WUNDEF()
#include "base/android/jni_android.h"
RTC_POP_IGNORING_WUNDEF()
namespace webrtc {
namespace test {
namespace {
static pthread_once_t g_initialize_once = PTHREAD_ONCE_INIT;
// There can only be one JNI_OnLoad in each binary. So since this is a GTEST
// C++ runner binary, we want to initialize the same global objects we normally
// do if this had been a Java binary.
void EnsureInitializedOnce() {
RTC_CHECK(::base::android::IsVMInitialized());
JNIEnv* env = ::base::android::AttachCurrentThread();
JavaVM* jvm = nullptr;
RTC_CHECK_EQ(0, env->GetJavaVM(&jvm));
InitAndroid(jvm);
}
} // namespace
void InitializeAndroidObjects() {
RTC_CHECK_EQ(0, pthread_once(&g_initialize_once, &EnsureInitializedOnce));
}
std::unique_ptr<VideoEncoderFactory> CreateAndroidEncoderFactory() {
JNIEnv* env = AttachCurrentThreadIfNeeded();
ScopedJavaLocalRef<jclass> factory_class =
GetClass(env, "org/webrtc/HardwareVideoEncoderFactory");
jmethodID factory_constructor = env->GetMethodID(
factory_class.obj(), "<init>", "(Lorg/webrtc/EglBase$Context;ZZ)V");
ScopedJavaLocalRef<jobject> factory_object(
env, env->NewObject(factory_class.obj(), factory_constructor,
nullptr /* shared_context */,
false /* enable_intel_vp8_encoder */,
true /* enable_h264_high_profile */));
return JavaToNativeVideoEncoderFactory(env, factory_object.obj());
}
std::unique_ptr<VideoDecoderFactory> CreateAndroidDecoderFactory() {
JNIEnv* env = AttachCurrentThreadIfNeeded();
ScopedJavaLocalRef<jclass> factory_class =
GetClass(env, "org/webrtc/HardwareVideoDecoderFactory");
jmethodID factory_constructor = env->GetMethodID(
factory_class.obj(), "<init>", "(Lorg/webrtc/EglBase$Context;)V");
ScopedJavaLocalRef<jobject> factory_object(
env, env->NewObject(factory_class.obj(), factory_constructor,
nullptr /* shared_context */));
return JavaToNativeVideoDecoderFactory(env, factory_object.obj());
}
} // namespace test
} // namespace webrtc

30
modules/video_coding/codecs/test/android_codec_factory_helper.h Normal file
View File

@ -0,0 +1,30 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_TEST_ANDROID_CODEC_FACTORY_HELPER_H_
#define MODULES_VIDEO_CODING_CODECS_TEST_ANDROID_CODEC_FACTORY_HELPER_H_
#include <memory>
#include "api/video_codecs/video_decoder_factory.h"
#include "api/video_codecs/video_encoder_factory.h"
namespace webrtc {
namespace test {
void InitializeAndroidObjects();
std::unique_ptr<VideoEncoderFactory> CreateAndroidEncoderFactory();
std::unique_ptr<VideoDecoderFactory> CreateAndroidDecoderFactory();
} // namespace test
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_TEST_ANDROID_CODEC_FACTORY_HELPER_H_

1
modules/video_coding/codecs/test/batch/empty-runtime-deps Normal file
View File

@ -0,0 +1 @@
does-not-exist

56
modules/video_coding/codecs/test/batch/run-instantiation-tests.sh Executable file
View File

@ -0,0 +1,56 @@
#!/bin/bash
# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
if [ $# -ne 1 ]; then
echo "Usage: run-instantiation-tests.sh ADB-DEVICE-ID"
exit 1
fi
# Paths: update these based on your git checkout and gn output folder names.
WEBRTC_DIR=$HOME/src/webrtc/src
BUILD_DIR=$WEBRTC_DIR/out/Android_Release
# Other settings.
ADB=`which adb`
SERIAL=$1
TIMEOUT=7200
# Ensure we are using the latest version.
ninja -C $BUILD_DIR modules_tests
# Transfer the required files by trying to run a test that doesn't exist.
echo "===> Transferring required resources to device $1."
$WEBRTC_DIR/build/android/test_runner.py gtest \
--output-directory $BUILD_DIR \
--suite modules_tests \
--gtest_filter "DoesNotExist" \
--shard-timeout $TIMEOUT \
--runtime-deps-path $BUILD_DIR/gen.runtime/modules/modules_tests__test_runner_script.runtime_deps \
--adb-path $ADB \
--device $SERIAL \
--verbose
# Run all tests as separate test invocations.
mkdir $SERIAL
pushd $SERIAL
$WEBRTC_DIR/build/android/test_runner.py gtest \
--output-directory $BUILD_DIR \
--suite modules_tests \
--gtest_filter "*InstantiationTest*" \
--gtest_also_run_disabled_tests \
--shard-timeout $TIMEOUT \
--runtime-deps-path ../empty-runtime-deps \
--test-launcher-retry-limit 0 \
--adb-path $ADB \
--device $SERIAL \
--verbose \
--num-retries 0 \
2>&1 | tee -a instantiation-tests.log
popd

70
modules/video_coding/codecs/test/batch/run-videoprocessor-tests.sh Executable file
View File

@ -0,0 +1,70 @@
#!/bin/bash
# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
if [ $# -ne 1 ]; then
echo "Usage: run.sh ADB-DEVICE-ID"
exit 1
fi
# Paths: update these based on your git checkout and gn output folder names.
WEBRTC_DIR=$HOME/src/webrtc/src
BUILD_DIR=$WEBRTC_DIR/out/Android_Release
# Clips: update these to encode/decode other content.
CLIPS=('Foreman')
RESOLUTIONS=('128x96' '160x120' '176x144' '320x240' '352x288')
FRAMERATES=(30)
# Other settings.
ADB=`which adb`
SERIAL=$1
TIMEOUT=7200
# Ensure we are using the latest version.
ninja -C $BUILD_DIR modules_tests
# Transfer the required files by trying to run a test that doesn't exist.
echo "===> Transferring required resources to device $1."
$WEBRTC_DIR/build/android/test_runner.py gtest \
--output-directory $BUILD_DIR \
--suite modules_tests \
--gtest_filter "DoesNotExist" \
--shard-timeout $TIMEOUT \
--runtime-deps-path $BUILD_DIR/gen.runtime/modules/modules_tests__test_runner_script.runtime_deps \
--adb-path $ADB \
--device $SERIAL \
--verbose
# Run all tests as separate test invocations.
mkdir $SERIAL
pushd $SERIAL
for clip in "${CLIPS[@]}"; do
for resolution in "${RESOLUTIONS[@]}"; do
for framerate in "${FRAMERATES[@]}"; do
test_name="${clip}_${resolution}_${framerate}"
log_name="${test_name}.log"
echo "===> Running ${test_name} on device $1."
$WEBRTC_DIR/build/android/test_runner.py gtest \
--output-directory $BUILD_DIR \
--suite modules_tests \
--gtest_filter "CodecSettings/*${test_name}*" \
--shard-timeout $TIMEOUT \
--runtime-deps-path ../empty-runtime-deps \
--test-launcher-retry-limit 0 \
--adb-path $ADB \
--device $SERIAL \
--verbose \
2>&1 | tee -a ${log_name}
done
done
done
popd

28
modules/video_coding/codecs/test/objc_codec_factory_helper.h Normal file
View File

@ -0,0 +1,28 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_TEST_OBJC_CODEC_FACTORY_HELPER_H_
#define MODULES_VIDEO_CODING_CODECS_TEST_OBJC_CODEC_FACTORY_HELPER_H_
#include <memory>
#include "api/video_codecs/video_decoder_factory.h"
#include "api/video_codecs/video_encoder_factory.h"
namespace webrtc {
namespace test {
std::unique_ptr<VideoEncoderFactory> CreateObjCEncoderFactory();
std::unique_ptr<VideoDecoderFactory> CreateObjCDecoderFactory();
} // namespace test
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_TEST_OBJC_CODEC_FACTORY_HELPER_H_

30
modules/video_coding/codecs/test/objc_codec_factory_helper.mm Normal file
View File

@ -0,0 +1,30 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/test/objc_codec_factory_helper.h"
#import "sdk/objc/components/video_codec/RTCVideoDecoderFactoryH264.h"
#import "sdk/objc/components/video_codec/RTCVideoEncoderFactoryH264.h"
#include "sdk/objc/native/api/video_decoder_factory.h"
#include "sdk/objc/native/api/video_encoder_factory.h"
namespace webrtc {
namespace test {
std::unique_ptr<VideoEncoderFactory> CreateObjCEncoderFactory() {
return ObjCToNativeVideoEncoderFactory([[RTC_OBJC_TYPE(RTCVideoEncoderFactoryH264) alloc] init]);
}
std::unique_ptr<VideoDecoderFactory> CreateObjCDecoderFactory() {
return ObjCToNativeVideoDecoderFactory([[RTC_OBJC_TYPE(RTCVideoDecoderFactoryH264) alloc] init]);
}
} // namespace test
} // namespace webrtc

430
modules/video_coding/codecs/test/plot_webrtc_test_logs.py Executable file
View File

@ -0,0 +1,430 @@
# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
"""Plots statistics from WebRTC integration test logs.
Usage: $ python plot_webrtc_test_logs.py filename.txt
"""
import numpy
import sys
import re
import matplotlib.pyplot as plt
# Log events.
EVENT_START = 'RUN ] CodecSettings/VideoCodecTestParameterized.'
EVENT_END = 'OK ] CodecSettings/VideoCodecTestParameterized.'
# Metrics to plot, tuple: (name to parse in file, label to use when plotting).
WIDTH = ('width', 'width')
HEIGHT = ('height', 'height')
FILENAME = ('filename', 'clip')
CODEC_TYPE = ('codec_type', 'Codec')
ENCODER_IMPLEMENTATION_NAME = ('enc_impl_name', 'enc name')
DECODER_IMPLEMENTATION_NAME = ('dec_impl_name', 'dec name')
CODEC_IMPLEMENTATION_NAME = ('codec_impl_name', 'codec name')
CORES = ('num_cores', 'CPU cores used')
DENOISING = ('denoising', 'denoising')
RESILIENCE = ('resilience', 'resilience')
ERROR_CONCEALMENT = ('error_concealment', 'error concealment')
CPU_USAGE = ('cpu_usage_percent', 'CPU usage (%)')
BITRATE = ('target_bitrate_kbps', 'target bitrate (kbps)')
FRAMERATE = ('input_framerate_fps', 'fps')
QP = ('avg_qp', 'QP avg')
PSNR = ('avg_psnr', 'PSNR (dB)')
SSIM = ('avg_ssim', 'SSIM')
ENC_BITRATE = ('bitrate_kbps', 'encoded bitrate (kbps)')
NUM_FRAMES = ('num_input_frames', 'num frames')
NUM_DROPPED_FRAMES = ('num_dropped_frames', 'num dropped frames')
TIME_TO_TARGET = ('time_to_reach_target_bitrate_sec',
'time to reach target rate (sec)')
ENCODE_SPEED_FPS = ('enc_speed_fps', 'encode speed (fps)')
DECODE_SPEED_FPS = ('dec_speed_fps', 'decode speed (fps)')
AVG_KEY_FRAME_SIZE = ('avg_key_frame_size_bytes', 'avg key frame size (bytes)')
AVG_DELTA_FRAME_SIZE = ('avg_delta_frame_size_bytes',
'avg delta frame size (bytes)')
# Settings.
SETTINGS = [
WIDTH,
HEIGHT,
FILENAME,
NUM_FRAMES,
]
# Settings, options for x-axis.
X_SETTINGS = [
CORES,
FRAMERATE,
DENOISING,
RESILIENCE,
ERROR_CONCEALMENT,
BITRATE, # TODO(asapersson): Needs to be last.
]
# Settings, options for subplots.
SUBPLOT_SETTINGS = [
CODEC_TYPE,
ENCODER_IMPLEMENTATION_NAME,
DECODER_IMPLEMENTATION_NAME,
CODEC_IMPLEMENTATION_NAME,
] + X_SETTINGS
# Results.
RESULTS = [
PSNR,
SSIM,
ENC_BITRATE,
NUM_DROPPED_FRAMES,
TIME_TO_TARGET,
ENCODE_SPEED_FPS,
DECODE_SPEED_FPS,
QP,
CPU_USAGE,
AVG_KEY_FRAME_SIZE,
AVG_DELTA_FRAME_SIZE,
]
METRICS_TO_PARSE = SETTINGS + SUBPLOT_SETTINGS + RESULTS
Y_METRICS = [res[1] for res in RESULTS]
# Parameters for plotting.
FIG_SIZE_SCALE_FACTOR_X = 1.6
FIG_SIZE_SCALE_FACTOR_Y = 1.8
GRID_COLOR = [0.45, 0.45, 0.45]
def ParseSetting(filename, setting):
"""Parses setting from file.
Args:
filename: The name of the file.
setting: Name of setting to parse (e.g. width).
Returns:
A list holding parsed settings, e.g. ['width: 128.0', 'width: 160.0'] """
settings = []
settings_file = open(filename)
while True:
line = settings_file.readline()
if not line:
break
if re.search(r'%s' % EVENT_START, line):
# Parse event.
parsed = {}
while True:
line = settings_file.readline()
if not line:
break
if re.search(r'%s' % EVENT_END, line):
# Add parsed setting to list.
if setting in parsed:
s = setting + ': ' + str(parsed[setting])
if s not in settings:
settings.append(s)
break
TryFindMetric(parsed, line)
settings_file.close()
return settings
def ParseMetrics(filename, setting1, setting2):
"""Parses metrics from file.
Args:
filename: The name of the file.
setting1: First setting for sorting metrics (e.g. width).
setting2: Second setting for sorting metrics (e.g. CPU cores used).
Returns:
A dictionary holding parsed metrics.
For example:
metrics[key1][key2][measurement]
metrics = {
"width: 352": {
"CPU cores used: 1.0": {
"encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642],
"PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
"bitrate (kbps)": [50, 100, 300, 500, 1000]
},
"CPU cores used: 2.0": {
"encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642],
"PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
"bitrate (kbps)": [50, 100, 300, 500, 1000]
},
},
"width: 176": {
"CPU cores used: 1.0": {
"encode time (us)": [0.857897, 0.91608, 0.959173, 0.971116, 0.980961],
"PSNR (dB)": [30.243646, 33.375592, 37.574387, 39.42184, 41.437897],
"bitrate (kbps)": [50, 100, 300, 500, 1000]
},
}
} """
metrics = {}
# Parse events.
settings_file = open(filename)
while True:
line = settings_file.readline()
if not line:
break
if re.search(r'%s' % EVENT_START, line):
# Parse event.
parsed = {}
while True:
line = settings_file.readline()
if not line:
break
if re.search(r'%s' % EVENT_END, line):
# Add parsed values to metrics.
key1 = setting1 + ': ' + str(parsed[setting1])
key2 = setting2 + ': ' + str(parsed[setting2])
if key1 not in metrics:
metrics[key1] = {}
if key2 not in metrics[key1]:
metrics[key1][key2] = {}
for label in parsed:
if label not in metrics[key1][key2]:
metrics[key1][key2][label] = []
metrics[key1][key2][label].append(parsed[label])
break
TryFindMetric(parsed, line)
settings_file.close()
return metrics
def TryFindMetric(parsed, line):
for metric in METRICS_TO_PARSE:
name = metric[0]
label = metric[1]
if re.search(r'%s' % name, line):
found, value = GetMetric(name, line)
if found:
parsed[label] = value
return
def GetMetric(name, string):
# Float (e.g. bitrate = 98.8253).
pattern = r'%s\s*[:=]\s*([+-]?\d+\.*\d*)' % name
m = re.search(r'%s' % pattern, string)
if m is not None:
return StringToFloat(m.group(1))
# Alphanumeric characters (e.g. codec type : VP8).
pattern = r'%s\s*[:=]\s*(\w+)' % name
m = re.search(r'%s' % pattern, string)
if m is not None:
return True, m.group(1)
return False, -1
def StringToFloat(value):
try:
value = float(value)
except ValueError:
print "Not a float, skipped %s" % value
return False, -1
return True, value
def Plot(y_metric, x_metric, metrics):
"""Plots y_metric vs x_metric per key in metrics.
For example:
y_metric = 'PSNR (dB)'
x_metric = 'bitrate (kbps)'
metrics = {
"CPU cores used: 1.0": {
"PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
"bitrate (kbps)": [50, 100, 300, 500, 1000]
},
"CPU cores used: 2.0": {
"PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
"bitrate (kbps)": [50, 100, 300, 500, 1000]
},
}
"""
for key in sorted(metrics):
data = metrics[key]
if y_metric not in data:
print "Failed to find metric: %s" % y_metric
continue
y = numpy.array(data[y_metric])
x = numpy.array(data[x_metric])
if len(y) != len(x):
print "Length mismatch for %s, %s" % (y, x)
continue
label = y_metric + ' - ' + str(key)
plt.plot(x, y, label=label, linewidth=1.5, marker='o', markersize=5,
markeredgewidth=0.0)
def PlotFigure(settings, y_metrics, x_metric, metrics, title):
"""Plots metrics in y_metrics list. One figure is plotted and each entry
in the list is plotted in a subplot (and sorted per settings).
For example:
settings = ['width: 128.0', 'width: 160.0']. Sort subplot per setting.
y_metrics = ['PSNR (dB)', 'PSNR (dB)']. Metric to plot per subplot.
x_metric = 'bitrate (kbps)'
"""
plt.figure()
plt.suptitle(title, fontsize='large', fontweight='bold')
settings.sort()
rows = len(settings)
cols = 1
pos = 1
while pos <= rows:
plt.rc('grid', color=GRID_COLOR)
ax = plt.subplot(rows, cols, pos)
plt.grid()
plt.setp(ax.get_xticklabels(), visible=(pos == rows), fontsize='large')
plt.setp(ax.get_yticklabels(), fontsize='large')
setting = settings[pos - 1]
Plot(y_metrics[pos - 1], x_metric, metrics[setting])
if setting.startswith(WIDTH[1]):
plt.title(setting, fontsize='medium')
plt.legend(fontsize='large', loc='best')
pos += 1
plt.xlabel(x_metric, fontsize='large')
plt.subplots_adjust(left=0.06, right=0.98, bottom=0.05, top=0.94, hspace=0.08)
def GetTitle(filename, setting):
title = ''
if setting != CODEC_IMPLEMENTATION_NAME[1] and setting != CODEC_TYPE[1]:
codec_types = ParseSetting(filename, CODEC_TYPE[1])
for i in range(0, len(codec_types)):
title += codec_types[i] + ', '
if setting != CORES[1]:
cores = ParseSetting(filename, CORES[1])
for i in range(0, len(cores)):
title += cores[i].split('.')[0] + ', '
if setting != FRAMERATE[1]:
framerate = ParseSetting(filename, FRAMERATE[1])
for i in range(0, len(framerate)):
title += framerate[i].split('.')[0] + ', '
if (setting != CODEC_IMPLEMENTATION_NAME[1] and
setting != ENCODER_IMPLEMENTATION_NAME[1]):
enc_names = ParseSetting(filename, ENCODER_IMPLEMENTATION_NAME[1])
for i in range(0, len(enc_names)):
title += enc_names[i] + ', '
if (setting != CODEC_IMPLEMENTATION_NAME[1] and
setting != DECODER_IMPLEMENTATION_NAME[1]):
dec_names = ParseSetting(filename, DECODER_IMPLEMENTATION_NAME[1])
for i in range(0, len(dec_names)):
title += dec_names[i] + ', '
filenames = ParseSetting(filename, FILENAME[1])
title += filenames[0].split('_')[0]
num_frames = ParseSetting(filename, NUM_FRAMES[1])
for i in range(0, len(num_frames)):
title += ' (' + num_frames[i].split('.')[0] + ')'
return title
def ToString(input_list):
return ToStringWithoutMetric(input_list, ('', ''))
def ToStringWithoutMetric(input_list, metric):
i = 1
output_str = ""
for m in input_list:
if m != metric:
output_str = output_str + ("%s. %s\n" % (i, m[1]))
i += 1
return output_str
def GetIdx(text_list):
return int(raw_input(text_list)) - 1
def main():
filename = sys.argv[1]
# Setup.
idx_metric = GetIdx("Choose metric:\n0. All\n%s" % ToString(RESULTS))
if idx_metric == -1:
# Plot all metrics. One subplot for each metric.
# Per subplot: metric vs bitrate (per resolution).
cores = ParseSetting(filename, CORES[1])
setting1 = CORES[1]
setting2 = WIDTH[1]
sub_keys = [cores[0]] * len(Y_METRICS)
y_metrics = Y_METRICS
x_metric = BITRATE[1]
else:
resolutions = ParseSetting(filename, WIDTH[1])
idx = GetIdx("Select metric for x-axis:\n%s" % ToString(X_SETTINGS))
if X_SETTINGS[idx] == BITRATE:
idx = GetIdx("Plot per:\n%s" % ToStringWithoutMetric(SUBPLOT_SETTINGS,
BITRATE))
idx_setting = METRICS_TO_PARSE.index(SUBPLOT_SETTINGS[idx])
# Plot one metric. One subplot for each resolution.
# Per subplot: metric vs bitrate (per setting).
setting1 = WIDTH[1]
setting2 = METRICS_TO_PARSE[idx_setting][1]
sub_keys = resolutions
y_metrics = [RESULTS[idx_metric][1]] * len(sub_keys)
x_metric = BITRATE[1]
else:
# Plot one metric. One subplot for each resolution.
# Per subplot: metric vs setting (per bitrate).
setting1 = WIDTH[1]
setting2 = BITRATE[1]
sub_keys = resolutions
y_metrics = [RESULTS[idx_metric][1]] * len(sub_keys)
x_metric = X_SETTINGS[idx][1]
metrics = ParseMetrics(filename, setting1, setting2)
# Stretch fig size.
figsize = plt.rcParams["figure.figsize"]
figsize[0] *= FIG_SIZE_SCALE_FACTOR_X
figsize[1] *= FIG_SIZE_SCALE_FACTOR_Y
plt.rcParams["figure.figsize"] = figsize
PlotFigure(sub_keys, y_metrics, x_metric, metrics,
GetTitle(filename, setting2))
plt.show()
if __name__ == '__main__':
main()

177
modules/video_coding/codecs/test/video_codec_unittest.cc Normal file
View File

@ -0,0 +1,177 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/test/video_codec_unittest.h"
#include <utility>
#include "api/test/create_frame_generator.h"
#include "api/video_codecs/video_encoder.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "test/video_codec_settings.h"
static const int kEncodeTimeoutMs = 100;
static const int kDecodeTimeoutMs = 25;
// Set bitrate to get higher quality.
static const int kStartBitrate = 300;
static const int kMaxBitrate = 4000;
static const int kWidth = 176; // Width of the input image.
static const int kHeight = 144; // Height of the input image.
static const int kMaxFramerate = 30; // Arbitrary value.
namespace webrtc {
namespace {
const VideoEncoder::Capabilities kCapabilities(false);
}
EncodedImageCallback::Result
VideoCodecUnitTest::FakeEncodeCompleteCallback::OnEncodedImage(
const EncodedImage& frame,
const CodecSpecificInfo* codec_specific_info,
const RTPFragmentationHeader* fragmentation) {
rtc::CritScope lock(&test_->encoded_frame_section_);
test_->encoded_frames_.push_back(frame);
RTC_DCHECK(codec_specific_info);
test_->codec_specific_infos_.push_back(*codec_specific_info);
if (!test_->wait_for_encoded_frames_threshold_) {
test_->encoded_frame_event_.Set();
return Result(Result::OK);
}
if (test_->encoded_frames_.size() ==
test_->wait_for_encoded_frames_threshold_) {
test_->wait_for_encoded_frames_threshold_ = 1;
test_->encoded_frame_event_.Set();
}
return Result(Result::OK);
}
void VideoCodecUnitTest::FakeDecodeCompleteCallback::Decoded(
VideoFrame& frame,
absl::optional<int32_t> decode_time_ms,
absl::optional<uint8_t> qp) {
rtc::CritScope lock(&test_->decoded_frame_section_);
test_->decoded_frame_.emplace(frame);
test_->decoded_qp_ = qp;
test_->decoded_frame_event_.Set();
}
void VideoCodecUnitTest::SetUp() {
webrtc::test::CodecSettings(kVideoCodecVP8, &codec_settings_);
codec_settings_.startBitrate = kStartBitrate;
codec_settings_.maxBitrate = kMaxBitrate;
codec_settings_.maxFramerate = kMaxFramerate;
codec_settings_.width = kWidth;
codec_settings_.height = kHeight;
ModifyCodecSettings(&codec_settings_);
input_frame_generator_ = test::CreateSquareFrameGenerator(
codec_settings_.width, codec_settings_.height,
test::FrameGeneratorInterface::OutputType::kI420, absl::optional<int>());
encoder_ = CreateEncoder();
decoder_ = CreateDecoder();
encoder_->RegisterEncodeCompleteCallback(&encode_complete_callback_);
decoder_->RegisterDecodeCompleteCallback(&decode_complete_callback_);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(
&codec_settings_,
VideoEncoder::Settings(kCapabilities, 1 /* number of cores */,
0 /* max payload size (unused) */)));
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
decoder_->InitDecode(&codec_settings_, 1 /* number of cores */));
}
void VideoCodecUnitTest::ModifyCodecSettings(VideoCodec* codec_settings) {}
VideoFrame VideoCodecUnitTest::NextInputFrame() {
test::FrameGeneratorInterface::VideoFrameData frame_data =
input_frame_generator_->NextFrame();
VideoFrame input_frame = VideoFrame::Builder()
.set_video_frame_buffer(frame_data.buffer)
.set_update_rect(frame_data.update_rect)
.build();
const uint32_t timestamp =
last_input_frame_timestamp_ +
kVideoPayloadTypeFrequency / codec_settings_.maxFramerate;
input_frame.set_timestamp(timestamp);
last_input_frame_timestamp_ = timestamp;
return input_frame;
}
bool VideoCodecUnitTest::WaitForEncodedFrame(
EncodedImage* frame,
CodecSpecificInfo* codec_specific_info) {
std::vector<EncodedImage> frames;
std::vector<CodecSpecificInfo> codec_specific_infos;
if (!WaitForEncodedFrames(&frames, &codec_specific_infos))
return false;
EXPECT_EQ(frames.size(), static_cast<size_t>(1));
EXPECT_EQ(frames.size(), codec_specific_infos.size());
*frame = frames[0];
*codec_specific_info = codec_specific_infos[0];
return true;
}
void VideoCodecUnitTest::SetWaitForEncodedFramesThreshold(size_t num_frames) {
rtc::CritScope lock(&encoded_frame_section_);
wait_for_encoded_frames_threshold_ = num_frames;
}
bool VideoCodecUnitTest::WaitForEncodedFrames(
std::vector<EncodedImage>* frames,
std::vector<CodecSpecificInfo>* codec_specific_info) {
EXPECT_TRUE(encoded_frame_event_.Wait(kEncodeTimeoutMs))
<< "Timed out while waiting for encoded frame.";
// This becomes unsafe if there are multiple threads waiting for frames.
rtc::CritScope lock(&encoded_frame_section_);
EXPECT_FALSE(encoded_frames_.empty());
EXPECT_FALSE(codec_specific_infos_.empty());
EXPECT_EQ(encoded_frames_.size(), codec_specific_infos_.size());
if (!encoded_frames_.empty()) {
*frames = encoded_frames_;
encoded_frames_.clear();
RTC_DCHECK(!codec_specific_infos_.empty());
*codec_specific_info = codec_specific_infos_;
codec_specific_infos_.clear();
return true;
} else {
return false;
}
}
bool VideoCodecUnitTest::WaitForDecodedFrame(std::unique_ptr<VideoFrame>* frame,
absl::optional<uint8_t>* qp) {
bool ret = decoded_frame_event_.Wait(kDecodeTimeoutMs);
EXPECT_TRUE(ret) << "Timed out while waiting for a decoded frame.";
// This becomes unsafe if there are multiple threads waiting for frames.
rtc::CritScope lock(&decoded_frame_section_);
EXPECT_TRUE(decoded_frame_);
if (decoded_frame_) {
frame->reset(new VideoFrame(std::move(*decoded_frame_)));
*qp = decoded_qp_;
decoded_frame_.reset();
return true;
} else {
return false;
}
}
size_t VideoCodecUnitTest::GetNumEncodedFrames() {
rtc::CritScope lock(&encoded_frame_section_);
return encoded_frames_.size();
}
} // namespace webrtc

129
modules/video_coding/codecs/test/video_codec_unittest.h Normal file
View File

@ -0,0 +1,129 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_UNITTEST_H_
#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_UNITTEST_H_
#include <memory>
#include <vector>
#include "api/test/frame_generator_interface.h"
#include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/video_encoder.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/utility/vp8_header_parser.h"
#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h"
#include "rtc_base/critical_section.h"
#include "rtc_base/event.h"
#include "rtc_base/thread_annotations.h"
#include "test/gtest.h"
namespace webrtc {
class VideoCodecUnitTest : public ::testing::Test {
public:
VideoCodecUnitTest()
: encode_complete_callback_(this),
decode_complete_callback_(this),
wait_for_encoded_frames_threshold_(1),
last_input_frame_timestamp_(0) {}
protected:
class FakeEncodeCompleteCallback : public webrtc::EncodedImageCallback {
public:
explicit FakeEncodeCompleteCallback(VideoCodecUnitTest* test)
: test_(test) {}
Result OnEncodedImage(const EncodedImage& frame,
const CodecSpecificInfo* codec_specific_info,
const RTPFragmentationHeader* fragmentation);
private:
VideoCodecUnitTest* const test_;
};
class FakeDecodeCompleteCallback : public webrtc::DecodedImageCallback {
public:
explicit FakeDecodeCompleteCallback(VideoCodecUnitTest* test)
: test_(test) {}
int32_t Decoded(VideoFrame& frame) override {
RTC_NOTREACHED();
return -1;
}
int32_t Decoded(VideoFrame& frame, int64_t decode_time_ms) override {
RTC_NOTREACHED();
return -1;
}
void Decoded(VideoFrame& frame,
absl::optional<int32_t> decode_time_ms,
absl::optional<uint8_t> qp) override;
private:
VideoCodecUnitTest* const test_;
};
virtual std::unique_ptr<VideoEncoder> CreateEncoder() = 0;
virtual std::unique_ptr<VideoDecoder> CreateDecoder() = 0;
void SetUp() override;
virtual void ModifyCodecSettings(VideoCodec* codec_settings);
VideoFrame NextInputFrame();
// Helper method for waiting a single encoded frame.
bool WaitForEncodedFrame(EncodedImage* frame,
CodecSpecificInfo* codec_specific_info);
// Helper methods for waiting for multiple encoded frames. Caller must
// define how many frames are to be waited for via |num_frames| before calling
// Encode(). Then, they can expect to retrive them via WaitForEncodedFrames().
void SetWaitForEncodedFramesThreshold(size_t num_frames);
bool WaitForEncodedFrames(
std::vector<EncodedImage>* frames,
std::vector<CodecSpecificInfo>* codec_specific_info);
// Helper method for waiting a single decoded frame.
bool WaitForDecodedFrame(std::unique_ptr<VideoFrame>* frame,
absl::optional<uint8_t>* qp);
size_t GetNumEncodedFrames();
VideoCodec codec_settings_;
std::unique_ptr<VideoEncoder> encoder_;
std::unique_ptr<VideoDecoder> decoder_;
std::unique_ptr<test::FrameGeneratorInterface> input_frame_generator_;
private:
FakeEncodeCompleteCallback encode_complete_callback_;
FakeDecodeCompleteCallback decode_complete_callback_;
rtc::Event encoded_frame_event_;
rtc::CriticalSection encoded_frame_section_;
size_t wait_for_encoded_frames_threshold_;
std::vector<EncodedImage> encoded_frames_
RTC_GUARDED_BY(encoded_frame_section_);
std::vector<CodecSpecificInfo> codec_specific_infos_
RTC_GUARDED_BY(encoded_frame_section_);
rtc::Event decoded_frame_event_;
rtc::CriticalSection decoded_frame_section_;
absl::optional<VideoFrame> decoded_frame_
RTC_GUARDED_BY(decoded_frame_section_);
absl::optional<uint8_t> decoded_qp_ RTC_GUARDED_BY(decoded_frame_section_);
uint32_t last_input_frame_timestamp_;
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_UNITTEST_H_

152
modules/video_coding/codecs/test/video_encoder_decoder_instantiation_tests.cc Normal file
View File

@ -0,0 +1,152 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <memory>
#include <vector>
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/video_decoder_factory.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/video_encoder_factory.h"
#if defined(WEBRTC_ANDROID)
#include "modules/video_coding/codecs/test/android_codec_factory_helper.h"
#elif defined(WEBRTC_IOS)
#include "modules/video_coding/codecs/test/objc_codec_factory_helper.h"
#endif
#include "test/gtest.h"
#include "test/video_codec_settings.h"
namespace webrtc {
namespace test {
namespace {
const VideoEncoder::Capabilities kCapabilities(false);
int32_t InitEncoder(VideoCodecType codec_type, VideoEncoder* encoder) {
VideoCodec codec;
CodecSettings(codec_type, &codec);
codec.width = 640;
codec.height = 480;
codec.maxFramerate = 30;
RTC_CHECK(encoder);
return encoder->InitEncode(
&codec, VideoEncoder::Settings(kCapabilities, 1 /* number_of_cores */,
1200 /* max_payload_size */));
}
int32_t InitDecoder(VideoCodecType codec_type, VideoDecoder* decoder) {
VideoCodec codec;
CodecSettings(codec_type, &codec);
codec.width = 640;
codec.height = 480;
codec.maxFramerate = 30;
RTC_CHECK(decoder);
return decoder->InitDecode(&codec, 1 /* number_of_cores */);
}
} // namespace
class VideoEncoderDecoderInstantiationTest
: public ::testing::Test,
public ::testing::WithParamInterface<::testing::tuple<int, int>> {
protected:
VideoEncoderDecoderInstantiationTest()
: vp8_format_("VP8"),
vp9_format_("VP9"),
h264cbp_format_("H264"),
num_encoders_(::testing::get<0>(GetParam())),
num_decoders_(::testing::get<1>(GetParam())) {
#if defined(WEBRTC_ANDROID)
InitializeAndroidObjects();
encoder_factory_ = CreateAndroidEncoderFactory();
decoder_factory_ = CreateAndroidDecoderFactory();
#elif defined(WEBRTC_IOS)
encoder_factory_ = CreateObjCEncoderFactory();
decoder_factory_ = CreateObjCDecoderFactory();
#else
RTC_NOTREACHED() << "Only support Android and iOS.";
#endif
}
~VideoEncoderDecoderInstantiationTest() {
for (auto& encoder : encoders_) {
encoder->Release();
}
for (auto& decoder : decoders_) {
decoder->Release();
}
}
const SdpVideoFormat vp8_format_;
const SdpVideoFormat vp9_format_;
const SdpVideoFormat h264cbp_format_;
std::unique_ptr<VideoEncoderFactory> encoder_factory_;
std::unique_ptr<VideoDecoderFactory> decoder_factory_;
const int num_encoders_;
const int num_decoders_;
std::vector<std::unique_ptr<VideoEncoder>> encoders_;
std::vector<std::unique_ptr<VideoDecoder>> decoders_;
};
INSTANTIATE_TEST_SUITE_P(MultipleEncoders,
VideoEncoderDecoderInstantiationTest,
::testing::Combine(::testing::Range(1, 4),
::testing::Range(1, 2)));
INSTANTIATE_TEST_SUITE_P(MultipleDecoders,
VideoEncoderDecoderInstantiationTest,
::testing::Combine(::testing::Range(1, 2),
::testing::Range(1, 9)));
INSTANTIATE_TEST_SUITE_P(MultipleEncodersDecoders,
VideoEncoderDecoderInstantiationTest,
::testing::Combine(::testing::Range(1, 4),
::testing::Range(1, 9)));
// TODO(brandtr): Check that the factories actually support the codecs before
// trying to instantiate. Currently, we will just crash with a Java exception
// if the factory does not support the codec.
TEST_P(VideoEncoderDecoderInstantiationTest, DISABLED_InstantiateVp8Codecs) {
for (int i = 0; i < num_encoders_; ++i) {
std::unique_ptr<VideoEncoder> encoder =
encoder_factory_->CreateVideoEncoder(vp8_format_);
EXPECT_EQ(0, InitEncoder(kVideoCodecVP8, encoder.get()));
encoders_.emplace_back(std::move(encoder));
}
for (int i = 0; i < num_decoders_; ++i) {
std::unique_ptr<VideoDecoder> decoder =
decoder_factory_->CreateVideoDecoder(vp8_format_);
EXPECT_EQ(0, InitDecoder(kVideoCodecVP8, decoder.get()));
decoders_.emplace_back(std::move(decoder));
}
}
TEST_P(VideoEncoderDecoderInstantiationTest,
DISABLED_InstantiateH264CBPCodecs) {
for (int i = 0; i < num_encoders_; ++i) {
std::unique_ptr<VideoEncoder> encoder =
encoder_factory_->CreateVideoEncoder(h264cbp_format_);
EXPECT_EQ(0, InitEncoder(kVideoCodecH264, encoder.get()));
encoders_.emplace_back(std::move(encoder));
}
for (int i = 0; i < num_decoders_; ++i) {
std::unique_ptr<VideoDecoder> decoder =
decoder_factory_->CreateVideoDecoder(h264cbp_format_);
EXPECT_EQ(0, InitDecoder(kVideoCodecH264, decoder.get()));
decoders_.emplace_back(std::move(decoder));
}
}
} // namespace test
} // namespace webrtc

63
modules/video_coding/codecs/test/videocodec_test_fixture_config_unittest.cc Normal file
View File

@ -0,0 +1,63 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <stddef.h>
#include "api/test/videocodec_test_fixture.h"
#include "api/video_codecs/video_codec.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "test/video_codec_settings.h"
using ::testing::ElementsAre;
namespace webrtc {
namespace test {
using Config = VideoCodecTestFixture::Config;
namespace {
const size_t kNumTemporalLayers = 2;
} // namespace
TEST(Config, NumberOfCoresWithUseSingleCore) {
Config config;
config.use_single_core = true;
EXPECT_EQ(1u, config.NumberOfCores());
}
TEST(Config, NumberOfCoresWithoutUseSingleCore) {
Config config;
config.use_single_core = false;
EXPECT_GE(config.NumberOfCores(), 1u);
}
TEST(Config, NumberOfTemporalLayersIsOne) {
Config config;
webrtc::test::CodecSettings(kVideoCodecH264, &config.codec_settings);
EXPECT_EQ(1u, config.NumberOfTemporalLayers());
}
TEST(Config, NumberOfTemporalLayers_Vp8) {
Config config;
webrtc::test::CodecSettings(kVideoCodecVP8, &config.codec_settings);
config.codec_settings.VP8()->numberOfTemporalLayers = kNumTemporalLayers;
EXPECT_EQ(kNumTemporalLayers, config.NumberOfTemporalLayers());
}
TEST(Config, NumberOfTemporalLayers_Vp9) {
Config config;
webrtc::test::CodecSettings(kVideoCodecVP9, &config.codec_settings);
config.codec_settings.VP9()->numberOfTemporalLayers = kNumTemporalLayers;
EXPECT_EQ(kNumTemporalLayers, config.NumberOfTemporalLayers());
}
} // namespace test
} // namespace webrtc

747
modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc Normal file
View File

@ -0,0 +1,747 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/test/videocodec_test_fixture_impl.h"
#include <stdint.h>
#include <stdio.h>
#include <algorithm>
#include <cmath>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/video/video_bitrate_allocation.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/video_encoder_config.h"
#include "common_video/h264/h264_common.h"
#include "media/base/h264_profile_level_id.h"
#include "media/base/media_constants.h"
#include "media/engine/internal_decoder_factory.h"
#include "media/engine/internal_encoder_factory.h"
#include "media/engine/simulcast.h"
#include "modules/video_coding/codecs/h264/include/h264_globals.h"
#include "modules/video_coding/codecs/vp9/svc_config.h"
#include "modules/video_coding/utility/ivf_file_writer.h"
#include "rtc_base/checks.h"
#include "rtc_base/cpu_time.h"
#include "rtc_base/logging.h"
#include "rtc_base/strings/string_builder.h"
#include "rtc_base/time_utils.h"
#include "system_wrappers/include/cpu_info.h"
#include "system_wrappers/include/sleep.h"
#include "test/gtest.h"
#include "test/testsupport/file_utils.h"
#include "test/testsupport/frame_writer.h"
#include "test/testsupport/perf_test.h"
#include "test/video_codec_settings.h"
namespace webrtc {
namespace test {
using VideoStatistics = VideoCodecTestStats::VideoStatistics;
namespace {
const int kBaseKeyFrameInterval = 3000;
const double kBitratePriority = 1.0;
const int kMaxFramerateFps = 30;
const int kMaxQp = 56;
void ConfigureSimulcast(VideoCodec* codec_settings) {
const std::vector<webrtc::VideoStream> streams = cricket::GetSimulcastConfig(
/*min_layer=*/1, codec_settings->numberOfSimulcastStreams,
codec_settings->width, codec_settings->height, kBitratePriority, kMaxQp,
/* is_screenshare = */ false, true);
for (size_t i = 0; i < streams.size(); ++i) {
SimulcastStream* ss = &codec_settings->simulcastStream[i];
ss->width = static_cast<uint16_t>(streams[i].width);
ss->height = static_cast<uint16_t>(streams[i].height);
ss->numberOfTemporalLayers =
static_cast<unsigned char>(*streams[i].num_temporal_layers);
ss->maxBitrate = streams[i].max_bitrate_bps / 1000;
ss->targetBitrate = streams[i].target_bitrate_bps / 1000;
ss->minBitrate = streams[i].min_bitrate_bps / 1000;
ss->qpMax = streams[i].max_qp;
ss->active = true;
}
}
void ConfigureSvc(VideoCodec* codec_settings) {
RTC_CHECK_EQ(kVideoCodecVP9, codec_settings->codecType);
const std::vector<SpatialLayer> layers = GetSvcConfig(
codec_settings->width, codec_settings->height, kMaxFramerateFps,
/*first_active_layer=*/0, codec_settings->VP9()->numberOfSpatialLayers,
codec_settings->VP9()->numberOfTemporalLayers,
/* is_screen_sharing = */ false);
ASSERT_EQ(codec_settings->VP9()->numberOfSpatialLayers, layers.size())
<< "GetSvcConfig returned fewer spatial layers than configured.";
for (size_t i = 0; i < layers.size(); ++i) {
codec_settings->spatialLayers[i] = layers[i];
}
}
std::string CodecSpecificToString(const VideoCodec& codec) {
char buf[1024];
rtc::SimpleStringBuilder ss(buf);
switch (codec.codecType) {
case kVideoCodecVP8:
ss << "complexity: " << static_cast<int>(codec.VP8().complexity);
ss << "\nnum_temporal_layers: "
<< static_cast<int>(codec.VP8().numberOfTemporalLayers);
ss << "\ndenoising: " << codec.VP8().denoisingOn;
ss << "\nautomatic_resize: " << codec.VP8().automaticResizeOn;
ss << "\nframe_dropping: " << codec.VP8().frameDroppingOn;
ss << "\nkey_frame_interval: " << codec.VP8().keyFrameInterval;
break;
case kVideoCodecVP9:
ss << "complexity: " << static_cast<int>(codec.VP9().complexity);
ss << "\nnum_temporal_layers: "
<< static_cast<int>(codec.VP9().numberOfTemporalLayers);
ss << "\nnum_spatial_layers: "
<< static_cast<int>(codec.VP9().numberOfSpatialLayers);
ss << "\ndenoising: " << codec.VP9().denoisingOn;
ss << "\nframe_dropping: " << codec.VP9().frameDroppingOn;
ss << "\nkey_frame_interval: " << codec.VP9().keyFrameInterval;
ss << "\nadaptive_qp_mode: " << codec.VP9().adaptiveQpMode;
ss << "\nautomatic_resize: " << codec.VP9().automaticResizeOn;
ss << "\nflexible_mode: " << codec.VP9().flexibleMode;
break;
case kVideoCodecH264:
ss << "frame_dropping: " << codec.H264().frameDroppingOn;
ss << "\nkey_frame_interval: " << codec.H264().keyFrameInterval;
break;
default:
break;
}
return ss.str();
}
bool RunEncodeInRealTime(const VideoCodecTestFixtureImpl::Config& config) {
if (config.measure_cpu || config.encode_in_real_time) {
return true;
}
return false;
}
std::string FilenameWithParams(
const VideoCodecTestFixtureImpl::Config& config) {
return config.filename + "_" + config.CodecName() + "_" +
std::to_string(config.codec_settings.startBitrate);
}
} // namespace
VideoCodecTestFixtureImpl::Config::Config() = default;
void VideoCodecTestFixtureImpl::Config::SetCodecSettings(
std::string codec_name,
size_t num_simulcast_streams,
size_t num_spatial_layers,
size_t num_temporal_layers,
bool denoising_on,
bool frame_dropper_on,
bool spatial_resize_on,
size_t width,
size_t height) {
this->codec_name = codec_name;
VideoCodecType codec_type = PayloadStringToCodecType(codec_name);
webrtc::test::CodecSettings(codec_type, &codec_settings);
// TODO(brandtr): Move the setting of |width| and |height| to the tests, and
// DCHECK that they are set before initializing the codec instead.
codec_settings.width = static_cast<uint16_t>(width);
codec_settings.height = static_cast<uint16_t>(height);
RTC_CHECK(num_simulcast_streams >= 1 &&
num_simulcast_streams <= kMaxSimulcastStreams);
RTC_CHECK(num_spatial_layers >= 1 && num_spatial_layers <= kMaxSpatialLayers);
RTC_CHECK(num_temporal_layers >= 1 &&
num_temporal_layers <= kMaxTemporalStreams);
// Simulcast is only available with VP8.
RTC_CHECK(num_simulcast_streams < 2 || codec_type == kVideoCodecVP8);
// Spatial scalability is only available with VP9.
RTC_CHECK(num_spatial_layers < 2 || codec_type == kVideoCodecVP9);
// Some base code requires numberOfSimulcastStreams to be set to zero
// when simulcast is not used.
codec_settings.numberOfSimulcastStreams =
num_simulcast_streams <= 1 ? 0
: static_cast<uint8_t>(num_simulcast_streams);
switch (codec_settings.codecType) {
case kVideoCodecVP8:
codec_settings.VP8()->numberOfTemporalLayers =
static_cast<uint8_t>(num_temporal_layers);
codec_settings.VP8()->denoisingOn = denoising_on;
codec_settings.VP8()->automaticResizeOn = spatial_resize_on;
codec_settings.VP8()->frameDroppingOn = frame_dropper_on;
codec_settings.VP8()->keyFrameInterval = kBaseKeyFrameInterval;
break;
case kVideoCodecVP9:
codec_settings.VP9()->numberOfTemporalLayers =
static_cast<uint8_t>(num_temporal_layers);
codec_settings.VP9()->denoisingOn = denoising_on;
codec_settings.VP9()->frameDroppingOn = frame_dropper_on;
codec_settings.VP9()->keyFrameInterval = kBaseKeyFrameInterval;
codec_settings.VP9()->automaticResizeOn = spatial_resize_on;
codec_settings.VP9()->numberOfSpatialLayers =
static_cast<uint8_t>(num_spatial_layers);
break;
case kVideoCodecH264:
codec_settings.H264()->frameDroppingOn = frame_dropper_on;
codec_settings.H264()->keyFrameInterval = kBaseKeyFrameInterval;
break;
default:
break;
}
if (codec_settings.numberOfSimulcastStreams > 1) {
ConfigureSimulcast(&codec_settings);
} else if (codec_settings.codecType == kVideoCodecVP9 &&
codec_settings.VP9()->numberOfSpatialLayers > 1) {
ConfigureSvc(&codec_settings);
}
}
size_t VideoCodecTestFixtureImpl::Config::NumberOfCores() const {
return use_single_core ? 1 : CpuInfo::DetectNumberOfCores();
}
size_t VideoCodecTestFixtureImpl::Config::NumberOfTemporalLayers() const {
if (codec_settings.codecType == kVideoCodecVP8) {
return codec_settings.VP8().numberOfTemporalLayers;
} else if (codec_settings.codecType == kVideoCodecVP9) {
return codec_settings.VP9().numberOfTemporalLayers;
} else {
return 1;
}
}
size_t VideoCodecTestFixtureImpl::Config::NumberOfSpatialLayers() const {
if (codec_settings.codecType == kVideoCodecVP9) {
return codec_settings.VP9().numberOfSpatialLayers;
} else {
return 1;
}
}
size_t VideoCodecTestFixtureImpl::Config::NumberOfSimulcastStreams() const {
return codec_settings.numberOfSimulcastStreams;
}
std::string VideoCodecTestFixtureImpl::Config::ToString() const {
std::string codec_type = CodecTypeToPayloadString(codec_settings.codecType);
rtc::StringBuilder ss;
ss << "test_name: " << test_name;
ss << "\nfilename: " << filename;
ss << "\nnum_frames: " << num_frames;
ss << "\nmax_payload_size_bytes: " << max_payload_size_bytes;
ss << "\ndecode: " << decode;
ss << "\nuse_single_core: " << use_single_core;
ss << "\nmeasure_cpu: " << measure_cpu;
ss << "\nnum_cores: " << NumberOfCores();
ss << "\ncodec_type: " << codec_type;
ss << "\n\n--> codec_settings";
ss << "\nwidth: " << codec_settings.width;
ss << "\nheight: " << codec_settings.height;
ss << "\nmax_framerate_fps: " << codec_settings.maxFramerate;
ss << "\nstart_bitrate_kbps: " << codec_settings.startBitrate;
ss << "\nmax_bitrate_kbps: " << codec_settings.maxBitrate;
ss << "\nmin_bitrate_kbps: " << codec_settings.minBitrate;
ss << "\nmax_qp: " << codec_settings.qpMax;
ss << "\nnum_simulcast_streams: "
<< static_cast<int>(codec_settings.numberOfSimulcastStreams);
ss << "\n\n--> codec_settings." << codec_type;
ss << "\n" << CodecSpecificToString(codec_settings);
if (codec_settings.numberOfSimulcastStreams > 1) {
for (int i = 0; i < codec_settings.numberOfSimulcastStreams; ++i) {
ss << "\n\n--> codec_settings.simulcastStream[" << i << "]";
const SimulcastStream& simulcast_stream =
codec_settings.simulcastStream[i];
ss << "\nwidth: " << simulcast_stream.width;
ss << "\nheight: " << simulcast_stream.height;
ss << "\nnum_temporal_layers: "
<< static_cast<int>(simulcast_stream.numberOfTemporalLayers);
ss << "\nmin_bitrate_kbps: " << simulcast_stream.minBitrate;
ss << "\ntarget_bitrate_kbps: " << simulcast_stream.targetBitrate;
ss << "\nmax_bitrate_kbps: " << simulcast_stream.maxBitrate;
ss << "\nmax_qp: " << simulcast_stream.qpMax;
ss << "\nactive: " << simulcast_stream.active;
}
}
ss << "\n";
return ss.Release();
}
std::string VideoCodecTestFixtureImpl::Config::CodecName() const {
std::string name = codec_name;
if (name.empty()) {
name = CodecTypeToPayloadString(codec_settings.codecType);
}
if (codec_settings.codecType == kVideoCodecH264) {
if (h264_codec_settings.profile == H264::kProfileConstrainedHigh) {
return name + "-CHP";
} else {
RTC_DCHECK_EQ(h264_codec_settings.profile,
H264::kProfileConstrainedBaseline);
return name + "-CBP";
}
}
return name;
}
// TODO(kthelgason): Move this out of the test fixture impl and
// make available as a shared utility class.
void VideoCodecTestFixtureImpl::H264KeyframeChecker::CheckEncodedFrame(
webrtc::VideoCodecType codec,
const EncodedImage& encoded_frame) const {
EXPECT_EQ(kVideoCodecH264, codec);
bool contains_sps = false;
bool contains_pps = false;
bool contains_idr = false;
const std::vector<webrtc::H264::NaluIndex> nalu_indices =
webrtc::H264::FindNaluIndices(encoded_frame.data(), encoded_frame.size());
for (const webrtc::H264::NaluIndex& index : nalu_indices) {
webrtc::H264::NaluType nalu_type = webrtc::H264::ParseNaluType(
encoded_frame.data()[index.payload_start_offset]);
if (nalu_type == webrtc::H264::NaluType::kSps) {
contains_sps = true;
} else if (nalu_type == webrtc::H264::NaluType::kPps) {
contains_pps = true;
} else if (nalu_type == webrtc::H264::NaluType::kIdr) {
contains_idr = true;
}
}
if (encoded_frame._frameType == VideoFrameType::kVideoFrameKey) {
EXPECT_TRUE(contains_sps) << "Keyframe should contain SPS.";
EXPECT_TRUE(contains_pps) << "Keyframe should contain PPS.";
EXPECT_TRUE(contains_idr) << "Keyframe should contain IDR.";
} else if (encoded_frame._frameType == VideoFrameType::kVideoFrameDelta) {
EXPECT_FALSE(contains_sps) << "Delta frame should not contain SPS.";
EXPECT_FALSE(contains_pps) << "Delta frame should not contain PPS.";
EXPECT_FALSE(contains_idr) << "Delta frame should not contain IDR.";
} else {
RTC_NOTREACHED();
}
}
class VideoCodecTestFixtureImpl::CpuProcessTime final {
public:
explicit CpuProcessTime(const Config& config) : config_(config) {}
~CpuProcessTime() {}
void Start() {
if (config_.measure_cpu) {
cpu_time_ -= rtc::GetProcessCpuTimeNanos();
wallclock_time_ -= rtc::SystemTimeNanos();
}
}
void Stop() {
if (config_.measure_cpu) {
cpu_time_ += rtc::GetProcessCpuTimeNanos();
wallclock_time_ += rtc::SystemTimeNanos();
}
}
void Print() const {
if (config_.measure_cpu) {
RTC_LOG(LS_INFO) << "cpu_usage_percent: "
<< GetUsagePercent() / config_.NumberOfCores();
}
}
private:
double GetUsagePercent() const {
return static_cast<double>(cpu_time_) / wallclock_time_ * 100.0;
}
const Config config_;
int64_t cpu_time_ = 0;
int64_t wallclock_time_ = 0;
};
VideoCodecTestFixtureImpl::VideoCodecTestFixtureImpl(Config config)
: encoder_factory_(std::make_unique<InternalEncoderFactory>()),
decoder_factory_(std::make_unique<InternalDecoderFactory>()),
config_(config) {}
VideoCodecTestFixtureImpl::VideoCodecTestFixtureImpl(
Config config,
std::unique_ptr<VideoDecoderFactory> decoder_factory,
std::unique_ptr<VideoEncoderFactory> encoder_factory)
: encoder_factory_(std::move(encoder_factory)),
decoder_factory_(std::move(decoder_factory)),
config_(config) {}
VideoCodecTestFixtureImpl::~VideoCodecTestFixtureImpl() = default;
// Processes all frames in the clip and verifies the result.
void VideoCodecTestFixtureImpl::RunTest(
const std::vector<RateProfile>& rate_profiles,
const std::vector<RateControlThresholds>* rc_thresholds,
const std::vector<QualityThresholds>* quality_thresholds,
const BitstreamThresholds* bs_thresholds) {
RTC_DCHECK(!rate_profiles.empty());
// To emulate operation on a production VideoStreamEncoder, we call the
// codecs on a task queue.
TaskQueueForTest task_queue("VidProc TQ");
SetUpAndInitObjects(&task_queue, rate_profiles[0].target_kbps,
rate_profiles[0].input_fps);
PrintSettings(&task_queue);
ProcessAllFrames(&task_queue, rate_profiles);
ReleaseAndCloseObjects(&task_queue);
AnalyzeAllFrames(rate_profiles, rc_thresholds, quality_thresholds,
bs_thresholds);
}
void VideoCodecTestFixtureImpl::ProcessAllFrames(
TaskQueueForTest* task_queue,
const std::vector<RateProfile>& rate_profiles) {
// Set initial rates.
auto rate_profile = rate_profiles.begin();
task_queue->PostTask([this, rate_profile] {
processor_->SetRates(rate_profile->target_kbps, rate_profile->input_fps);
});
cpu_process_time_->Start();
for (size_t frame_num = 0; frame_num < config_.num_frames; ++frame_num) {
auto next_rate_profile = std::next(rate_profile);
if (next_rate_profile != rate_profiles.end() &&
frame_num == next_rate_profile->frame_num) {
rate_profile = next_rate_profile;
task_queue->PostTask([this, rate_profile] {
processor_->SetRates(rate_profile->target_kbps,
rate_profile->input_fps);
});
}
task_queue->PostTask([this] { processor_->ProcessFrame(); });
if (RunEncodeInRealTime(config_)) {
// Roughly pace the frames.
const int frame_duration_ms =
std::ceil(rtc::kNumMillisecsPerSec / rate_profile->input_fps);
SleepMs(frame_duration_ms);
}
}
// Wait until we know that the last frame has been sent for encode.
task_queue->SendTask([] {}, RTC_FROM_HERE);
// Give the VideoProcessor pipeline some time to process the last frame,
// and then release the codecs.
SleepMs(1 * rtc::kNumMillisecsPerSec);
cpu_process_time_->Stop();
}
void VideoCodecTestFixtureImpl::AnalyzeAllFrames(
const std::vector<RateProfile>& rate_profiles,
const std::vector<RateControlThresholds>* rc_thresholds,
const std::vector<QualityThresholds>* quality_thresholds,
const BitstreamThresholds* bs_thresholds) {
for (size_t rate_profile_idx = 0; rate_profile_idx < rate_profiles.size();
++rate_profile_idx) {
const size_t first_frame_num = rate_profiles[rate_profile_idx].frame_num;
const size_t last_frame_num =
rate_profile_idx + 1 < rate_profiles.size()
? rate_profiles[rate_profile_idx + 1].frame_num - 1
: config_.num_frames - 1;
RTC_CHECK(last_frame_num >= first_frame_num);
VideoStatistics send_stat = stats_.SliceAndCalcAggregatedVideoStatistic(
first_frame_num, last_frame_num);
RTC_LOG(LS_INFO) << "==> Send stats";
RTC_LOG(LS_INFO) << send_stat.ToString("send_") << "\n";
std::vector<VideoStatistics> layer_stats =
stats_.SliceAndCalcLayerVideoStatistic(first_frame_num, last_frame_num);
RTC_LOG(LS_INFO) << "==> Receive stats";
for (const auto& layer_stat : layer_stats) {
RTC_LOG(LS_INFO) << layer_stat.ToString("recv_") << "\n";
// For perf dashboard.
char modifier_buf[256];
rtc::SimpleStringBuilder modifier(modifier_buf);
modifier << "_r" << rate_profile_idx << "_sl" << layer_stat.spatial_idx;
auto PrintResultHelper = [&modifier, this](const std::string& measurement,
double value,
const std::string& units) {
PrintResult(measurement, modifier.str(), config_.test_name, value,
units, /*important=*/false);
};
if (layer_stat.temporal_idx == config_.NumberOfTemporalLayers() - 1) {
PrintResultHelper("enc_speed", layer_stat.enc_speed_fps, "fps");
PrintResultHelper("avg_key_frame_size",
layer_stat.avg_key_frame_size_bytes, "bytes");
PrintResultHelper("num_key_frames", layer_stat.num_key_frames,
"frames");
printf("\n");
}
modifier << "tl" << layer_stat.temporal_idx;
PrintResultHelper("dec_speed", layer_stat.dec_speed_fps, "fps");
PrintResultHelper("avg_delta_frame_size",
layer_stat.avg_delta_frame_size_bytes, "bytes");
PrintResultHelper("bitrate", layer_stat.bitrate_kbps, "kbps");
PrintResultHelper("framerate", layer_stat.framerate_fps, "fps");
PrintResultHelper("avg_psnr_y", layer_stat.avg_psnr_y, "dB");
PrintResultHelper("avg_psnr_u", layer_stat.avg_psnr_u, "dB");
PrintResultHelper("avg_psnr_v", layer_stat.avg_psnr_v, "dB");
PrintResultHelper("min_psnr_yuv", layer_stat.min_psnr, "dB");
PrintResultHelper("avg_qp", layer_stat.avg_qp, "");
printf("\n");
if (layer_stat.temporal_idx == config_.NumberOfTemporalLayers() - 1) {
printf("\n");
}
}
const RateControlThresholds* rc_threshold =
rc_thresholds ? &(*rc_thresholds)[rate_profile_idx] : nullptr;
const QualityThresholds* quality_threshold =
quality_thresholds ? &(*quality_thresholds)[rate_profile_idx] : nullptr;
VerifyVideoStatistic(send_stat, rc_threshold, quality_threshold,
bs_thresholds,
rate_profiles[rate_profile_idx].target_kbps,
rate_profiles[rate_profile_idx].input_fps);
}
if (config_.print_frame_level_stats) {
RTC_LOG(LS_INFO) << "==> Frame stats";
std::vector<VideoCodecTestStats::FrameStatistics> frame_stats =
stats_.GetFrameStatistics();
for (const auto& frame_stat : frame_stats) {
RTC_LOG(LS_INFO) << frame_stat.ToString();
}
}
cpu_process_time_->Print();
}
void VideoCodecTestFixtureImpl::VerifyVideoStatistic(
const VideoStatistics& video_stat,
const RateControlThresholds* rc_thresholds,
const QualityThresholds* quality_thresholds,
const BitstreamThresholds* bs_thresholds,
size_t target_bitrate_kbps,
double input_framerate_fps) {
if (rc_thresholds) {
const float bitrate_mismatch_percent =
100 * std::fabs(1.0f * video_stat.bitrate_kbps - target_bitrate_kbps) /
target_bitrate_kbps;
const float framerate_mismatch_percent =
100 * std::fabs(video_stat.framerate_fps - input_framerate_fps) /
input_framerate_fps;
EXPECT_LE(bitrate_mismatch_percent,
rc_thresholds->max_avg_bitrate_mismatch_percent);
EXPECT_LE(video_stat.time_to_reach_target_bitrate_sec,
rc_thresholds->max_time_to_reach_target_bitrate_sec);
EXPECT_LE(framerate_mismatch_percent,
rc_thresholds->max_avg_framerate_mismatch_percent);
EXPECT_LE(video_stat.avg_delay_sec,
rc_thresholds->max_avg_buffer_level_sec);
EXPECT_LE(video_stat.max_key_frame_delay_sec,
rc_thresholds->max_max_key_frame_delay_sec);
EXPECT_LE(video_stat.max_delta_frame_delay_sec,
rc_thresholds->max_max_delta_frame_delay_sec);
EXPECT_LE(video_stat.num_spatial_resizes,
rc_thresholds->max_num_spatial_resizes);
EXPECT_LE(video_stat.num_key_frames, rc_thresholds->max_num_key_frames);
}
if (quality_thresholds) {
EXPECT_GT(video_stat.avg_psnr, quality_thresholds->min_avg_psnr);
EXPECT_GT(video_stat.min_psnr, quality_thresholds->min_min_psnr);
// SSIM calculation is not optimized and thus it is disabled in real-time
// mode.
if (!config_.encode_in_real_time) {
EXPECT_GT(video_stat.avg_ssim, quality_thresholds->min_avg_ssim);
EXPECT_GT(video_stat.min_ssim, quality_thresholds->min_min_ssim);
}
}
if (bs_thresholds) {
EXPECT_LE(video_stat.max_nalu_size_bytes,
bs_thresholds->max_max_nalu_size_bytes);
}
}
void VideoCodecTestFixtureImpl::CreateEncoderAndDecoder() {
SdpVideoFormat::Parameters params;
if (config_.codec_settings.codecType == kVideoCodecH264) {
const char* packetization_mode =
config_.h264_codec_settings.packetization_mode ==
H264PacketizationMode::NonInterleaved
? "1"
: "0";
params = {{cricket::kH264FmtpProfileLevelId,
*H264::ProfileLevelIdToString(H264::ProfileLevelId(
config_.h264_codec_settings.profile, H264::kLevel3_1))},
{cricket::kH264FmtpPacketizationMode, packetization_mode}};
} else {
params = {};
}
SdpVideoFormat format(config_.codec_name, params);
encoder_ = encoder_factory_->CreateVideoEncoder(format);
EXPECT_TRUE(encoder_) << "Encoder not successfully created.";
const size_t num_simulcast_or_spatial_layers = std::max(
config_.NumberOfSimulcastStreams(), config_.NumberOfSpatialLayers());
for (size_t i = 0; i < num_simulcast_or_spatial_layers; ++i) {
decoders_.push_back(std::unique_ptr<VideoDecoder>(
decoder_factory_->CreateVideoDecoder(format)));
}
for (const auto& decoder : decoders_) {
EXPECT_TRUE(decoder) << "Decoder not successfully created.";
}
}
void VideoCodecTestFixtureImpl::DestroyEncoderAndDecoder() {
decoders_.clear();
encoder_.reset();
}
VideoCodecTestStats& VideoCodecTestFixtureImpl::GetStats() {
return stats_;
}
void VideoCodecTestFixtureImpl::SetUpAndInitObjects(
TaskQueueForTest* task_queue,
size_t initial_bitrate_kbps,
double initial_framerate_fps) {
config_.codec_settings.minBitrate = 0;
config_.codec_settings.startBitrate = static_cast<int>(initial_bitrate_kbps);
config_.codec_settings.maxFramerate = std::ceil(initial_framerate_fps);
// Create file objects for quality analysis.
source_frame_reader_.reset(
new YuvFrameReaderImpl(config_.filepath, config_.codec_settings.width,
config_.codec_settings.height));
EXPECT_TRUE(source_frame_reader_->Init());
RTC_DCHECK(encoded_frame_writers_.empty());
RTC_DCHECK(decoded_frame_writers_.empty());
if (config_.visualization_params.save_encoded_ivf ||
config_.visualization_params.save_decoded_y4m) {
const size_t num_simulcast_or_spatial_layers = std::max(
config_.NumberOfSimulcastStreams(), config_.NumberOfSpatialLayers());
const size_t num_temporal_layers = config_.NumberOfTemporalLayers();
for (size_t simulcast_svc_idx = 0;
simulcast_svc_idx < num_simulcast_or_spatial_layers;
++simulcast_svc_idx) {
const std::string output_filename_base = JoinFilename(
config_.output_path, FilenameWithParams(config_) + "_sl" +
std::to_string(simulcast_svc_idx));
if (config_.visualization_params.save_encoded_ivf) {
for (size_t temporal_idx = 0; temporal_idx < num_temporal_layers;
++temporal_idx) {
const std::string output_file_path = output_filename_base + "tl" +
std::to_string(temporal_idx) +
".ivf";
FileWrapper ivf_file = FileWrapper::OpenWriteOnly(output_file_path);
const VideoProcessor::LayerKey layer_key(simulcast_svc_idx,
temporal_idx);
encoded_frame_writers_[layer_key] =
IvfFileWriter::Wrap(std::move(ivf_file), /*byte_limit=*/0);
}
}
if (config_.visualization_params.save_decoded_y4m) {
FrameWriter* decoded_frame_writer = new Y4mFrameWriterImpl(
output_filename_base + ".y4m", config_.codec_settings.width,
config_.codec_settings.height, config_.codec_settings.maxFramerate);
EXPECT_TRUE(decoded_frame_writer->Init());
decoded_frame_writers_.push_back(
std::unique_ptr<FrameWriter>(decoded_frame_writer));
}
}
}
stats_.Clear();
cpu_process_time_.reset(new CpuProcessTime(config_));
task_queue->SendTask(
[this]() {
CreateEncoderAndDecoder();
processor_ = std::make_unique<VideoProcessor>(
encoder_.get(), &decoders_, source_frame_reader_.get(), config_,
&stats_, &encoded_frame_writers_,
decoded_frame_writers_.empty() ? nullptr : &decoded_frame_writers_);
},
RTC_FROM_HERE);
}
void VideoCodecTestFixtureImpl::ReleaseAndCloseObjects(
TaskQueueForTest* task_queue) {
task_queue->SendTask(
[this]() {
processor_.reset();
// The VideoProcessor must be destroyed before the codecs.
DestroyEncoderAndDecoder();
},
RTC_FROM_HERE);
source_frame_reader_->Close();
// Close visualization files.
for (auto& encoded_frame_writer : encoded_frame_writers_) {
EXPECT_TRUE(encoded_frame_writer.second->Close());
}
encoded_frame_writers_.clear();
for (auto& decoded_frame_writer : decoded_frame_writers_) {
decoded_frame_writer->Close();
}
decoded_frame_writers_.clear();
}
void VideoCodecTestFixtureImpl::PrintSettings(
TaskQueueForTest* task_queue) const {
RTC_LOG(LS_INFO) << "==> Config";
RTC_LOG(LS_INFO) << config_.ToString();
RTC_LOG(LS_INFO) << "==> Codec names";
std::string encoder_name;
std::string decoder_name;
task_queue->SendTask(
[this, &encoder_name, &decoder_name] {
encoder_name = encoder_->GetEncoderInfo().implementation_name;
decoder_name = decoders_.at(0)->ImplementationName();
},
RTC_FROM_HERE);
RTC_LOG(LS_INFO) << "enc_impl_name: " << encoder_name;
RTC_LOG(LS_INFO) << "dec_impl_name: " << decoder_name;
}
} // namespace test
} // namespace webrtc

106
modules/video_coding/codecs/test/videocodec_test_fixture_impl.h Normal file
View File

@ -0,0 +1,106 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEOCODEC_TEST_FIXTURE_IMPL_H_
#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEOCODEC_TEST_FIXTURE_IMPL_H_
#include <memory>
#include <string>
#include <vector>
#include "api/test/videocodec_test_fixture.h"
#include "api/video_codecs/video_decoder_factory.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "common_video/h264/h264_common.h"
#include "modules/video_coding/codecs/test/videocodec_test_stats_impl.h"
#include "modules/video_coding/codecs/test/videoprocessor.h"
#include "modules/video_coding/utility/ivf_file_writer.h"
#include "rtc_base/task_queue_for_test.h"
#include "test/testsupport/frame_reader.h"
#include "test/testsupport/frame_writer.h"
namespace webrtc {
namespace test {
// Integration test for video processor. It does rate control and frame quality
// analysis using frame statistics collected by video processor and logs the
// results. If thresholds are specified it checks that corresponding metrics
// are in desirable range.
class VideoCodecTestFixtureImpl : public VideoCodecTestFixture {
// Verifies that all H.264 keyframes contain SPS/PPS/IDR NALUs.
public:
class H264KeyframeChecker : public EncodedFrameChecker {
public:
void CheckEncodedFrame(webrtc::VideoCodecType codec,
const EncodedImage& encoded_frame) const override;
};
explicit VideoCodecTestFixtureImpl(Config config);
VideoCodecTestFixtureImpl(
Config config,
std::unique_ptr<VideoDecoderFactory> decoder_factory,
std::unique_ptr<VideoEncoderFactory> encoder_factory);
~VideoCodecTestFixtureImpl() override;
void RunTest(const std::vector<RateProfile>& rate_profiles,
const std::vector<RateControlThresholds>* rc_thresholds,
const std::vector<QualityThresholds>* quality_thresholds,
const BitstreamThresholds* bs_thresholds) override;
VideoCodecTestStats& GetStats() override;
private:
class CpuProcessTime;
void CreateEncoderAndDecoder();
void DestroyEncoderAndDecoder();
void SetUpAndInitObjects(TaskQueueForTest* task_queue,
size_t initial_bitrate_kbps,
double initial_framerate_fps);
void ReleaseAndCloseObjects(TaskQueueForTest* task_queue);
void ProcessAllFrames(TaskQueueForTest* task_queue,
const std::vector<RateProfile>& rate_profiles);
void AnalyzeAllFrames(
const std::vector<RateProfile>& rate_profiles,
const std::vector<RateControlThresholds>* rc_thresholds,
const std::vector<QualityThresholds>* quality_thresholds,
const BitstreamThresholds* bs_thresholds);
void VerifyVideoStatistic(
const VideoCodecTestStats::VideoStatistics& video_stat,
const RateControlThresholds* rc_thresholds,
const QualityThresholds* quality_thresholds,
const BitstreamThresholds* bs_thresholds,
size_t target_bitrate_kbps,
double input_framerate_fps);
void PrintSettings(TaskQueueForTest* task_queue) const;
// Codecs.
const std::unique_ptr<VideoEncoderFactory> encoder_factory_;
std::unique_ptr<VideoEncoder> encoder_;
const std::unique_ptr<VideoDecoderFactory> decoder_factory_;
VideoProcessor::VideoDecoderList decoders_;
// Helper objects.
Config config_;
VideoCodecTestStatsImpl stats_;
std::unique_ptr<FrameReader> source_frame_reader_;
VideoProcessor::IvfFileWriterMap encoded_frame_writers_;
VideoProcessor::FrameWriterList decoded_frame_writers_;
std::unique_ptr<VideoProcessor> processor_;
std::unique_ptr<CpuProcessTime> cpu_process_time_;
};
} // namespace test
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEOCODEC_TEST_FIXTURE_IMPL_H_

486
modules/video_coding/codecs/test/videocodec_test_libvpx.cc Normal file
View File

@ -0,0 +1,486 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <memory>
#include <vector>
#include "api/test/create_videocodec_test_fixture.h"
#include "api/test/video/function_video_encoder_factory.h"
#include "api/video_codecs/sdp_video_format.h"
#include "media/base/media_constants.h"
#include "media/engine/internal_decoder_factory.h"
#include "media/engine/internal_encoder_factory.h"
#include "media/engine/simulcast_encoder_adapter.h"
#include "modules/video_coding/utility/vp8_header_parser.h"
#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h"
#include "test/gtest.h"
#include "test/testsupport/file_utils.h"
namespace webrtc {
namespace test {
using VideoStatistics = VideoCodecTestStats::VideoStatistics;
namespace {
// Codec settings.
const int kCifWidth = 352;
const int kCifHeight = 288;
const int kNumFramesShort = 100;
const int kNumFramesLong = 300;
const size_t kBitrateRdPerfKbps[] = {100, 200, 300, 400, 500, 600,
700, 800, 1000, 1250, 1400, 1600,
1800, 2000, 2200, 2500};
const size_t kNumFirstFramesToSkipAtRdPerfAnalysis = 60;
class QpFrameChecker : public VideoCodecTestFixture::EncodedFrameChecker {
public:
void CheckEncodedFrame(webrtc::VideoCodecType codec,
const EncodedImage& encoded_frame) const override {
int qp;
if (codec == kVideoCodecVP8) {
EXPECT_TRUE(vp8::GetQp(encoded_frame.data(), encoded_frame.size(), &qp));
} else if (codec == kVideoCodecVP9) {
EXPECT_TRUE(vp9::GetQp(encoded_frame.data(), encoded_frame.size(), &qp));
} else {
RTC_NOTREACHED();
}
EXPECT_EQ(encoded_frame.qp_, qp) << "Encoder QP != parsed bitstream QP.";
}
};
VideoCodecTestFixture::Config CreateConfig() {
VideoCodecTestFixture::Config config;
config.filename = "foreman_cif";
config.filepath = ResourcePath(config.filename, "yuv");
config.num_frames = kNumFramesLong;
config.use_single_core = true;
return config;
}
void PrintRdPerf(std::map<size_t, std::vector<VideoStatistics>> rd_stats) {
printf("--> Summary\n");
printf("%11s %5s %6s %11s %12s %11s %13s %13s %5s %7s %7s %7s %13s %13s\n",
"uplink_kbps", "width", "height", "spatial_idx", "temporal_idx",
"target_kbps", "downlink_kbps", "framerate_fps", "psnr", "psnr_y",
"psnr_u", "psnr_v", "enc_speed_fps", "dec_speed_fps");
for (const auto& rd_stat : rd_stats) {
const size_t bitrate_kbps = rd_stat.first;
for (const auto& layer_stat : rd_stat.second) {
printf(
"%11zu %5zu %6zu %11zu %12zu %11zu %13zu %13.2f %5.2f %7.2f %7.2f "
"%7.2f"
"%13.2f %13.2f\n",
bitrate_kbps, layer_stat.width, layer_stat.height,
layer_stat.spatial_idx, layer_stat.temporal_idx,
layer_stat.target_bitrate_kbps, layer_stat.bitrate_kbps,
layer_stat.framerate_fps, layer_stat.avg_psnr, layer_stat.avg_psnr_y,
layer_stat.avg_psnr_u, layer_stat.avg_psnr_v,
layer_stat.enc_speed_fps, layer_stat.dec_speed_fps);
}
}
}
} // namespace
#if defined(RTC_ENABLE_VP9)
TEST(VideoCodecTestLibvpx, HighBitrateVP9) {
auto config = CreateConfig();
config.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, false,
kCifWidth, kCifHeight);
config.num_frames = kNumFramesShort;
const auto frame_checker = std::make_unique<QpFrameChecker>();
config.encoded_frame_checker = frame_checker.get();
auto fixture = CreateVideoCodecTestFixture(config);
std::vector<RateProfile> rate_profiles = {{500, 30, 0}};
std::vector<RateControlThresholds> rc_thresholds = {
{5, 1, 0, 1, 0.3, 0.1, 0, 1}};
std::vector<QualityThresholds> quality_thresholds = {{37, 36, 0.94, 0.92}};
fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
}
TEST(VideoCodecTestLibvpx, ChangeBitrateVP9) {
auto config = CreateConfig();
config.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, false,
kCifWidth, kCifHeight);
const auto frame_checker = std::make_unique<QpFrameChecker>();
config.encoded_frame_checker = frame_checker.get();
auto fixture = CreateVideoCodecTestFixture(config);
std::vector<RateProfile> rate_profiles = {
{200, 30, 0}, // target_kbps, input_fps, frame_num
{700, 30, 100},
{500, 30, 200}};
std::vector<RateControlThresholds> rc_thresholds = {
{5, 2, 0, 1, 0.5, 0.1, 0, 1},
{15, 3, 0, 1, 0.5, 0.1, 0, 0},
{11, 2, 0, 1, 0.5, 0.1, 0, 0}};
std::vector<QualityThresholds> quality_thresholds = {
{34, 33, 0.90, 0.88}, {38, 35, 0.95, 0.91}, {35, 34, 0.93, 0.90}};
fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
}
TEST(VideoCodecTestLibvpx, ChangeFramerateVP9) {
auto config = CreateConfig();
config.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, false,
kCifWidth, kCifHeight);
const auto frame_checker = std::make_unique<QpFrameChecker>();
config.encoded_frame_checker = frame_checker.get();
auto fixture = CreateVideoCodecTestFixture(config);
std::vector<RateProfile> rate_profiles = {
{100, 24, 0}, // target_kbps, input_fps, frame_num
{100, 15, 100},
{100, 10, 200}};
// Framerate mismatch should be lower for lower framerate.
std::vector<RateControlThresholds> rc_thresholds = {
{10, 2, 40, 1, 0.5, 0.2, 0, 1},
{8, 2, 5, 1, 0.5, 0.2, 0, 0},
{5, 2, 0, 1, 0.5, 0.3, 0, 0}};
// Quality should be higher for lower framerates for the same content.
std::vector<QualityThresholds> quality_thresholds = {
{33, 32, 0.88, 0.86}, {33.5, 32, 0.90, 0.86}, {33.5, 31.5, 0.90, 0.85}};
fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
}
TEST(VideoCodecTestLibvpx, DenoiserOnVP9) {
auto config = CreateConfig();
config.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, true, true, false,
kCifWidth, kCifHeight);
config.num_frames = kNumFramesShort;
const auto frame_checker = std::make_unique<QpFrameChecker>();
config.encoded_frame_checker = frame_checker.get();
auto fixture = CreateVideoCodecTestFixture(config);
std::vector<RateProfile> rate_profiles = {{500, 30, 0}};
std::vector<RateControlThresholds> rc_thresholds = {
{5, 1, 0, 1, 0.3, 0.1, 0, 1}};
std::vector<QualityThresholds> quality_thresholds = {{37.5, 36, 0.94, 0.93}};
fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
}
TEST(VideoCodecTestLibvpx, VeryLowBitrateVP9) {
auto config = CreateConfig();
config.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, true,
kCifWidth, kCifHeight);
const auto frame_checker = std::make_unique<QpFrameChecker>();
config.encoded_frame_checker = frame_checker.get();
auto fixture = CreateVideoCodecTestFixture(config);
std::vector<RateProfile> rate_profiles = {{50, 30, 0}};
std::vector<RateControlThresholds> rc_thresholds = {
{15, 3, 75, 1, 0.5, 0.4, 2, 1}};
std::vector<QualityThresholds> quality_thresholds = {{28, 25, 0.80, 0.65}};
fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
}
// TODO(marpan): Add temporal layer test for VP9, once changes are in
// vp9 wrapper for this.
#endif // defined(RTC_ENABLE_VP9)
TEST(VideoCodecTestLibvpx, HighBitrateVP8) {
auto config = CreateConfig();
config.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, true, true, false,
kCifWidth, kCifHeight);
config.num_frames = kNumFramesShort;
const auto frame_checker = std::make_unique<QpFrameChecker>();
config.encoded_frame_checker = frame_checker.get();
auto fixture = CreateVideoCodecTestFixture(config);
std::vector<RateProfile> rate_profiles = {{500, 30, 0}};
std::vector<RateControlThresholds> rc_thresholds = {
{5, 1, 0, 1, 0.2, 0.1, 0, 1}};
#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64)
std::vector<QualityThresholds> quality_thresholds = {{35, 33, 0.91, 0.89}};
#else
std::vector<QualityThresholds> quality_thresholds = {{37, 35, 0.93, 0.91}};
#endif
fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
}
// The tests below are currently disabled for Android. For ARM, the encoder
// uses |cpu_speed| = 12, as opposed to default |cpu_speed| <= 6 for x86,
// which leads to significantly different quality. The quality and rate control
// settings in the tests below are defined for encoder speed setting
// |cpu_speed| <= ~6. A number of settings would need to be significantly
// modified for the |cpu_speed| = 12 case. For now, keep the tests below
// disabled on Android. Some quality parameter in the above test has been
// adjusted to also pass for |cpu_speed| <= 12.
// TODO(webrtc:9267): Fails on iOS
#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
#define MAYBE_ChangeBitrateVP8 DISABLED_ChangeBitrateVP8
#else
#define MAYBE_ChangeBitrateVP8 ChangeBitrateVP8
#endif
TEST(VideoCodecTestLibvpx, MAYBE_ChangeBitrateVP8) {
auto config = CreateConfig();
config.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, true, true, false,
kCifWidth, kCifHeight);
const auto frame_checker = std::make_unique<QpFrameChecker>();
config.encoded_frame_checker = frame_checker.get();
auto fixture = CreateVideoCodecTestFixture(config);
std::vector<RateProfile> rate_profiles = {
{200, 30, 0}, // target_kbps, input_fps, frame_num
{800, 30, 100},
{500, 30, 200}};
std::vector<RateControlThresholds> rc_thresholds = {
{5, 1, 0, 1, 0.2, 0.1, 0, 1},
{15.5, 1, 0, 1, 0.2, 0.1, 0, 0},
{15, 1, 0, 1, 0.2, 0.1, 0, 0}};
#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64)
std::vector<QualityThresholds> quality_thresholds = {
{31.8, 31, 0.86, 0.85}, {36, 34.8, 0.92, 0.90}, {33.5, 32, 0.90, 0.88}};
#else
std::vector<QualityThresholds> quality_thresholds = {
{33, 32, 0.89, 0.88}, {38, 36, 0.94, 0.93}, {35, 34, 0.92, 0.91}};
#endif
fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
}
// TODO(webrtc:9267): Fails on iOS
#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
#define MAYBE_ChangeFramerateVP8 DISABLED_ChangeFramerateVP8
#else
#define MAYBE_ChangeFramerateVP8 ChangeFramerateVP8
#endif
TEST(VideoCodecTestLibvpx, MAYBE_ChangeFramerateVP8) {
auto config = CreateConfig();
config.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, true, true, false,
kCifWidth, kCifHeight);
const auto frame_checker = std::make_unique<QpFrameChecker>();
config.encoded_frame_checker = frame_checker.get();
auto fixture = CreateVideoCodecTestFixture(config);
std::vector<RateProfile> rate_profiles = {
{80, 24, 0}, // target_kbps, input_fps, frame_index_rate_update
{80, 15, 100},
{80, 10, 200}};
#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64)
std::vector<RateControlThresholds> rc_thresholds = {
{10, 2, 60, 1, 0.3, 0.3, 0, 1},
{10, 2, 30, 1, 0.3, 0.3, 0, 0},
{10, 2, 10, 1, 0.3, 0.2, 0, 0}};
#else
std::vector<RateControlThresholds> rc_thresholds = {
{10, 2, 20, 1, 0.3, 0.15, 0, 1},
{5, 2, 5, 1, 0.3, 0.15, 0, 0},
{4, 2, 1, 1, 0.3, 0.2, 0, 0}};
#endif
#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64)
std::vector<QualityThresholds> quality_thresholds = {
{31, 30, 0.85, 0.84}, {31.5, 30.5, 0.86, 0.84}, {30.5, 29, 0.83, 0.78}};
#else
std::vector<QualityThresholds> quality_thresholds = {
{31, 30, 0.87, 0.86}, {32, 31, 0.89, 0.86}, {32, 30, 0.87, 0.82}};
#endif
fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
}
#if defined(WEBRTC_ANDROID)
#define MAYBE_TemporalLayersVP8 DISABLED_TemporalLayersVP8
#else
#define MAYBE_TemporalLayersVP8 TemporalLayersVP8
#endif
TEST(VideoCodecTestLibvpx, MAYBE_TemporalLayersVP8) {
auto config = CreateConfig();
config.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 3, true, true, false,
kCifWidth, kCifHeight);
const auto frame_checker = std::make_unique<QpFrameChecker>();
config.encoded_frame_checker = frame_checker.get();
auto fixture = CreateVideoCodecTestFixture(config);
std::vector<RateProfile> rate_profiles = {{200, 30, 0}, {400, 30, 150}};
#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64)
std::vector<RateControlThresholds> rc_thresholds = {
{10, 1, 2.1, 1, 0.2, 0.1, 0, 1}, {12, 2, 3, 1, 0.2, 0.1, 0, 1}};
#else
std::vector<RateControlThresholds> rc_thresholds = {
{5, 1, 0, 1, 0.2, 0.1, 0, 1}, {10, 2, 0, 1, 0.2, 0.1, 0, 1}};
#endif
// Min SSIM drops because of high motion scene with complex backgound (trees).
#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64)
std::vector<QualityThresholds> quality_thresholds = {{31, 30, 0.85, 0.83},
{31, 28, 0.85, 0.75}};
#else
std::vector<QualityThresholds> quality_thresholds = {{32, 30, 0.88, 0.85},
{33, 30, 0.89, 0.83}};
#endif
fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
}
#if defined(WEBRTC_ANDROID)
#define MAYBE_MultiresVP8 DISABLED_MultiresVP8
#else
#define MAYBE_MultiresVP8 MultiresVP8
#endif
TEST(VideoCodecTestLibvpx, MAYBE_MultiresVP8) {
auto config = CreateConfig();
config.filename = "ConferenceMotion_1280_720_50";
config.filepath = ResourcePath(config.filename, "yuv");
config.num_frames = 100;
config.SetCodecSettings(cricket::kVp8CodecName, 3, 1, 3, true, true, false,
1280, 720);
const auto frame_checker = std::make_unique<QpFrameChecker>();
config.encoded_frame_checker = frame_checker.get();
auto fixture = CreateVideoCodecTestFixture(config);
std::vector<RateProfile> rate_profiles = {{1500, 30, 0}};
#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64)
std::vector<RateControlThresholds> rc_thresholds = {
{4.1, 1.04, 6, 0.18, 0.14, 0.08, 0, 1}};
#else
std::vector<RateControlThresholds> rc_thresholds = {
{5, 1, 5, 1, 0.3, 0.1, 0, 1}};
#endif
std::vector<QualityThresholds> quality_thresholds = {{34, 32, 0.90, 0.88}};
fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
}
#if defined(WEBRTC_ANDROID)
#define MAYBE_SimulcastVP8 DISABLED_SimulcastVP8
#else
#define MAYBE_SimulcastVP8 SimulcastVP8
#endif
TEST(VideoCodecTestLibvpx, MAYBE_SimulcastVP8) {
auto config = CreateConfig();
config.filename = "ConferenceMotion_1280_720_50";
config.filepath = ResourcePath(config.filename, "yuv");
config.num_frames = 100;
config.SetCodecSettings(cricket::kVp8CodecName, 3, 1, 3, true, true, false,
1280, 720);
const auto frame_checker = std::make_unique<QpFrameChecker>();
config.encoded_frame_checker = frame_checker.get();
InternalEncoderFactory internal_encoder_factory;
std::unique_ptr<VideoEncoderFactory> adapted_encoder_factory =
std::make_unique<FunctionVideoEncoderFactory>([&]() {
return std::make_unique<SimulcastEncoderAdapter>(
&internal_encoder_factory, SdpVideoFormat(cricket::kVp8CodecName));
});
std::unique_ptr<InternalDecoderFactory> internal_decoder_factory(
new InternalDecoderFactory());
auto fixture =
CreateVideoCodecTestFixture(config, std::move(internal_decoder_factory),
std::move(adapted_encoder_factory));
std::vector<RateProfile> rate_profiles = {{1500, 30, 0}};
std::vector<RateControlThresholds> rc_thresholds = {
{20, 5, 90, 1, 0.5, 0.3, 0, 1}};
std::vector<QualityThresholds> quality_thresholds = {{34, 32, 0.90, 0.88}};
fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
}
#if defined(WEBRTC_ANDROID)
#define MAYBE_SvcVP9 DISABLED_SvcVP9
#else
#define MAYBE_SvcVP9 SvcVP9
#endif
TEST(VideoCodecTestLibvpx, MAYBE_SvcVP9) {
auto config = CreateConfig();
config.filename = "ConferenceMotion_1280_720_50";
config.filepath = ResourcePath(config.filename, "yuv");
config.num_frames = 100;
config.SetCodecSettings(cricket::kVp9CodecName, 1, 3, 3, true, true, false,
1280, 720);
const auto frame_checker = std::make_unique<QpFrameChecker>();
config.encoded_frame_checker = frame_checker.get();
auto fixture = CreateVideoCodecTestFixture(config);
std::vector<RateProfile> rate_profiles = {{1500, 30, 0}};
std::vector<RateControlThresholds> rc_thresholds = {
{5, 1, 5, 1, 0.3, 0.1, 0, 1}};
std::vector<QualityThresholds> quality_thresholds = {{36, 34, 0.93, 0.90}};
fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
}
TEST(VideoCodecTestLibvpx, DISABLED_MultiresVP8RdPerf) {
auto config = CreateConfig();
config.filename = "FourPeople_1280x720_30";
config.filepath = ResourcePath(config.filename, "yuv");
config.num_frames = 300;
config.print_frame_level_stats = true;
config.SetCodecSettings(cricket::kVp8CodecName, 3, 1, 3, true, true, false,
1280, 720);
const auto frame_checker = std::make_unique<QpFrameChecker>();
config.encoded_frame_checker = frame_checker.get();
auto fixture = CreateVideoCodecTestFixture(config);
std::map<size_t, std::vector<VideoStatistics>> rd_stats;
for (size_t bitrate_kbps : kBitrateRdPerfKbps) {
std::vector<RateProfile> rate_profiles = {{bitrate_kbps, 30, 0}};
fixture->RunTest(rate_profiles, nullptr, nullptr, nullptr);
rd_stats[bitrate_kbps] =
fixture->GetStats().SliceAndCalcLayerVideoStatistic(
kNumFirstFramesToSkipAtRdPerfAnalysis, config.num_frames - 1);
}
PrintRdPerf(rd_stats);
}
TEST(VideoCodecTestLibvpx, DISABLED_SvcVP9RdPerf) {
auto config = CreateConfig();
config.filename = "FourPeople_1280x720_30";
config.filepath = ResourcePath(config.filename, "yuv");
config.num_frames = 300;
config.print_frame_level_stats = true;
config.SetCodecSettings(cricket::kVp9CodecName, 1, 3, 3, true, true, false,
1280, 720);
const auto frame_checker = std::make_unique<QpFrameChecker>();
config.encoded_frame_checker = frame_checker.get();
auto fixture = CreateVideoCodecTestFixture(config);
std::map<size_t, std::vector<VideoStatistics>> rd_stats;
for (size_t bitrate_kbps : kBitrateRdPerfKbps) {
std::vector<RateProfile> rate_profiles = {{bitrate_kbps, 30, 0}};
fixture->RunTest(rate_profiles, nullptr, nullptr, nullptr);
rd_stats[bitrate_kbps] =
fixture->GetStats().SliceAndCalcLayerVideoStatistic(
kNumFirstFramesToSkipAtRdPerfAnalysis, config.num_frames - 1);
}
PrintRdPerf(rd_stats);
}
} // namespace test
} // namespace webrtc

148
modules/video_coding/codecs/test/videocodec_test_mediacodec.cc Normal file
View File

@ -0,0 +1,148 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <memory>
#include <string>
#include <tuple>
#include <vector>
#include "api/test/create_videocodec_test_fixture.h"
#include "media/base/media_constants.h"
#include "modules/video_coding/codecs/test/android_codec_factory_helper.h"
#include "modules/video_coding/codecs/test/videocodec_test_fixture_impl.h"
#include "test/gtest.h"
#include "test/testsupport/file_utils.h"
namespace webrtc {
namespace test {
namespace {
const int kForemanNumFrames = 300;
const int kForemanFramerateFps = 30;
VideoCodecTestFixture::Config CreateConfig() {
VideoCodecTestFixture::Config config;
config.filename = "foreman_cif";
config.filepath = ResourcePath(config.filename, "yuv");
config.num_frames = kForemanNumFrames;
// In order to not overwhelm the OpenMAX buffers in the Android MediaCodec.
config.encode_in_real_time = true;
return config;
}
std::unique_ptr<VideoCodecTestFixture> CreateTestFixtureWithConfig(
VideoCodecTestFixture::Config config) {
InitializeAndroidObjects(); // Idempotent.
auto encoder_factory = CreateAndroidEncoderFactory();
auto decoder_factory = CreateAndroidDecoderFactory();
return CreateVideoCodecTestFixture(config, std::move(decoder_factory),
std::move(encoder_factory));
}
} // namespace
TEST(VideoCodecTestMediaCodec, ForemanCif500kbpsVp8) {
auto config = CreateConfig();
config.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, false, false, false,
352, 288);
auto fixture = CreateTestFixtureWithConfig(config);
std::vector<RateProfile> rate_profiles = {{500, kForemanFramerateFps, 0}};
// The thresholds below may have to be tweaked to let even poor MediaCodec
// implementations pass. If this test fails on the bots, disable it and
// ping brandtr@.
std::vector<RateControlThresholds> rc_thresholds = {
{10, 1, 1, 0.1, 0.2, 0.1, 0, 1}};
std::vector<QualityThresholds> quality_thresholds = {{36, 31, 0.92, 0.86}};
fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
}
TEST(VideoCodecTestMediaCodec, ForemanCif500kbpsH264CBP) {
auto config = CreateConfig();
const auto frame_checker =
std::make_unique<VideoCodecTestFixtureImpl::H264KeyframeChecker>();
config.encoded_frame_checker = frame_checker.get();
config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, false,
352, 288);
auto fixture = CreateTestFixtureWithConfig(config);
std::vector<RateProfile> rate_profiles = {{500, kForemanFramerateFps, 0}};
// The thresholds below may have to be tweaked to let even poor MediaCodec
// implementations pass. If this test fails on the bots, disable it and
// ping brandtr@.
std::vector<RateControlThresholds> rc_thresholds = {
{10, 1, 1, 0.1, 0.2, 0.1, 0, 1}};
std::vector<QualityThresholds> quality_thresholds = {{36, 31, 0.92, 0.86}};
fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
}
// TODO(brandtr): Enable this test when we have trybots/buildbots with
// HW encoders that support CHP.
TEST(VideoCodecTestMediaCodec, DISABLED_ForemanCif500kbpsH264CHP) {
auto config = CreateConfig();
const auto frame_checker =
std::make_unique<VideoCodecTestFixtureImpl::H264KeyframeChecker>();
config.h264_codec_settings.profile = H264::kProfileConstrainedHigh;
config.encoded_frame_checker = frame_checker.get();
config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, false,
352, 288);
auto fixture = CreateTestFixtureWithConfig(config);
std::vector<RateProfile> rate_profiles = {{500, kForemanFramerateFps, 0}};
// The thresholds below may have to be tweaked to let even poor MediaCodec
// implementations pass. If this test fails on the bots, disable it and
// ping brandtr@.
std::vector<RateControlThresholds> rc_thresholds = {
{5, 1, 0, 0.1, 0.2, 0.1, 0, 1}};
std::vector<QualityThresholds> quality_thresholds = {{37, 35, 0.93, 0.91}};
fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
}
TEST(VideoCodecTestMediaCodec, ForemanMixedRes100kbpsVp8H264) {
auto config = CreateConfig();
const int kNumFrames = 30;
const std::vector<std::string> codecs = {cricket::kVp8CodecName,
cricket::kH264CodecName};
const std::vector<std::tuple<int, int>> resolutions = {
{128, 96}, {160, 120}, {176, 144}, {240, 136}, {320, 240}, {480, 272}};
const std::vector<RateProfile> rate_profiles = {
{100, kForemanFramerateFps, 0}};
const std::vector<QualityThresholds> quality_thresholds = {
{29, 26, 0.8, 0.75}};
for (const auto& codec : codecs) {
for (const auto& resolution : resolutions) {
const int width = std::get<0>(resolution);
const int height = std::get<1>(resolution);
config.filename = std::string("foreman_") + std::to_string(width) + "x" +
std::to_string(height);
config.filepath = ResourcePath(config.filename, "yuv");
config.num_frames = kNumFrames;
config.SetCodecSettings(codec, 1, 1, 1, false, false, false, width,
height);
auto fixture = CreateTestFixtureWithConfig(config);
fixture->RunTest(rate_profiles, nullptr /* rc_thresholds */,
&quality_thresholds, nullptr /* bs_thresholds */);
}
}
}
} // namespace test
} // namespace webrtc

87
modules/video_coding/codecs/test/videocodec_test_openh264.cc Normal file
View File

@ -0,0 +1,87 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <memory>
#include <vector>
#include "api/test/create_videocodec_test_fixture.h"
#include "media/base/media_constants.h"
#include "modules/video_coding/codecs/test/videocodec_test_fixture_impl.h"
#include "test/gtest.h"
#include "test/testsupport/file_utils.h"
namespace webrtc {
namespace test {
namespace {
// Codec settings.
const int kCifWidth = 352;
const int kCifHeight = 288;
const int kNumFrames = 100;
VideoCodecTestFixture::Config CreateConfig() {
VideoCodecTestFixture::Config config;
config.filename = "foreman_cif";
config.filepath = ResourcePath(config.filename, "yuv");
config.num_frames = kNumFrames;
// Only allow encoder/decoder to use single core, for predictability.
config.use_single_core = true;
return config;
}
} // namespace
TEST(VideoCodecTestOpenH264, ConstantHighBitrate) {
auto frame_checker =
std::make_unique<VideoCodecTestFixtureImpl::H264KeyframeChecker>();
auto config = CreateConfig();
config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, true, false,
kCifWidth, kCifHeight);
config.encoded_frame_checker = frame_checker.get();
auto fixture = CreateVideoCodecTestFixture(config);
std::vector<RateProfile> rate_profiles = {{500, 30, 0}};
std::vector<RateControlThresholds> rc_thresholds = {
{5, 1, 0, 0.1, 0.2, 0.1, 0, 1}};
std::vector<QualityThresholds> quality_thresholds = {{37, 35, 0.93, 0.91}};
fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
}
// H264: Enable SingleNalUnit packetization mode. Encoder should split
// large frames into multiple slices and limit length of NAL units.
TEST(VideoCodecTestOpenH264, SingleNalUnit) {
auto frame_checker =
std::make_unique<VideoCodecTestFixtureImpl::H264KeyframeChecker>();
auto config = CreateConfig();
config.h264_codec_settings.packetization_mode =
H264PacketizationMode::SingleNalUnit;
config.max_payload_size_bytes = 500;
config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, true, false,
kCifWidth, kCifHeight);
config.encoded_frame_checker = frame_checker.get();
auto fixture = CreateVideoCodecTestFixture(config);
std::vector<RateProfile> rate_profiles = {{500, 30, 0}};
std::vector<RateControlThresholds> rc_thresholds = {
{5, 1, 0, 0.1, 0.2, 0.1, 0, 1}};
std::vector<QualityThresholds> quality_thresholds = {{37, 35, 0.93, 0.91}};
BitstreamThresholds bs_thresholds = {config.max_payload_size_bytes};
fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds,
&bs_thresholds);
}
} // namespace test
} // namespace webrtc

386
modules/video_coding/codecs/test/videocodec_test_stats_impl.cc Normal file
View File

@ -0,0 +1,386 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/test/videocodec_test_stats_impl.h"
#include <algorithm>
#include <cmath>
#include <iterator>
#include <limits>
#include <numeric>
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "rtc_base/checks.h"
#include "rtc_base/numerics/running_statistics.h"
#include "rtc_base/strings/string_builder.h"
namespace webrtc {
namespace test {
using FrameStatistics = VideoCodecTestStats::FrameStatistics;
using VideoStatistics = VideoCodecTestStats::VideoStatistics;
namespace {
const int kMaxBitrateMismatchPercent = 20;
}
VideoCodecTestStatsImpl::VideoCodecTestStatsImpl() = default;
VideoCodecTestStatsImpl::~VideoCodecTestStatsImpl() = default;
void VideoCodecTestStatsImpl::AddFrame(const FrameStatistics& frame_stat) {
const size_t timestamp = frame_stat.rtp_timestamp;
const size_t layer_idx = frame_stat.spatial_idx;
RTC_DCHECK(rtp_timestamp_to_frame_num_[layer_idx].find(timestamp) ==
rtp_timestamp_to_frame_num_[layer_idx].end());
rtp_timestamp_to_frame_num_[layer_idx][timestamp] = frame_stat.frame_number;
layer_stats_[layer_idx].push_back(frame_stat);
}
FrameStatistics* VideoCodecTestStatsImpl::GetFrame(size_t frame_num,
size_t layer_idx) {
RTC_CHECK_LT(frame_num, layer_stats_[layer_idx].size());
return &layer_stats_[layer_idx][frame_num];
}
FrameStatistics* VideoCodecTestStatsImpl::GetFrameWithTimestamp(
size_t timestamp,
size_t layer_idx) {
RTC_DCHECK(rtp_timestamp_to_frame_num_[layer_idx].find(timestamp) !=
rtp_timestamp_to_frame_num_[layer_idx].end());
return GetFrame(rtp_timestamp_to_frame_num_[layer_idx][timestamp], layer_idx);
}
std::vector<FrameStatistics> VideoCodecTestStatsImpl::GetFrameStatistics() {
size_t capacity = 0;
for (const auto& layer_stat : layer_stats_) {
capacity += layer_stat.second.size();
}
std::vector<FrameStatistics> frame_statistics;
frame_statistics.reserve(capacity);
for (const auto& layer_stat : layer_stats_) {
std::copy(layer_stat.second.cbegin(), layer_stat.second.cend(),
std::back_inserter(frame_statistics));
}
return frame_statistics;
}
std::vector<VideoStatistics>
VideoCodecTestStatsImpl::SliceAndCalcLayerVideoStatistic(
size_t first_frame_num,
size_t last_frame_num) {
std::vector<VideoStatistics> layer_stats;
size_t num_spatial_layers = 0;
size_t num_temporal_layers = 0;
GetNumberOfEncodedLayers(first_frame_num, last_frame_num, &num_spatial_layers,
&num_temporal_layers);
RTC_CHECK_GT(num_spatial_layers, 0);
RTC_CHECK_GT(num_temporal_layers, 0);
for (size_t spatial_idx = 0; spatial_idx < num_spatial_layers;
++spatial_idx) {
for (size_t temporal_idx = 0; temporal_idx < num_temporal_layers;
++temporal_idx) {
VideoStatistics layer_stat = SliceAndCalcVideoStatistic(
first_frame_num, last_frame_num, spatial_idx, temporal_idx, false);
layer_stats.push_back(layer_stat);
}
}
return layer_stats;
}
VideoStatistics VideoCodecTestStatsImpl::SliceAndCalcAggregatedVideoStatistic(
size_t first_frame_num,
size_t last_frame_num) {
size_t num_spatial_layers = 0;
size_t num_temporal_layers = 0;
GetNumberOfEncodedLayers(first_frame_num, last_frame_num, &num_spatial_layers,
&num_temporal_layers);
RTC_CHECK_GT(num_spatial_layers, 0);
RTC_CHECK_GT(num_temporal_layers, 0);
return SliceAndCalcVideoStatistic(first_frame_num, last_frame_num,
num_spatial_layers - 1,
num_temporal_layers - 1, true);
}
size_t VideoCodecTestStatsImpl::Size(size_t spatial_idx) {
return layer_stats_[spatial_idx].size();
}
void VideoCodecTestStatsImpl::Clear() {
layer_stats_.clear();
rtp_timestamp_to_frame_num_.clear();
}
FrameStatistics VideoCodecTestStatsImpl::AggregateFrameStatistic(
size_t frame_num,
size_t spatial_idx,
bool aggregate_independent_layers) {
FrameStatistics frame_stat = *GetFrame(frame_num, spatial_idx);
bool inter_layer_predicted = frame_stat.inter_layer_predicted;
while (spatial_idx-- > 0) {
if (aggregate_independent_layers || inter_layer_predicted) {
FrameStatistics* base_frame_stat = GetFrame(frame_num, spatial_idx);
frame_stat.length_bytes += base_frame_stat->length_bytes;
frame_stat.target_bitrate_kbps += base_frame_stat->target_bitrate_kbps;
inter_layer_predicted = base_frame_stat->inter_layer_predicted;
}
}
return frame_stat;
}
size_t VideoCodecTestStatsImpl::CalcLayerTargetBitrateKbps(
size_t first_frame_num,
size_t last_frame_num,
size_t spatial_idx,
size_t temporal_idx,
bool aggregate_independent_layers) {
size_t target_bitrate_kbps = 0;
// We don't know if superframe includes all required spatial layers because
// of possible frame drops. Run through all frames in specified range, find
// and return maximum target bitrate. Assume that target bitrate in frame
// statistic is specified per temporal layer.
for (size_t frame_num = first_frame_num; frame_num <= last_frame_num;
++frame_num) {
FrameStatistics superframe = AggregateFrameStatistic(
frame_num, spatial_idx, aggregate_independent_layers);
if (superframe.temporal_idx <= temporal_idx) {
target_bitrate_kbps =
std::max(target_bitrate_kbps, superframe.target_bitrate_kbps);
}
}
RTC_DCHECK_GT(target_bitrate_kbps, 0);
return target_bitrate_kbps;
}
VideoStatistics VideoCodecTestStatsImpl::SliceAndCalcVideoStatistic(
size_t first_frame_num,
size_t last_frame_num,
size_t spatial_idx,
size_t temporal_idx,
bool aggregate_independent_layers) {
VideoStatistics video_stat;
float buffer_level_bits = 0.0f;
RunningStatistics<float> buffer_level_sec;
RunningStatistics<size_t> key_frame_size_bytes;
RunningStatistics<size_t> delta_frame_size_bytes;
RunningStatistics<size_t> frame_encoding_time_us;
RunningStatistics<size_t> frame_decoding_time_us;
RunningStatistics<float> psnr_y;
RunningStatistics<float> psnr_u;
RunningStatistics<float> psnr_v;
RunningStatistics<float> psnr;
RunningStatistics<float> ssim;
RunningStatistics<int> qp;
size_t rtp_timestamp_first_frame = 0;
size_t rtp_timestamp_prev_frame = 0;
FrameStatistics last_successfully_decoded_frame(0, 0, 0);
const size_t target_bitrate_kbps =
CalcLayerTargetBitrateKbps(first_frame_num, last_frame_num, spatial_idx,
temporal_idx, aggregate_independent_layers);
RTC_CHECK_GT(target_bitrate_kbps, 0); // We divide by |target_bitrate_kbps|.
for (size_t frame_num = first_frame_num; frame_num <= last_frame_num;
++frame_num) {
FrameStatistics frame_stat = AggregateFrameStatistic(
frame_num, spatial_idx, aggregate_independent_layers);
float time_since_first_frame_sec =
1.0f * (frame_stat.rtp_timestamp - rtp_timestamp_first_frame) /
kVideoPayloadTypeFrequency;
float time_since_prev_frame_sec =
1.0f * (frame_stat.rtp_timestamp - rtp_timestamp_prev_frame) /
kVideoPayloadTypeFrequency;
if (frame_stat.temporal_idx > temporal_idx) {
continue;
}
buffer_level_bits -= time_since_prev_frame_sec * 1000 * target_bitrate_kbps;
buffer_level_bits = std::max(0.0f, buffer_level_bits);
buffer_level_bits += 8.0 * frame_stat.length_bytes;
buffer_level_sec.AddSample(buffer_level_bits /
(1000 * target_bitrate_kbps));
video_stat.length_bytes += frame_stat.length_bytes;
if (frame_stat.encoding_successful) {
++video_stat.num_encoded_frames;
if (frame_stat.frame_type == VideoFrameType::kVideoFrameKey) {
key_frame_size_bytes.AddSample(frame_stat.length_bytes);
++video_stat.num_key_frames;
} else {
delta_frame_size_bytes.AddSample(frame_stat.length_bytes);
}
frame_encoding_time_us.AddSample(frame_stat.encode_time_us);
qp.AddSample(frame_stat.qp);
video_stat.max_nalu_size_bytes = std::max(video_stat.max_nalu_size_bytes,
frame_stat.max_nalu_size_bytes);
}
if (frame_stat.decoding_successful) {
++video_stat.num_decoded_frames;
video_stat.width = std::max(video_stat.width, frame_stat.decoded_width);
video_stat.height =
std::max(video_stat.height, frame_stat.decoded_height);
psnr_y.AddSample(frame_stat.psnr_y);
psnr_u.AddSample(frame_stat.psnr_u);
psnr_v.AddSample(frame_stat.psnr_v);
psnr.AddSample(frame_stat.psnr);
ssim.AddSample(frame_stat.ssim);
if (video_stat.num_decoded_frames > 1) {
if (last_successfully_decoded_frame.decoded_width !=
frame_stat.decoded_width ||
last_successfully_decoded_frame.decoded_height !=
frame_stat.decoded_height) {
++video_stat.num_spatial_resizes;
}
}
frame_decoding_time_us.AddSample(frame_stat.decode_time_us);
last_successfully_decoded_frame = frame_stat;
}
if (video_stat.num_input_frames > 0) {
if (video_stat.time_to_reach_target_bitrate_sec == 0.0f) {
RTC_CHECK_GT(time_since_first_frame_sec, 0);
const float curr_kbps =
8.0 * video_stat.length_bytes / 1000 / time_since_first_frame_sec;
const float bitrate_mismatch_percent =
100 * std::fabs(curr_kbps - target_bitrate_kbps) /
target_bitrate_kbps;
if (bitrate_mismatch_percent < kMaxBitrateMismatchPercent) {
video_stat.time_to_reach_target_bitrate_sec =
time_since_first_frame_sec;
}
}
}
rtp_timestamp_prev_frame = frame_stat.rtp_timestamp;
if (video_stat.num_input_frames == 0) {
rtp_timestamp_first_frame = frame_stat.rtp_timestamp;
}
++video_stat.num_input_frames;
}
const size_t num_frames = last_frame_num - first_frame_num + 1;
const size_t timestamp_delta =
GetFrame(first_frame_num + 1, spatial_idx)->rtp_timestamp -
GetFrame(first_frame_num, spatial_idx)->rtp_timestamp;
RTC_CHECK_GT(timestamp_delta, 0);
const float input_framerate_fps =
1.0 * kVideoPayloadTypeFrequency / timestamp_delta;
RTC_CHECK_GT(input_framerate_fps, 0);
const float duration_sec = num_frames / input_framerate_fps;
video_stat.target_bitrate_kbps = target_bitrate_kbps;
video_stat.input_framerate_fps = input_framerate_fps;
video_stat.spatial_idx = spatial_idx;
video_stat.temporal_idx = temporal_idx;
RTC_CHECK_GT(duration_sec, 0);
video_stat.bitrate_kbps =
static_cast<size_t>(8 * video_stat.length_bytes / 1000 / duration_sec);
video_stat.framerate_fps = video_stat.num_encoded_frames / duration_sec;
// http://bugs.webrtc.org/10400: On Windows, we only get millisecond
// granularity in the frame encode/decode timing measurements.
// So we need to softly avoid a div-by-zero here.
const float mean_encode_time_us =
frame_encoding_time_us.GetMean().value_or(0);
video_stat.enc_speed_fps = mean_encode_time_us > 0.0f
? 1000000.0f / mean_encode_time_us
: std::numeric_limits<float>::max();
const float mean_decode_time_us =
frame_decoding_time_us.GetMean().value_or(0);
video_stat.dec_speed_fps = mean_decode_time_us > 0.0f
? 1000000.0f / mean_decode_time_us
: std::numeric_limits<float>::max();
auto MaxDelaySec =
[target_bitrate_kbps](const RunningStatistics<size_t>& stats) {
return 8 * stats.GetMax().value_or(0) / 1000 / target_bitrate_kbps;
};
video_stat.avg_delay_sec = buffer_level_sec.GetMean().value_or(0);
video_stat.max_key_frame_delay_sec = MaxDelaySec(key_frame_size_bytes);
video_stat.max_delta_frame_delay_sec = MaxDelaySec(key_frame_size_bytes);
video_stat.avg_key_frame_size_bytes =
key_frame_size_bytes.GetMean().value_or(0);
video_stat.avg_delta_frame_size_bytes =
delta_frame_size_bytes.GetMean().value_or(0);
video_stat.avg_qp = qp.GetMean().value_or(0);
video_stat.avg_psnr_y = psnr_y.GetMean().value_or(0);
video_stat.avg_psnr_u = psnr_u.GetMean().value_or(0);
video_stat.avg_psnr_v = psnr_v.GetMean().value_or(0);
video_stat.avg_psnr = psnr.GetMean().value_or(0);
video_stat.min_psnr =
psnr.GetMin().value_or(std::numeric_limits<float>::max());
video_stat.avg_ssim = ssim.GetMean().value_or(0);
video_stat.min_ssim =
ssim.GetMin().value_or(std::numeric_limits<float>::max());
return video_stat;
}
void VideoCodecTestStatsImpl::GetNumberOfEncodedLayers(
size_t first_frame_num,
size_t last_frame_num,
size_t* num_encoded_spatial_layers,
size_t* num_encoded_temporal_layers) {
*num_encoded_spatial_layers = 0;
*num_encoded_temporal_layers = 0;
const size_t num_spatial_layers = layer_stats_.size();
for (size_t frame_num = first_frame_num; frame_num <= last_frame_num;
++frame_num) {
for (size_t spatial_idx = 0; spatial_idx < num_spatial_layers;
++spatial_idx) {
FrameStatistics* frame_stat = GetFrame(frame_num, spatial_idx);
if (frame_stat->encoding_successful) {
*num_encoded_spatial_layers =
std::max(*num_encoded_spatial_layers, frame_stat->spatial_idx + 1);
*num_encoded_temporal_layers = std::max(*num_encoded_temporal_layers,
frame_stat->temporal_idx + 1);
}
}
}
}
} // namespace test
} // namespace webrtc

84
modules/video_coding/codecs/test/videocodec_test_stats_impl.h Normal file
View File

@ -0,0 +1,84 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEOCODEC_TEST_STATS_IMPL_H_
#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEOCODEC_TEST_STATS_IMPL_H_
#include <stddef.h>
#include <map>
#include <string>
#include <vector>
#include "api/test/videocodec_test_stats.h" // NOLINT(build/include)
namespace webrtc {
namespace test {
// Statistics for a sequence of processed frames. This class is not thread safe.
class VideoCodecTestStatsImpl : public VideoCodecTestStats {
public:
VideoCodecTestStatsImpl();
~VideoCodecTestStatsImpl() override;
// Creates a FrameStatistics for the next frame to be processed.
void AddFrame(const FrameStatistics& frame_stat);
// Returns the FrameStatistics corresponding to |frame_number| or |timestamp|.
FrameStatistics* GetFrame(size_t frame_number, size_t spatial_idx);
FrameStatistics* GetFrameWithTimestamp(size_t timestamp, size_t spatial_idx);
// Implements VideoCodecTestStats.
std::vector<FrameStatistics> GetFrameStatistics() override;
std::vector<VideoStatistics> SliceAndCalcLayerVideoStatistic(
size_t first_frame_num,
size_t last_frame_num) override;
VideoStatistics SliceAndCalcAggregatedVideoStatistic(size_t first_frame_num,
size_t last_frame_num);
size_t Size(size_t spatial_idx);
void Clear();
private:
VideoCodecTestStats::FrameStatistics AggregateFrameStatistic(
size_t frame_num,
size_t spatial_idx,
bool aggregate_independent_layers);
size_t CalcLayerTargetBitrateKbps(size_t first_frame_num,
size_t last_frame_num,
size_t spatial_idx,
size_t temporal_idx,
bool aggregate_independent_layers);
VideoCodecTestStats::VideoStatistics SliceAndCalcVideoStatistic(
size_t first_frame_num,
size_t last_frame_num,
size_t spatial_idx,
size_t temporal_idx,
bool aggregate_independent_layers);
void GetNumberOfEncodedLayers(size_t first_frame_num,
size_t last_frame_num,
size_t* num_encoded_spatial_layers,
size_t* num_encoded_temporal_layers);
// layer_idx -> stats.
std::map<size_t, std::vector<FrameStatistics>> layer_stats_;
// layer_idx -> rtp_timestamp -> frame_num.
std::map<size_t, std::map<size_t, size_t>> rtp_timestamp_to_frame_num_;
};
} // namespace test
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEOCODEC_TEST_STATS_IMPL_H_

90
modules/video_coding/codecs/test/videocodec_test_stats_impl_unittest.cc Normal file
View File

@ -0,0 +1,90 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/test/videocodec_test_stats_impl.h"
#include <vector>
#include "test/gmock.h"
#include "test/gtest.h"
namespace webrtc {
namespace test {
using FrameStatistics = VideoCodecTestStatsImpl::FrameStatistics;
namespace {
const size_t kTimestamp = 12345;
using ::testing::AllOf;
using ::testing::Contains;
using ::testing::Field;
} // namespace
TEST(StatsTest, AddAndGetFrame) {
VideoCodecTestStatsImpl stats;
stats.AddFrame(FrameStatistics(0, kTimestamp, 0));
FrameStatistics* frame_stat = stats.GetFrame(0u, 0);
EXPECT_EQ(0u, frame_stat->frame_number);
EXPECT_EQ(kTimestamp, frame_stat->rtp_timestamp);
}
TEST(StatsTest, AddAndGetFrames) {
VideoCodecTestStatsImpl stats;
const size_t kNumFrames = 1000;
for (size_t i = 0; i < kNumFrames; ++i) {
stats.AddFrame(FrameStatistics(i, kTimestamp + i, 0));
FrameStatistics* frame_stat = stats.GetFrame(i, 0);
EXPECT_EQ(i, frame_stat->frame_number);
EXPECT_EQ(kTimestamp + i, frame_stat->rtp_timestamp);
}
EXPECT_EQ(kNumFrames, stats.Size(0));
// Get frame.
size_t i = 22;
FrameStatistics* frame_stat = stats.GetFrameWithTimestamp(kTimestamp + i, 0);
EXPECT_EQ(i, frame_stat->frame_number);
EXPECT_EQ(kTimestamp + i, frame_stat->rtp_timestamp);
}
TEST(StatsTest, AddFrameLayering) {
VideoCodecTestStatsImpl stats;
for (size_t i = 0; i < 3; ++i) {
stats.AddFrame(FrameStatistics(0, kTimestamp + i, i));
FrameStatistics* frame_stat = stats.GetFrame(0u, i);
EXPECT_EQ(0u, frame_stat->frame_number);
EXPECT_EQ(kTimestamp, frame_stat->rtp_timestamp - i);
EXPECT_EQ(1u, stats.Size(i));
}
}
TEST(StatsTest, GetFrameStatistics) {
VideoCodecTestStatsImpl stats;
stats.AddFrame(FrameStatistics(0, kTimestamp, 0));
stats.AddFrame(FrameStatistics(0, kTimestamp, 1));
stats.AddFrame(FrameStatistics(1, kTimestamp + 3000, 0));
stats.AddFrame(FrameStatistics(1, kTimestamp + 3000, 1));
const std::vector<FrameStatistics> frame_stats = stats.GetFrameStatistics();
auto field_matcher = [](size_t frame_number, size_t spatial_idx) {
return AllOf(Field(&FrameStatistics::frame_number, frame_number),
Field(&FrameStatistics::spatial_idx, spatial_idx));
};
EXPECT_THAT(frame_stats, Contains(field_matcher(0, 0)));
EXPECT_THAT(frame_stats, Contains(field_matcher(0, 1)));
EXPECT_THAT(frame_stats, Contains(field_matcher(1, 0)));
EXPECT_THAT(frame_stats, Contains(field_matcher(1, 1)));
}
} // namespace test
} // namespace webrtc

88
modules/video_coding/codecs/test/videocodec_test_videotoolbox.cc Normal file
View File

@ -0,0 +1,88 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <memory>
#include <vector>
#include "api/test/create_videocodec_test_fixture.h"
#include "media/base/media_constants.h"
#include "modules/video_coding/codecs/test/objc_codec_factory_helper.h"
#include "modules/video_coding/codecs/test/videocodec_test_fixture_impl.h"
#include "test/gtest.h"
#include "test/testsupport/file_utils.h"
namespace webrtc {
namespace test {
namespace {
const int kForemanNumFrames = 300;
VideoCodecTestFixture::Config CreateConfig() {
VideoCodecTestFixture::Config config;
config.filename = "foreman_cif";
config.filepath = ResourcePath(config.filename, "yuv");
config.num_frames = kForemanNumFrames;
return config;
}
std::unique_ptr<VideoCodecTestFixture> CreateTestFixtureWithConfig(
VideoCodecTestFixture::Config config) {
auto decoder_factory = CreateObjCDecoderFactory();
auto encoder_factory = CreateObjCEncoderFactory();
return CreateVideoCodecTestFixture(config, std::move(decoder_factory),
std::move(encoder_factory));
}
} // namespace
// TODO(webrtc:9099): Disabled until the issue is fixed.
// HW codecs don't work on simulators. Only run these tests on device.
// #if TARGET_OS_IPHONE && !TARGET_IPHONE_SIMULATOR
// #define MAYBE_TEST TEST
// #else
#define MAYBE_TEST(s, name) TEST(s, DISABLED_##name)
// #endif
// TODO(kthelgason): Use RC Thresholds when the internal bitrateAdjuster is no
// longer in use.
MAYBE_TEST(VideoCodecTestVideoToolbox, ForemanCif500kbpsH264CBP) {
const auto frame_checker =
std::make_unique<VideoCodecTestFixtureImpl::H264KeyframeChecker>();
auto config = CreateConfig();
config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, false,
352, 288);
config.encoded_frame_checker = frame_checker.get();
auto fixture = CreateTestFixtureWithConfig(config);
std::vector<RateProfile> rate_profiles = {{500, 30, 0}};
std::vector<QualityThresholds> quality_thresholds = {{33, 29, 0.9, 0.82}};
fixture->RunTest(rate_profiles, nullptr, &quality_thresholds, nullptr);
}
MAYBE_TEST(VideoCodecTestVideoToolbox, ForemanCif500kbpsH264CHP) {
const auto frame_checker =
std::make_unique<VideoCodecTestFixtureImpl::H264KeyframeChecker>();
auto config = CreateConfig();
config.h264_codec_settings.profile = H264::kProfileConstrainedHigh;
config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, false,
352, 288);
config.encoded_frame_checker = frame_checker.get();
auto fixture = CreateTestFixtureWithConfig(config);
std::vector<RateProfile> rate_profiles = {{500, 30, 0}};
std::vector<QualityThresholds> quality_thresholds = {{33, 30, 0.91, 0.83}};
fixture->RunTest(rate_profiles, nullptr, &quality_thresholds, nullptr);
}
} // namespace test
} // namespace webrtc

592
modules/video_coding/codecs/test/videoprocessor.cc Normal file
View File

@ -0,0 +1,592 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/test/videoprocessor.h"
#include <string.h>
#include <algorithm>
#include <cstddef>
#include <limits>
#include <memory>
#include <utility>
#include "api/scoped_refptr.h"
#include "api/video/builtin_video_bitrate_allocator_factory.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_bitrate_allocator_factory.h"
#include "api/video/video_frame_buffer.h"
#include "api/video/video_rotation.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_encoder.h"
#include "common_video/h264/h264_common.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/video_coding/codecs/interface/common_constants.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/checks.h"
#include "rtc_base/task_utils/to_queued_task.h"
#include "rtc_base/time_utils.h"
#include "test/gtest.h"
#include "third_party/libyuv/include/libyuv/compare.h"
#include "third_party/libyuv/include/libyuv/scale.h"
namespace webrtc {
namespace test {
using FrameStatistics = VideoCodecTestStats::FrameStatistics;
namespace {
const int kMsToRtpTimestamp = kVideoPayloadTypeFrequency / 1000;
const int kMaxBufferedInputFrames = 20;
const VideoEncoder::Capabilities kCapabilities(false);
size_t GetMaxNaluSizeBytes(const EncodedImage& encoded_frame,
const VideoCodecTestFixture::Config& config) {
if (config.codec_settings.codecType != kVideoCodecH264)
return 0;
std::vector<webrtc::H264::NaluIndex> nalu_indices =
webrtc::H264::FindNaluIndices(encoded_frame.data(), encoded_frame.size());
RTC_CHECK(!nalu_indices.empty());
size_t max_size = 0;
for (const webrtc::H264::NaluIndex& index : nalu_indices)
max_size = std::max(max_size, index.payload_size);
return max_size;
}
size_t GetTemporalLayerIndex(const CodecSpecificInfo& codec_specific) {
size_t temporal_idx = 0;
if (codec_specific.codecType == kVideoCodecVP8) {
temporal_idx = codec_specific.codecSpecific.VP8.temporalIdx;
} else if (codec_specific.codecType == kVideoCodecVP9) {
temporal_idx = codec_specific.codecSpecific.VP9.temporal_idx;
}
if (temporal_idx == kNoTemporalIdx) {
temporal_idx = 0;
}
return temporal_idx;
}
int GetElapsedTimeMicroseconds(int64_t start_ns, int64_t stop_ns) {
int64_t diff_us = (stop_ns - start_ns) / rtc::kNumNanosecsPerMicrosec;
RTC_DCHECK_GE(diff_us, std::numeric_limits<int>::min());
RTC_DCHECK_LE(diff_us, std::numeric_limits<int>::max());
return static_cast<int>(diff_us);
}
void ExtractI420BufferWithSize(const VideoFrame& image,
int width,
int height,
rtc::Buffer* buffer) {
if (image.width() != width || image.height() != height) {
EXPECT_DOUBLE_EQ(static_cast<double>(width) / height,
static_cast<double>(image.width()) / image.height());
// Same aspect ratio, no cropping needed.
rtc::scoped_refptr<I420Buffer> scaled(I420Buffer::Create(width, height));
scaled->ScaleFrom(*image.video_frame_buffer()->ToI420());
size_t length =
CalcBufferSize(VideoType::kI420, scaled->width(), scaled->height());
buffer->SetSize(length);
RTC_CHECK_NE(ExtractBuffer(scaled, length, buffer->data()), -1);
return;
}
// No resize.
size_t length =
CalcBufferSize(VideoType::kI420, image.width(), image.height());
buffer->SetSize(length);
RTC_CHECK_NE(ExtractBuffer(image, length, buffer->data()), -1);
}
void CalculateFrameQuality(const I420BufferInterface& ref_buffer,
const I420BufferInterface& dec_buffer,
FrameStatistics* frame_stat,
bool calc_ssim) {
if (ref_buffer.width() != dec_buffer.width() ||
ref_buffer.height() != dec_buffer.height()) {
RTC_CHECK_GE(ref_buffer.width(), dec_buffer.width());
RTC_CHECK_GE(ref_buffer.height(), dec_buffer.height());
// Downscale reference frame.
rtc::scoped_refptr<I420Buffer> scaled_buffer =
I420Buffer::Create(dec_buffer.width(), dec_buffer.height());
I420Scale(ref_buffer.DataY(), ref_buffer.StrideY(), ref_buffer.DataU(),
ref_buffer.StrideU(), ref_buffer.DataV(), ref_buffer.StrideV(),
ref_buffer.width(), ref_buffer.height(),
scaled_buffer->MutableDataY(), scaled_buffer->StrideY(),
scaled_buffer->MutableDataU(), scaled_buffer->StrideU(),
scaled_buffer->MutableDataV(), scaled_buffer->StrideV(),
scaled_buffer->width(), scaled_buffer->height(),
libyuv::kFilterBox);
CalculateFrameQuality(*scaled_buffer, dec_buffer, frame_stat, calc_ssim);
} else {
const uint64_t sse_y = libyuv::ComputeSumSquareErrorPlane(
dec_buffer.DataY(), dec_buffer.StrideY(), ref_buffer.DataY(),
ref_buffer.StrideY(), dec_buffer.width(), dec_buffer.height());
const uint64_t sse_u = libyuv::ComputeSumSquareErrorPlane(
dec_buffer.DataU(), dec_buffer.StrideU(), ref_buffer.DataU(),
ref_buffer.StrideU(), dec_buffer.width() / 2, dec_buffer.height() / 2);
const uint64_t sse_v = libyuv::ComputeSumSquareErrorPlane(
dec_buffer.DataV(), dec_buffer.StrideV(), ref_buffer.DataV(),
ref_buffer.StrideV(), dec_buffer.width() / 2, dec_buffer.height() / 2);
const size_t num_y_samples = dec_buffer.width() * dec_buffer.height();
const size_t num_u_samples =
dec_buffer.width() / 2 * dec_buffer.height() / 2;
frame_stat->psnr_y = libyuv::SumSquareErrorToPsnr(sse_y, num_y_samples);
frame_stat->psnr_u = libyuv::SumSquareErrorToPsnr(sse_u, num_u_samples);
frame_stat->psnr_v = libyuv::SumSquareErrorToPsnr(sse_v, num_u_samples);
frame_stat->psnr = libyuv::SumSquareErrorToPsnr(
sse_y + sse_u + sse_v, num_y_samples + 2 * num_u_samples);
if (calc_ssim) {
frame_stat->ssim = I420SSIM(ref_buffer, dec_buffer);
}
}
}
} // namespace
VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder,
VideoDecoderList* decoders,
FrameReader* input_frame_reader,
const VideoCodecTestFixture::Config& config,
VideoCodecTestStatsImpl* stats,
IvfFileWriterMap* encoded_frame_writers,
FrameWriterList* decoded_frame_writers)
: config_(config),
num_simulcast_or_spatial_layers_(
std::max(config_.NumberOfSimulcastStreams(),
config_.NumberOfSpatialLayers())),
stats_(stats),
encoder_(encoder),
decoders_(decoders),
bitrate_allocator_(
CreateBuiltinVideoBitrateAllocatorFactory()
->CreateVideoBitrateAllocator(config_.codec_settings)),
framerate_fps_(0),
encode_callback_(this),
input_frame_reader_(input_frame_reader),
merged_encoded_frames_(num_simulcast_or_spatial_layers_),
encoded_frame_writers_(encoded_frame_writers),
decoded_frame_writers_(decoded_frame_writers),
last_inputed_frame_num_(0),
last_inputed_timestamp_(0),
first_encoded_frame_(num_simulcast_or_spatial_layers_, true),
last_encoded_frame_num_(num_simulcast_or_spatial_layers_),
first_decoded_frame_(num_simulcast_or_spatial_layers_, true),
last_decoded_frame_num_(num_simulcast_or_spatial_layers_),
decoded_frame_buffer_(num_simulcast_or_spatial_layers_),
post_encode_time_ns_(0) {
// Sanity checks.
RTC_CHECK(TaskQueueBase::Current())
<< "VideoProcessor must be run on a task queue.";
RTC_CHECK(stats_);
RTC_CHECK(encoder_);
RTC_CHECK(decoders_);
RTC_CHECK_EQ(decoders_->size(), num_simulcast_or_spatial_layers_);
RTC_CHECK(input_frame_reader_);
RTC_CHECK(encoded_frame_writers_);
RTC_CHECK(!decoded_frame_writers ||
decoded_frame_writers->size() == num_simulcast_or_spatial_layers_);
// Setup required callbacks for the encoder and decoder and initialize them.
RTC_CHECK_EQ(encoder_->RegisterEncodeCompleteCallback(&encode_callback_),
WEBRTC_VIDEO_CODEC_OK);
// Initialize codecs so that they are ready to receive frames.
RTC_CHECK_EQ(encoder_->InitEncode(
&config_.codec_settings,
VideoEncoder::Settings(
kCapabilities, static_cast<int>(config_.NumberOfCores()),
config_.max_payload_size_bytes)),
WEBRTC_VIDEO_CODEC_OK);
for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) {
decode_callback_.push_back(
std::make_unique<VideoProcessorDecodeCompleteCallback>(this, i));
RTC_CHECK_EQ(
decoders_->at(i)->InitDecode(&config_.codec_settings,
static_cast<int>(config_.NumberOfCores())),
WEBRTC_VIDEO_CODEC_OK);
RTC_CHECK_EQ(decoders_->at(i)->RegisterDecodeCompleteCallback(
decode_callback_.at(i).get()),
WEBRTC_VIDEO_CODEC_OK);
}
}
VideoProcessor::~VideoProcessor() {
RTC_DCHECK_RUN_ON(&sequence_checker_);
// Explicitly reset codecs, in case they don't do that themselves when they
// go out of scope.
RTC_CHECK_EQ(encoder_->Release(), WEBRTC_VIDEO_CODEC_OK);
encoder_->RegisterEncodeCompleteCallback(nullptr);
for (auto& decoder : *decoders_) {
RTC_CHECK_EQ(decoder->Release(), WEBRTC_VIDEO_CODEC_OK);
decoder->RegisterDecodeCompleteCallback(nullptr);
}
// Sanity check.
RTC_CHECK_LE(input_frames_.size(), kMaxBufferedInputFrames);
}
void VideoProcessor::ProcessFrame() {
RTC_DCHECK_RUN_ON(&sequence_checker_);
const size_t frame_number = last_inputed_frame_num_++;
// Get input frame and store for future quality calculation.
rtc::scoped_refptr<I420BufferInterface> buffer =
input_frame_reader_->ReadFrame();
RTC_CHECK(buffer) << "Tried to read too many frames from the file.";
const size_t timestamp =
last_inputed_timestamp_ +
static_cast<size_t>(kVideoPayloadTypeFrequency / framerate_fps_);
VideoFrame input_frame =
VideoFrame::Builder()
.set_video_frame_buffer(buffer)
.set_timestamp_rtp(static_cast<uint32_t>(timestamp))
.set_timestamp_ms(static_cast<int64_t>(timestamp / kMsToRtpTimestamp))
.set_rotation(webrtc::kVideoRotation_0)
.build();
// Store input frame as a reference for quality calculations.
if (config_.decode && !config_.measure_cpu) {
if (input_frames_.size() == kMaxBufferedInputFrames) {
input_frames_.erase(input_frames_.begin());
}
input_frames_.emplace(frame_number, input_frame);
}
last_inputed_timestamp_ = timestamp;
post_encode_time_ns_ = 0;
// Create frame statistics object for all simulcast/spatial layers.
for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) {
FrameStatistics frame_stat(frame_number, timestamp, i);
stats_->AddFrame(frame_stat);
}
// For the highest measurement accuracy of the encode time, the start/stop
// time recordings should wrap the Encode call as tightly as possible.
const int64_t encode_start_ns = rtc::TimeNanos();
for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) {
FrameStatistics* frame_stat = stats_->GetFrame(frame_number, i);
frame_stat->encode_start_ns = encode_start_ns;
}
// Encode.
const std::vector<VideoFrameType> frame_types =
(frame_number == 0)
? std::vector<VideoFrameType>{VideoFrameType::kVideoFrameKey}
: std::vector<VideoFrameType>{VideoFrameType::kVideoFrameDelta};
const int encode_return_code = encoder_->Encode(input_frame, &frame_types);
for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) {
FrameStatistics* frame_stat = stats_->GetFrame(frame_number, i);
frame_stat->encode_return_code = encode_return_code;
}
}
void VideoProcessor::SetRates(size_t bitrate_kbps, double framerate_fps) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
framerate_fps_ = framerate_fps;
bitrate_allocation_ =
bitrate_allocator_->Allocate(VideoBitrateAllocationParameters(
static_cast<uint32_t>(bitrate_kbps * 1000), framerate_fps_));
encoder_->SetRates(
VideoEncoder::RateControlParameters(bitrate_allocation_, framerate_fps_));
}
int32_t VideoProcessor::VideoProcessorDecodeCompleteCallback::Decoded(
VideoFrame& image) {
// Post the callback to the right task queue, if needed.
if (!task_queue_->IsCurrent()) {
// There might be a limited amount of output buffers, make a copy to make
// sure we don't block the decoder.
VideoFrame copy = VideoFrame::Builder()
.set_video_frame_buffer(I420Buffer::Copy(
*image.video_frame_buffer()->ToI420()))
.set_rotation(image.rotation())
.set_timestamp_us(image.timestamp_us())
.set_id(image.id())
.build();
copy.set_timestamp(image.timestamp());
task_queue_->PostTask(ToQueuedTask([this, copy]() {
video_processor_->FrameDecoded(copy, simulcast_svc_idx_);
}));
return 0;
}
video_processor_->FrameDecoded(image, simulcast_svc_idx_);
return 0;
}
void VideoProcessor::FrameEncoded(
const webrtc::EncodedImage& encoded_image,
const webrtc::CodecSpecificInfo& codec_specific) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
// For the highest measurement accuracy of the encode time, the start/stop
// time recordings should wrap the Encode call as tightly as possible.
const int64_t encode_stop_ns = rtc::TimeNanos();
const VideoCodecType codec_type = codec_specific.codecType;
if (config_.encoded_frame_checker) {
config_.encoded_frame_checker->CheckEncodedFrame(codec_type, encoded_image);
}
// Layer metadata.
size_t spatial_idx = encoded_image.SpatialIndex().value_or(0);
size_t temporal_idx = GetTemporalLayerIndex(codec_specific);
FrameStatistics* frame_stat =
stats_->GetFrameWithTimestamp(encoded_image.Timestamp(), spatial_idx);
const size_t frame_number = frame_stat->frame_number;
// Ensure that the encode order is monotonically increasing, within this
// simulcast/spatial layer.
RTC_CHECK(first_encoded_frame_[spatial_idx] ||
last_encoded_frame_num_[spatial_idx] < frame_number);
// Ensure SVC spatial layers are delivered in ascending order.
const size_t num_spatial_layers = config_.NumberOfSpatialLayers();
if (!first_encoded_frame_[spatial_idx] && num_spatial_layers > 1) {
for (size_t i = 0; i < spatial_idx; ++i) {
RTC_CHECK_LE(last_encoded_frame_num_[i], frame_number);
}
for (size_t i = spatial_idx + 1; i < num_simulcast_or_spatial_layers_;
++i) {
RTC_CHECK_GT(frame_number, last_encoded_frame_num_[i]);
}
}
first_encoded_frame_[spatial_idx] = false;
last_encoded_frame_num_[spatial_idx] = frame_number;
// Update frame statistics.
frame_stat->encoding_successful = true;
frame_stat->encode_time_us = GetElapsedTimeMicroseconds(
frame_stat->encode_start_ns, encode_stop_ns - post_encode_time_ns_);
frame_stat->target_bitrate_kbps =
bitrate_allocation_.GetTemporalLayerSum(spatial_idx, temporal_idx) / 1000;
frame_stat->target_framerate_fps = framerate_fps_;
frame_stat->length_bytes = encoded_image.size();
frame_stat->frame_type = encoded_image._frameType;
frame_stat->temporal_idx = temporal_idx;
frame_stat->max_nalu_size_bytes = GetMaxNaluSizeBytes(encoded_image, config_);
frame_stat->qp = encoded_image.qp_;
bool end_of_picture = false;
if (codec_type == kVideoCodecVP9) {
const CodecSpecificInfoVP9& vp9_info = codec_specific.codecSpecific.VP9;
frame_stat->inter_layer_predicted = vp9_info.inter_layer_predicted;
frame_stat->non_ref_for_inter_layer_pred =
vp9_info.non_ref_for_inter_layer_pred;
end_of_picture = vp9_info.end_of_picture;
} else {
frame_stat->inter_layer_predicted = false;
frame_stat->non_ref_for_inter_layer_pred = true;
}
const webrtc::EncodedImage* encoded_image_for_decode = &encoded_image;
if (config_.decode || !encoded_frame_writers_->empty()) {
if (num_spatial_layers > 1) {
encoded_image_for_decode = BuildAndStoreSuperframe(
encoded_image, codec_type, frame_number, spatial_idx,
frame_stat->inter_layer_predicted);
}
}
if (config_.decode) {
DecodeFrame(*encoded_image_for_decode, spatial_idx);
if (end_of_picture && num_spatial_layers > 1) {
// If inter-layer prediction is enabled and upper layer was dropped then
// base layer should be passed to upper layer decoder. Otherwise decoder
// won't be able to decode next superframe.
const EncodedImage* base_image = nullptr;
const FrameStatistics* base_stat = nullptr;
for (size_t i = 0; i < num_spatial_layers; ++i) {
const bool layer_dropped = (first_decoded_frame_[i] ||
last_decoded_frame_num_[i] < frame_number);
// Ensure current layer was decoded.
RTC_CHECK(layer_dropped == false || i != spatial_idx);
if (!layer_dropped) {
base_image = &merged_encoded_frames_[i];
base_stat =
stats_->GetFrameWithTimestamp(encoded_image.Timestamp(), i);
} else if (base_image && !base_stat->non_ref_for_inter_layer_pred) {
DecodeFrame(*base_image, i);
}
}
}
} else {
frame_stat->decode_return_code = WEBRTC_VIDEO_CODEC_NO_OUTPUT;
}
// Since frames in higher TLs typically depend on frames in lower TLs,
// write out frames in lower TLs to bitstream dumps of higher TLs.
for (size_t write_temporal_idx = temporal_idx;
write_temporal_idx < config_.NumberOfTemporalLayers();
++write_temporal_idx) {
const VideoProcessor::LayerKey layer_key(spatial_idx, write_temporal_idx);
auto it = encoded_frame_writers_->find(layer_key);
if (it != encoded_frame_writers_->cend()) {
RTC_CHECK(it->second->WriteFrame(*encoded_image_for_decode,
config_.codec_settings.codecType));
}
}
if (!config_.encode_in_real_time) {
// To get pure encode time for next layers, measure time spent in encode
// callback and subtract it from encode time of next layers.
post_encode_time_ns_ += rtc::TimeNanos() - encode_stop_ns;
}
}
void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame,
size_t spatial_idx) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
// For the highest measurement accuracy of the decode time, the start/stop
// time recordings should wrap the Decode call as tightly as possible.
const int64_t decode_stop_ns = rtc::TimeNanos();
FrameStatistics* frame_stat =
stats_->GetFrameWithTimestamp(decoded_frame.timestamp(), spatial_idx);
const size_t frame_number = frame_stat->frame_number;
if (decoded_frame_writers_ && !first_decoded_frame_[spatial_idx]) {
// Fill drops with last decoded frame to make them look like freeze at
// playback and to keep decoded layers in sync.
for (size_t i = last_decoded_frame_num_[spatial_idx] + 1; i < frame_number;
++i) {
RTC_CHECK(decoded_frame_writers_->at(spatial_idx)
->WriteFrame(decoded_frame_buffer_[spatial_idx].data()));
}
}
// Ensure that the decode order is monotonically increasing, within this
// simulcast/spatial layer.
RTC_CHECK(first_decoded_frame_[spatial_idx] ||
last_decoded_frame_num_[spatial_idx] < frame_number);
first_decoded_frame_[spatial_idx] = false;
last_decoded_frame_num_[spatial_idx] = frame_number;
// Update frame statistics.
frame_stat->decoding_successful = true;
frame_stat->decode_time_us =
GetElapsedTimeMicroseconds(frame_stat->decode_start_ns, decode_stop_ns);
frame_stat->decoded_width = decoded_frame.width();
frame_stat->decoded_height = decoded_frame.height();
// Skip quality metrics calculation to not affect CPU usage.
if (!config_.measure_cpu) {
const auto reference_frame = input_frames_.find(frame_number);
RTC_CHECK(reference_frame != input_frames_.cend())
<< "The codecs are either buffering too much, dropping too much, or "
"being too slow relative the input frame rate.";
// SSIM calculation is not optimized. Skip it in real-time mode.
const bool calc_ssim = !config_.encode_in_real_time;
CalculateFrameQuality(
*reference_frame->second.video_frame_buffer()->ToI420(),
*decoded_frame.video_frame_buffer()->ToI420(), frame_stat, calc_ssim);
// Erase all buffered input frames that we have moved past for all
// simulcast/spatial layers. Never buffer more than
// |kMaxBufferedInputFrames| frames, to protect against long runs of
// consecutive frame drops for a particular layer.
const auto min_last_decoded_frame_num = std::min_element(
last_decoded_frame_num_.cbegin(), last_decoded_frame_num_.cend());
const size_t min_buffered_frame_num = std::max(
0, static_cast<int>(frame_number) - kMaxBufferedInputFrames + 1);
RTC_CHECK(min_last_decoded_frame_num != last_decoded_frame_num_.cend());
const auto input_frames_erase_before = input_frames_.lower_bound(
std::max(*min_last_decoded_frame_num, min_buffered_frame_num));
input_frames_.erase(input_frames_.cbegin(), input_frames_erase_before);
}
if (decoded_frame_writers_) {
ExtractI420BufferWithSize(decoded_frame, config_.codec_settings.width,
config_.codec_settings.height,
&decoded_frame_buffer_[spatial_idx]);
RTC_CHECK_EQ(decoded_frame_buffer_[spatial_idx].size(),
decoded_frame_writers_->at(spatial_idx)->FrameLength());
RTC_CHECK(decoded_frame_writers_->at(spatial_idx)
->WriteFrame(decoded_frame_buffer_[spatial_idx].data()));
}
}
void VideoProcessor::DecodeFrame(const EncodedImage& encoded_image,
size_t spatial_idx) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
FrameStatistics* frame_stat =
stats_->GetFrameWithTimestamp(encoded_image.Timestamp(), spatial_idx);
frame_stat->decode_start_ns = rtc::TimeNanos();
frame_stat->decode_return_code =
decoders_->at(spatial_idx)->Decode(encoded_image, false, 0);
}
const webrtc::EncodedImage* VideoProcessor::BuildAndStoreSuperframe(
const EncodedImage& encoded_image,
const VideoCodecType codec,
size_t frame_number,
size_t spatial_idx,
bool inter_layer_predicted) {
// Should only be called for SVC.
RTC_CHECK_GT(config_.NumberOfSpatialLayers(), 1);
EncodedImage base_image;
RTC_CHECK_EQ(base_image.size(), 0);
// Each SVC layer is decoded with dedicated decoder. Find the nearest
// non-dropped base frame and merge it and current frame into superframe.
if (inter_layer_predicted) {
for (int base_idx = static_cast<int>(spatial_idx) - 1; base_idx >= 0;
--base_idx) {
EncodedImage lower_layer = merged_encoded_frames_.at(base_idx);
if (lower_layer.Timestamp() == encoded_image.Timestamp()) {
base_image = lower_layer;
break;
}
}
}
const size_t payload_size_bytes = base_image.size() + encoded_image.size();
EncodedImage copied_image = encoded_image;
copied_image.SetEncodedData(EncodedImageBuffer::Create(payload_size_bytes));
if (base_image.size()) {
RTC_CHECK(base_image.data());
memcpy(copied_image.data(), base_image.data(), base_image.size());
}
memcpy(copied_image.data() + base_image.size(), encoded_image.data(),
encoded_image.size());
copied_image.set_size(payload_size_bytes);
// Replace previous EncodedImage for this spatial layer.
merged_encoded_frames_.at(spatial_idx) = std::move(copied_image);
return &merged_encoded_frames_.at(spatial_idx);
}
} // namespace test
} // namespace webrtc

261
modules/video_coding/codecs/test/videoprocessor.h Normal file
View File

@ -0,0 +1,261 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_
#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_
#include <stddef.h>
#include <stdint.h>
#include <map>
#include <memory>
#include <utility>
#include <vector>
#include "absl/types/optional.h"
#include "api/task_queue/queued_task.h"
#include "api/task_queue/task_queue_base.h"
#include "api/test/videocodec_test_fixture.h"
#include "api/video/encoded_image.h"
#include "api/video/video_bitrate_allocation.h"
#include "api/video/video_bitrate_allocator.h"
#include "api/video/video_frame.h"
#include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/video_encoder.h"
#include "modules/include/module_common_types.h"
#include "modules/video_coding/codecs/test/videocodec_test_stats_impl.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/utility/ivf_file_writer.h"
#include "rtc_base/buffer.h"
#include "rtc_base/checks.h"
#include "rtc_base/constructor_magic.h"
#include "rtc_base/synchronization/sequence_checker.h"
#include "rtc_base/thread_annotations.h"
#include "rtc_base/thread_checker.h"
#include "test/testsupport/frame_reader.h"
#include "test/testsupport/frame_writer.h"
namespace webrtc {
namespace test {
// Handles encoding/decoding of video using the VideoEncoder/VideoDecoder
// interfaces. This is done in a sequential manner in order to be able to
// measure times properly.
// The class processes a frame at the time for the configured input file.
// It maintains state of where in the source input file the processing is at.
class VideoProcessor {
public:
using VideoDecoderList = std::vector<std::unique_ptr<VideoDecoder>>;
using LayerKey = std::pair<int /* spatial_idx */, int /* temporal_idx */>;
using IvfFileWriterMap = std::map<LayerKey, std::unique_ptr<IvfFileWriter>>;
// TODO(brandtr): Consider changing FrameWriterList to be a FrameWriterMap,
// to be able to save different TLs separately.
using FrameWriterList = std::vector<std::unique_ptr<FrameWriter>>;
VideoProcessor(webrtc::VideoEncoder* encoder,
VideoDecoderList* decoders,
FrameReader* input_frame_reader,
const VideoCodecTestFixture::Config& config,
VideoCodecTestStatsImpl* stats,
IvfFileWriterMap* encoded_frame_writers,
FrameWriterList* decoded_frame_writers);
~VideoProcessor();
// Reads a frame and sends it to the encoder. When the encode callback
// is received, the encoded frame is buffered. After encoding is finished
// buffered frame is sent to decoder. Quality evaluation is done in
// the decode callback.
void ProcessFrame();
// Updates the encoder with target rates. Must be called at least once.
void SetRates(size_t bitrate_kbps, double framerate_fps);
private:
class VideoProcessorEncodeCompleteCallback
: public webrtc::EncodedImageCallback {
public:
explicit VideoProcessorEncodeCompleteCallback(
VideoProcessor* video_processor)
: video_processor_(video_processor),
task_queue_(TaskQueueBase::Current()) {
RTC_DCHECK(video_processor_);
RTC_DCHECK(task_queue_);
}
Result OnEncodedImage(
const webrtc::EncodedImage& encoded_image,
const webrtc::CodecSpecificInfo* codec_specific_info,
const webrtc::RTPFragmentationHeader* fragmentation) override {
RTC_CHECK(codec_specific_info);
// Post the callback to the right task queue, if needed.
if (!task_queue_->IsCurrent()) {
task_queue_->PostTask(std::make_unique<EncodeCallbackTask>(
video_processor_, encoded_image, codec_specific_info));
return Result(Result::OK, 0);
}
video_processor_->FrameEncoded(encoded_image, *codec_specific_info);
return Result(Result::OK, 0);
}
private:
class EncodeCallbackTask : public QueuedTask {
public:
EncodeCallbackTask(VideoProcessor* video_processor,
const webrtc::EncodedImage& encoded_image,
const webrtc::CodecSpecificInfo* codec_specific_info)
: video_processor_(video_processor),
encoded_image_(encoded_image),
codec_specific_info_(*codec_specific_info) {
encoded_image_.Retain();
}
bool Run() override {
video_processor_->FrameEncoded(encoded_image_, codec_specific_info_);
return true;
}
private:
VideoProcessor* const video_processor_;
webrtc::EncodedImage encoded_image_;
const webrtc::CodecSpecificInfo codec_specific_info_;
};
VideoProcessor* const video_processor_;
TaskQueueBase* const task_queue_;
};
class VideoProcessorDecodeCompleteCallback
: public webrtc::DecodedImageCallback {
public:
explicit VideoProcessorDecodeCompleteCallback(
VideoProcessor* video_processor,
size_t simulcast_svc_idx)
: video_processor_(video_processor),
simulcast_svc_idx_(simulcast_svc_idx),
task_queue_(TaskQueueBase::Current()) {
RTC_DCHECK(video_processor_);
RTC_DCHECK(task_queue_);
}
int32_t Decoded(webrtc::VideoFrame& image) override;
int32_t Decoded(webrtc::VideoFrame& image,
int64_t decode_time_ms) override {
return Decoded(image);
}
void Decoded(webrtc::VideoFrame& image,
absl::optional<int32_t> decode_time_ms,
absl::optional<uint8_t> qp) override {
Decoded(image);
}
private:
VideoProcessor* const video_processor_;
const size_t simulcast_svc_idx_;
TaskQueueBase* const task_queue_;
};
// Invoked by the callback adapter when a frame has completed encoding.
void FrameEncoded(const webrtc::EncodedImage& encoded_image,
const webrtc::CodecSpecificInfo& codec_specific);
// Invoked by the callback adapter when a frame has completed decoding.
void FrameDecoded(const webrtc::VideoFrame& image, size_t simulcast_svc_idx);
void DecodeFrame(const EncodedImage& encoded_image, size_t simulcast_svc_idx);
// In order to supply the SVC decoders with super frames containing all
// lower layer frames, we merge and store the layer frames in this method.
const webrtc::EncodedImage* BuildAndStoreSuperframe(
const EncodedImage& encoded_image,
const VideoCodecType codec,
size_t frame_number,
size_t simulcast_svc_idx,
bool inter_layer_predicted) RTC_RUN_ON(sequence_checker_);
// Test input/output.
VideoCodecTestFixture::Config config_ RTC_GUARDED_BY(sequence_checker_);
const size_t num_simulcast_or_spatial_layers_;
VideoCodecTestStatsImpl* const stats_;
// Codecs.
webrtc::VideoEncoder* const encoder_;
VideoDecoderList* const decoders_;
const std::unique_ptr<VideoBitrateAllocator> bitrate_allocator_;
VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(sequence_checker_);
double framerate_fps_ RTC_GUARDED_BY(sequence_checker_);
// Adapters for the codec callbacks.
VideoProcessorEncodeCompleteCallback encode_callback_;
// Assign separate callback object to each decoder. This allows us to identify
// decoded layer in frame decode callback.
// simulcast_svc_idx -> decode callback.
std::vector<std::unique_ptr<VideoProcessorDecodeCompleteCallback>>
decode_callback_;
// Each call to ProcessFrame() will read one frame from |input_frame_reader_|.
FrameReader* const input_frame_reader_;
// Input frames are used as reference for frame quality evaluations.
// Async codecs might queue frames. To handle that we keep input frame
// and release it after corresponding coded frame is decoded and quality
// measurement is done.
// frame_number -> frame.
std::map<size_t, VideoFrame> input_frames_ RTC_GUARDED_BY(sequence_checker_);
// Encoder delivers coded frame layer-by-layer. We store coded frames and
// then, after all layers are encoded, decode them. Such separation of
// frame processing on superframe level simplifies encoding/decoding time
// measurement.
// simulcast_svc_idx -> merged SVC encoded frame.
std::vector<EncodedImage> merged_encoded_frames_
RTC_GUARDED_BY(sequence_checker_);
// These (optional) file writers are used to persistently store the encoded
// and decoded bitstreams. Each frame writer is enabled by being non-null.
IvfFileWriterMap* const encoded_frame_writers_;
FrameWriterList* const decoded_frame_writers_;
// Metadata for inputed/encoded/decoded frames. Used for frame identification,
// frame drop detection, etc. We assume that encoded/decoded frames are
// ordered within each simulcast/spatial layer, but we do not make any
// assumptions of frame ordering between layers.
size_t last_inputed_frame_num_ RTC_GUARDED_BY(sequence_checker_);
size_t last_inputed_timestamp_ RTC_GUARDED_BY(sequence_checker_);
// simulcast_svc_idx -> encode status.
std::vector<bool> first_encoded_frame_ RTC_GUARDED_BY(sequence_checker_);
// simulcast_svc_idx -> frame_number.
std::vector<size_t> last_encoded_frame_num_ RTC_GUARDED_BY(sequence_checker_);
// simulcast_svc_idx -> decode status.
std::vector<bool> first_decoded_frame_ RTC_GUARDED_BY(sequence_checker_);
// simulcast_svc_idx -> frame_number.
std::vector<size_t> last_decoded_frame_num_ RTC_GUARDED_BY(sequence_checker_);
// simulcast_svc_idx -> buffer.
std::vector<rtc::Buffer> decoded_frame_buffer_
RTC_GUARDED_BY(sequence_checker_);
// Time spent in frame encode callback. It is accumulated for layers and
// reset when frame encode starts. When next layer is encoded post-encode time
// is substracted from measured encode time. Thus we get pure encode time.
int64_t post_encode_time_ns_ RTC_GUARDED_BY(sequence_checker_);
// This class must be operated on a TaskQueue.
SequenceChecker sequence_checker_;
RTC_DISALLOW_COPY_AND_ASSIGN(VideoProcessor);
};
} // namespace test
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_

206
modules/video_coding/codecs/test/videoprocessor_unittest.cc Normal file
View File

@ -0,0 +1,206 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/test/videoprocessor.h"
#include <memory>
#include "api/scoped_refptr.h"
#include "api/test/mock_video_decoder.h"
#include "api/test/mock_video_encoder.h"
#include "api/test/videocodec_test_fixture.h"
#include "api/video/i420_buffer.h"
#include "media/base/media_constants.h"
#include "modules/video_coding/codecs/test/videocodec_test_stats_impl.h"
#include "rtc_base/task_queue_for_test.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "test/testsupport/mock/mock_frame_reader.h"
using ::testing::_;
using ::testing::AllOf;
using ::testing::Field;
using ::testing::Property;
using ::testing::ResultOf;
using ::testing::Return;
namespace webrtc {
namespace test {
namespace {
const int kWidth = 352;
const int kHeight = 288;
const int kFrameSize = kWidth * kHeight * 3 / 2; // I420.
} // namespace
class VideoProcessorTest : public ::testing::Test {
protected:
VideoProcessorTest() : q_("VP queue") {
config_.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, false, false,
false, kWidth, kHeight);
decoder_mock_ = new MockVideoDecoder();
decoders_.push_back(std::unique_ptr<VideoDecoder>(decoder_mock_));
ExpectInit();
EXPECT_CALL(frame_reader_mock_, FrameLength())
.WillRepeatedly(Return(kFrameSize));
q_.SendTask(
[this] {
video_processor_ = std::make_unique<VideoProcessor>(
&encoder_mock_, &decoders_, &frame_reader_mock_, config_, &stats_,
&encoded_frame_writers_, /*decoded_frame_writers=*/nullptr);
},
RTC_FROM_HERE);
}
~VideoProcessorTest() {
q_.SendTask([this] { video_processor_.reset(); }, RTC_FROM_HERE);
}
void ExpectInit() {
EXPECT_CALL(encoder_mock_, InitEncode(_, _)).Times(1);
EXPECT_CALL(encoder_mock_, RegisterEncodeCompleteCallback(_)).Times(1);
EXPECT_CALL(*decoder_mock_, InitDecode(_, _)).Times(1);
EXPECT_CALL(*decoder_mock_, RegisterDecodeCompleteCallback(_)).Times(1);
}
void ExpectRelease() {
EXPECT_CALL(encoder_mock_, Release()).Times(1);
EXPECT_CALL(encoder_mock_, RegisterEncodeCompleteCallback(_)).Times(1);
EXPECT_CALL(*decoder_mock_, Release()).Times(1);
EXPECT_CALL(*decoder_mock_, RegisterDecodeCompleteCallback(_)).Times(1);
}
TaskQueueForTest q_;
VideoCodecTestFixture::Config config_;
MockVideoEncoder encoder_mock_;
MockVideoDecoder* decoder_mock_;
std::vector<std::unique_ptr<VideoDecoder>> decoders_;
MockFrameReader frame_reader_mock_;
VideoCodecTestStatsImpl stats_;
VideoProcessor::IvfFileWriterMap encoded_frame_writers_;
std::unique_ptr<VideoProcessor> video_processor_;
};
TEST_F(VideoProcessorTest, InitRelease) {
ExpectRelease();
}
TEST_F(VideoProcessorTest, ProcessFrames_FixedFramerate) {
const int kBitrateKbps = 456;
const int kFramerateFps = 31;
EXPECT_CALL(
encoder_mock_,
SetRates(Field(&VideoEncoder::RateControlParameters::framerate_fps,
static_cast<double>(kFramerateFps))))
.Times(1);
q_.SendTask([=] { video_processor_->SetRates(kBitrateKbps, kFramerateFps); },
RTC_FROM_HERE);
EXPECT_CALL(frame_reader_mock_, ReadFrame())
.WillRepeatedly(Return(I420Buffer::Create(kWidth, kHeight)));
EXPECT_CALL(
encoder_mock_,
Encode(Property(&VideoFrame::timestamp, 1 * 90000 / kFramerateFps), _))
.Times(1);
q_.SendTask([this] { video_processor_->ProcessFrame(); }, RTC_FROM_HERE);
EXPECT_CALL(
encoder_mock_,
Encode(Property(&VideoFrame::timestamp, 2 * 90000 / kFramerateFps), _))
.Times(1);
q_.SendTask([this] { video_processor_->ProcessFrame(); }, RTC_FROM_HERE);
ExpectRelease();
}
TEST_F(VideoProcessorTest, ProcessFrames_VariableFramerate) {
const int kBitrateKbps = 456;
const int kStartFramerateFps = 27;
const int kStartTimestamp = 90000 / kStartFramerateFps;
EXPECT_CALL(
encoder_mock_,
SetRates(Field(&VideoEncoder::RateControlParameters::framerate_fps,
static_cast<double>(kStartFramerateFps))))
.Times(1);
q_.SendTask(
[=] { video_processor_->SetRates(kBitrateKbps, kStartFramerateFps); },
RTC_FROM_HERE);
EXPECT_CALL(frame_reader_mock_, ReadFrame())
.WillRepeatedly(Return(I420Buffer::Create(kWidth, kHeight)));
EXPECT_CALL(encoder_mock_,
Encode(Property(&VideoFrame::timestamp, kStartTimestamp), _))
.Times(1);
q_.SendTask([this] { video_processor_->ProcessFrame(); }, RTC_FROM_HERE);
const int kNewFramerateFps = 13;
EXPECT_CALL(
encoder_mock_,
SetRates(Field(&VideoEncoder::RateControlParameters::framerate_fps,
static_cast<double>(kNewFramerateFps))))
.Times(1);
q_.SendTask(
[=] { video_processor_->SetRates(kBitrateKbps, kNewFramerateFps); },
RTC_FROM_HERE);
EXPECT_CALL(encoder_mock_,
Encode(Property(&VideoFrame::timestamp,
kStartTimestamp + 90000 / kNewFramerateFps),
_))
.Times(1);
q_.SendTask([this] { video_processor_->ProcessFrame(); }, RTC_FROM_HERE);
ExpectRelease();
}
TEST_F(VideoProcessorTest, SetRates) {
const uint32_t kBitrateKbps = 123;
const int kFramerateFps = 17;
EXPECT_CALL(
encoder_mock_,
SetRates(AllOf(ResultOf(
[](const VideoEncoder::RateControlParameters& params) {
return params.bitrate.get_sum_kbps();
},
kBitrateKbps),
Field(&VideoEncoder::RateControlParameters::framerate_fps,
static_cast<double>(kFramerateFps)))))
.Times(1);
q_.SendTask([=] { video_processor_->SetRates(kBitrateKbps, kFramerateFps); },
RTC_FROM_HERE);
const uint32_t kNewBitrateKbps = 456;
const int kNewFramerateFps = 34;
EXPECT_CALL(
encoder_mock_,
SetRates(AllOf(ResultOf(
[](const VideoEncoder::RateControlParameters& params) {
return params.bitrate.get_sum_kbps();
},
kNewBitrateKbps),
Field(&VideoEncoder::RateControlParameters::framerate_fps,
static_cast<double>(kNewFramerateFps)))))
.Times(1);
q_.SendTask(
[=] { video_processor_->SetRates(kNewBitrateKbps, kNewFramerateFps); },
RTC_FROM_HERE);
ExpectRelease();
}
} // namespace test
} // namespace webrtc

853
modules/video_coding/codecs/vp8/default_temporal_layers.cc Normal file
View File

@ -0,0 +1,853 @@
/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/vp8/default_temporal_layers.h"
#include <stdlib.h>
#include <algorithm>
#include <array>
#include <memory>
#include <set>
#include <utility>
#include <vector>
#include "modules/video_coding/include/video_codec_interface.h"
#include "rtc_base/arraysize.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "system_wrappers/include/field_trial.h"
namespace webrtc {
DefaultTemporalLayers::PendingFrame::PendingFrame() = default;
DefaultTemporalLayers::PendingFrame::PendingFrame(
bool expired,
uint8_t updated_buffers_mask,
const DependencyInfo& dependency_info)
: expired(expired),
updated_buffer_mask(updated_buffers_mask),
dependency_info(dependency_info) {}
namespace {
using BufferFlags = Vp8FrameConfig::BufferFlags;
using FreezeEntropy = Vp8FrameConfig::FreezeEntropy;
using Vp8BufferReference = Vp8FrameConfig::Vp8BufferReference;
constexpr BufferFlags kNone = BufferFlags::kNone;
constexpr BufferFlags kReference = BufferFlags::kReference;
constexpr BufferFlags kUpdate = BufferFlags::kUpdate;
constexpr BufferFlags kReferenceAndUpdate = BufferFlags::kReferenceAndUpdate;
constexpr FreezeEntropy kFreezeEntropy = FreezeEntropy::kFreezeEntropy;
static constexpr uint8_t kUninitializedPatternIndex =
std::numeric_limits<uint8_t>::max();
static constexpr std::array<Vp8BufferReference, 3> kAllBuffers = {
{Vp8BufferReference::kLast, Vp8BufferReference::kGolden,
Vp8BufferReference::kAltref}};
std::vector<unsigned int> GetTemporalIds(size_t num_layers) {
switch (num_layers) {
case 1:
// Temporal layer structure (single layer):
// 0 0 0 0 ...
return {0};
case 2:
// Temporal layer structure:
// 1 1 ...
// 0 0 ...
return {0, 1};
case 3:
// Temporal layer structure:
// 2 2 2 2 ...
// 1 1 ...
// 0 0 ...
return {0, 2, 1, 2};
case 4:
// Temporal layer structure:
// 3 3 3 3 3 3 3 3 ...
// 2 2 2 2 ...
// 1 1 ...
// 0 0 ...
return {0, 3, 2, 3, 1, 3, 2, 3};
default:
RTC_NOTREACHED();
break;
}
RTC_NOTREACHED();
return {0};
}
uint8_t GetUpdatedBuffers(const Vp8FrameConfig& config) {
uint8_t flags = 0;
if (config.last_buffer_flags & BufferFlags::kUpdate) {
flags |= static_cast<uint8_t>(Vp8BufferReference::kLast);
}
if (config.golden_buffer_flags & BufferFlags::kUpdate) {
flags |= static_cast<uint8_t>(Vp8BufferReference::kGolden);
}
if (config.arf_buffer_flags & BufferFlags::kUpdate) {
flags |= static_cast<uint8_t>(Vp8BufferReference::kAltref);
}
return flags;
}
} // namespace
std::vector<DefaultTemporalLayers::DependencyInfo>
DefaultTemporalLayers::GetDependencyInfo(size_t num_layers) {
// For indexing in the patterns described below (which temporal layers they
// belong to), see the diagram above.
// Layer sync is done similarly for all patterns (except single stream) and
// happens every 8 frames:
// TL1 layer syncs by periodically by only referencing TL0 ('last'), but still
// updating 'golden', so it can be used as a reference by future TL1 frames.
// TL2 layer syncs just before TL1 by only depending on TL0 (and not depending
// on TL1's buffer before TL1 has layer synced).
// TODO(pbos): Consider cyclically updating 'arf' (and 'golden' for 1TL) for
// the base layer in 1-3TL instead of 'last' periodically on long intervals,
// so that if scene changes occur (user walks between rooms or rotates webcam)
// the 'arf' (or 'golden' respectively) is not stuck on a no-longer relevant
// keyframe.
switch (num_layers) {
case 1:
// Always reference and update the same buffer.
return {{"S", {kReferenceAndUpdate, kNone, kNone}}};
case 2:
// All layers can reference but not update the 'alt' buffer, this means
// that the 'alt' buffer reference is effectively the last keyframe.
// TL0 also references and updates the 'last' buffer.
// TL1 also references 'last' and references and updates 'golden'.
if (!field_trial::IsDisabled("WebRTC-UseShortVP8TL2Pattern")) {
// Shortened 4-frame pattern:
// 1---1 1---1 ...
// / / / /
// 0---0---0---0 ...
return {{"SS", {kReferenceAndUpdate, kNone, kNone}},
{"-S", {kReference, kUpdate, kNone}},
{"SR", {kReferenceAndUpdate, kNone, kNone}},
{"-D", {kReference, kReference, kNone, kFreezeEntropy}}};
} else {
// "Default" 8-frame pattern:
// 1---1---1---1 1---1---1---1 ...
// / / / / / / / /
// 0---0---0---0---0---0---0---0 ...
return {{"SS", {kReferenceAndUpdate, kNone, kNone}},
{"-S", {kReference, kUpdate, kNone}},
{"SR", {kReferenceAndUpdate, kNone, kNone}},
{"-R", {kReference, kReferenceAndUpdate, kNone}},
{"SR", {kReferenceAndUpdate, kNone, kNone}},
{"-R", {kReference, kReferenceAndUpdate, kNone}},
{"SR", {kReferenceAndUpdate, kNone, kNone}},
{"-D", {kReference, kReference, kNone, kFreezeEntropy}}};
}
case 3:
if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) {
// This field trial is intended to check if it is worth using a shorter
// temporal pattern, trading some coding efficiency for less risk of
// dropped frames.
// The coding efficiency will decrease somewhat since the higher layer
// state is more volatile, but it will be offset slightly by updating
// the altref buffer with TL2 frames, instead of just referencing lower
// layers.
// If a frame is dropped in a higher layer, the jitter
// buffer on the receive side won't be able to decode any higher layer
// frame until the next sync frame. So we expect a noticeable decrease
// in frame drops on links with high packet loss.
// TL0 references and updates the 'last' buffer.
// TL1 references 'last' and references and updates 'golden'.
// TL2 references both 'last' & 'golden' and references and updates
// 'arf'.
// 2-------2 2-------2 2
// / __/ / __/ /
// / __1 / __1 /
// /___/ /___/ /
// 0---------------0---------------0-----
// 0 1 2 3 4 5 6 7 8 9 ...
return {{"SSS", {kReferenceAndUpdate, kNone, kNone}},
{"--S", {kReference, kNone, kUpdate}},
{"-DR", {kReference, kUpdate, kNone}},
{"--D", {kReference, kReference, kReference, kFreezeEntropy}}};
} else {
// All layers can reference but not update the 'alt' buffer, this means
// that the 'alt' buffer reference is effectively the last keyframe.
// TL0 also references and updates the 'last' buffer.
// TL1 also references 'last' and references and updates 'golden'.
// TL2 references both 'last' and 'golden' but updates no buffer.
// 2 __2 _____2 __2 2
// / /____/ / / /
// / 1---------/-----1 /
// /_____/ /_____/ /
// 0---------------0---------------0-----
// 0 1 2 3 4 5 6 7 8 9 ...
return {{"SSS", {kReferenceAndUpdate, kNone, kNone}},
{"--D", {kReference, kNone, kNone, kFreezeEntropy}},
{"-SS", {kReference, kUpdate, kNone}},
{"--D", {kReference, kReference, kNone, kFreezeEntropy}},
{"SRR", {kReferenceAndUpdate, kNone, kNone}},
{"--D", {kReference, kReference, kNone, kFreezeEntropy}},
{"-DS", {kReference, kReferenceAndUpdate, kNone}},
{"--D", {kReference, kReference, kNone, kFreezeEntropy}}};
}
case 4:
// TL0 references and updates only the 'last' buffer.
// TL1 references 'last' and updates and references 'golden'.
// TL2 references 'last' and 'golden', and references and updates 'arf'.
// TL3 references all buffers but update none of them.
// TODO(philipel): Set decode target information for this structure.
return {{"----", {kReferenceAndUpdate, kNone, kNone}},
{"----", {kReference, kNone, kNone, kFreezeEntropy}},
{"----", {kReference, kNone, kUpdate}},
{"----", {kReference, kNone, kReference, kFreezeEntropy}},
{"----", {kReference, kUpdate, kNone}},
{"----", {kReference, kReference, kReference, kFreezeEntropy}},
{"----", {kReference, kReference, kReferenceAndUpdate}},
{"----", {kReference, kReference, kReference, kFreezeEntropy}},
{"----", {kReferenceAndUpdate, kNone, kNone}},
{"----", {kReference, kReference, kReference, kFreezeEntropy}},
{"----", {kReference, kReference, kReferenceAndUpdate}},
{"----", {kReference, kReference, kReference, kFreezeEntropy}},
{"----", {kReference, kReferenceAndUpdate, kNone}},
{"----", {kReference, kReference, kReference, kFreezeEntropy}},
{"----", {kReference, kReference, kReferenceAndUpdate}},
{"----", {kReference, kReference, kReference, kFreezeEntropy}}};
default:
RTC_NOTREACHED();
break;
}
RTC_NOTREACHED();
return {{"", {kNone, kNone, kNone}}};
}
DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers)
: num_layers_(std::max(1, number_of_temporal_layers)),
temporal_ids_(GetTemporalIds(num_layers_)),
temporal_pattern_(GetDependencyInfo(num_layers_)),
pattern_idx_(kUninitializedPatternIndex) {
RTC_CHECK_GE(kMaxTemporalStreams, number_of_temporal_layers);
RTC_CHECK_GE(number_of_temporal_layers, 0);
RTC_CHECK_LE(number_of_temporal_layers, 4);
// pattern_idx_ wraps around temporal_pattern_.size, this is incorrect if
// temporal_ids_ are ever longer. If this is no longer correct it needs to
// wrap at max(temporal_ids_.size(), temporal_pattern_.size()).
RTC_DCHECK_LE(temporal_ids_.size(), temporal_pattern_.size());
#if RTC_DCHECK_IS_ON
checker_ = TemporalLayersChecker::CreateTemporalLayersChecker(
Vp8TemporalLayersType::kFixedPattern, number_of_temporal_layers);
#endif
// Always need to start with a keyframe, so pre-populate all frame counters.
for (Vp8BufferReference buffer : kAllBuffers) {
frames_since_buffer_refresh_[buffer] = 0;
}
kf_buffers_ = {kAllBuffers.begin(), kAllBuffers.end()};
for (const DependencyInfo& info : temporal_pattern_) {
uint8_t updated_buffers = GetUpdatedBuffers(info.frame_config);
for (Vp8BufferReference buffer : kAllBuffers) {
if (static_cast<uint8_t>(buffer) & updated_buffers)
kf_buffers_.erase(buffer);
}
}
}
DefaultTemporalLayers::~DefaultTemporalLayers() = default;
void DefaultTemporalLayers::SetQpLimits(size_t stream_index,
int min_qp,
int max_qp) {
RTC_DCHECK_LT(stream_index, StreamCount());
// Ignore.
}
size_t DefaultTemporalLayers::StreamCount() const {
return 1;
}
bool DefaultTemporalLayers::SupportsEncoderFrameDropping(
size_t stream_index) const {
RTC_DCHECK_LT(stream_index, StreamCount());
// This class allows the encoder drop frames as it sees fit.
return true;
}
void DefaultTemporalLayers::OnRatesUpdated(
size_t stream_index,
const std::vector<uint32_t>& bitrates_bps,
int framerate_fps) {
RTC_DCHECK_LT(stream_index, StreamCount());
RTC_DCHECK_GT(bitrates_bps.size(), 0);
RTC_DCHECK_LE(bitrates_bps.size(), num_layers_);
// |bitrates_bps| uses individual rate per layer, but Vp8EncoderConfig wants
// the accumulated rate, so sum them up.
new_bitrates_bps_ = bitrates_bps;
new_bitrates_bps_->resize(num_layers_);
for (size_t i = 1; i < num_layers_; ++i) {
(*new_bitrates_bps_)[i] += (*new_bitrates_bps_)[i - 1];
}
}
Vp8EncoderConfig DefaultTemporalLayers::UpdateConfiguration(
size_t stream_index) {
RTC_DCHECK_LT(stream_index, StreamCount());
Vp8EncoderConfig config;
if (!new_bitrates_bps_) {
return config;
}
config.temporal_layer_config.emplace();
Vp8EncoderConfig::TemporalLayerConfig& ts_config =
config.temporal_layer_config.value();
for (size_t i = 0; i < num_layers_; ++i) {
ts_config.ts_target_bitrate[i] = (*new_bitrates_bps_)[i] / 1000;
// ..., 4, 2, 1
ts_config.ts_rate_decimator[i] = 1 << (num_layers_ - i - 1);
}
ts_config.ts_number_layers = num_layers_;
ts_config.ts_periodicity = temporal_ids_.size();
std::copy(temporal_ids_.begin(), temporal_ids_.end(),
ts_config.ts_layer_id.begin());
new_bitrates_bps_.reset();
return config;
}
bool DefaultTemporalLayers::IsSyncFrame(const Vp8FrameConfig& config) const {
// Since we always assign TL0 to 'last' in these patterns, we can infer layer
// sync by checking if temporal id > 0 and we only reference TL0 or buffers
// containing the last key-frame.
if (config.packetizer_temporal_idx == 0) {
// TL0 frames are per definition not sync frames.
return false;
}
if ((config.last_buffer_flags & BufferFlags::kReference) == 0) {
// Sync frames must reference TL0.
return false;
}
if ((config.golden_buffer_flags & BufferFlags::kReference) &&
kf_buffers_.find(Vp8BufferReference::kGolden) == kf_buffers_.end()) {
// Referencing a golden frame that contains a non-(base layer|key frame).
return false;
}
if ((config.arf_buffer_flags & BufferFlags::kReference) &&
kf_buffers_.find(Vp8BufferReference::kAltref) == kf_buffers_.end()) {
// Referencing an altref frame that contains a non-(base layer|key frame).
return false;
}
return true;
}
Vp8FrameConfig DefaultTemporalLayers::NextFrameConfig(size_t stream_index,
uint32_t timestamp) {
RTC_DCHECK_LT(stream_index, StreamCount());
RTC_DCHECK_GT(num_layers_, 0);
RTC_DCHECK_GT(temporal_pattern_.size(), 0);
RTC_DCHECK_GT(kUninitializedPatternIndex, temporal_pattern_.size());
const bool first_frame = (pattern_idx_ == kUninitializedPatternIndex);
pattern_idx_ = (pattern_idx_ + 1) % temporal_pattern_.size();
DependencyInfo dependency_info = temporal_pattern_[pattern_idx_];
Vp8FrameConfig& tl_config = dependency_info.frame_config;
tl_config.encoder_layer_id = tl_config.packetizer_temporal_idx =
temporal_ids_[pattern_idx_ % temporal_ids_.size()];
if (pattern_idx_ == 0) {
// Start of new pattern iteration, set up clear state by invalidating any
// pending frames, so that we don't make an invalid reference to a buffer
// containing data from a previous iteration.
for (auto& it : pending_frames_) {
it.second.expired = true;
}
}
if (first_frame) {
tl_config = Vp8FrameConfig::GetIntraFrameConfig();
} else {
// Last is always ok to reference as it contains the base layer. For other
// buffers though, we need to check if the buffer has actually been
// refreshed this cycle of the temporal pattern. If the encoder dropped
// a frame, it might not have.
ValidateReferences(&tl_config.golden_buffer_flags,
Vp8BufferReference::kGolden);
ValidateReferences(&tl_config.arf_buffer_flags,
Vp8BufferReference::kAltref);
// Update search order to let the encoder know which buffers contains the
// most recent data.
UpdateSearchOrder(&tl_config);
// Figure out if this a sync frame (non-base-layer frame with only
// base-layer references).
tl_config.layer_sync = IsSyncFrame(tl_config);
// Increment frame age, this needs to be in sync with |pattern_idx_|,
// so must update it here. Resetting age to 0 must be done when encoding is
// complete though, and so in the case of pipelining encoder it might lag.
// To prevent this data spill over into the next iteration,
// the |pedning_frames_| map is reset in loops. If delay is constant,
// the relative age should still be OK for the search order.
for (Vp8BufferReference buffer : kAllBuffers) {
++frames_since_buffer_refresh_[buffer];
}
}
// Add frame to set of pending frames, awaiting completion.
pending_frames_[timestamp] =
PendingFrame{false, GetUpdatedBuffers(tl_config), dependency_info};
#if RTC_DCHECK_IS_ON
// Checker does not yet support encoder frame dropping, so validate flags
// here before they can be dropped.
// TODO(sprang): Update checker to support dropping.
RTC_DCHECK(checker_->CheckTemporalConfig(first_frame, tl_config));
#endif
return tl_config;
}
void DefaultTemporalLayers::ValidateReferences(BufferFlags* flags,
Vp8BufferReference ref) const {
// Check if the buffer specified by |ref| is actually referenced, and if so
// if it also a dynamically updating one (buffers always just containing
// keyframes are always safe to reference).
if ((*flags & BufferFlags::kReference) &&
kf_buffers_.find(ref) == kf_buffers_.end()) {
auto it = frames_since_buffer_refresh_.find(ref);
if (it == frames_since_buffer_refresh_.end() ||
it->second >= pattern_idx_) {
// No valid buffer state, or buffer contains frame that is older than the
// current pattern. This reference is not valid, so remove it.
*flags = static_cast<BufferFlags>(*flags & ~BufferFlags::kReference);
}
}
}
void DefaultTemporalLayers::UpdateSearchOrder(Vp8FrameConfig* config) {
// Figure out which of the buffers we can reference, and order them so that
// the most recently refreshed is first. Otherwise prioritize last first,
// golden second, and altref third.
using BufferRefAge = std::pair<Vp8BufferReference, size_t>;
std::vector<BufferRefAge> eligible_buffers;
if (config->last_buffer_flags & BufferFlags::kReference) {
eligible_buffers.emplace_back(
Vp8BufferReference::kLast,
frames_since_buffer_refresh_[Vp8BufferReference::kLast]);
}
if (config->golden_buffer_flags & BufferFlags::kReference) {
eligible_buffers.emplace_back(
Vp8BufferReference::kGolden,
frames_since_buffer_refresh_[Vp8BufferReference::kGolden]);
}
if (config->arf_buffer_flags & BufferFlags::kReference) {
eligible_buffers.emplace_back(
Vp8BufferReference::kAltref,
frames_since_buffer_refresh_[Vp8BufferReference::kAltref]);
}
std::sort(eligible_buffers.begin(), eligible_buffers.end(),
[](const BufferRefAge& lhs, const BufferRefAge& rhs) {
if (lhs.second != rhs.second) {
// Lower count has highest precedence.
return lhs.second < rhs.second;
}
return lhs.first < rhs.first;
});
// Populate the search order fields where possible.
if (!eligible_buffers.empty()) {
config->first_reference = eligible_buffers.front().first;
if (eligible_buffers.size() > 1)
config->second_reference = eligible_buffers[1].first;
}
}
void DefaultTemporalLayers::OnEncodeDone(size_t stream_index,
uint32_t rtp_timestamp,
size_t size_bytes,
bool is_keyframe,
int qp,
CodecSpecificInfo* info) {
RTC_DCHECK_LT(stream_index, StreamCount());
RTC_DCHECK_GT(num_layers_, 0);
if (size_bytes == 0) {
RTC_LOG(LS_WARNING) << "Empty frame; treating as dropped.";
OnFrameDropped(stream_index, rtp_timestamp);
return;
}
auto pending_frame = pending_frames_.find(rtp_timestamp);
RTC_DCHECK(pending_frame != pending_frames_.end());
PendingFrame& frame = pending_frame->second;
const Vp8FrameConfig& frame_config = frame.dependency_info.frame_config;
#if RTC_DCHECK_IS_ON
if (is_keyframe) {
// Signal key-frame so checker resets state.
RTC_DCHECK(checker_->CheckTemporalConfig(true, frame_config));
}
#endif
CodecSpecificInfoVP8& vp8_info = info->codecSpecific.VP8;
if (num_layers_ == 1) {
vp8_info.temporalIdx = kNoTemporalIdx;
vp8_info.layerSync = false;
} else {
if (is_keyframe) {
// Restart the temporal pattern on keyframes.
pattern_idx_ = 0;
vp8_info.temporalIdx = 0;
vp8_info.layerSync = true; // Keyframes are always sync frames.
for (Vp8BufferReference buffer : kAllBuffers) {
if (kf_buffers_.find(buffer) != kf_buffers_.end()) {
// Update frame count of all kf-only buffers, regardless of state of
// |pending_frames_|.
frames_since_buffer_refresh_[buffer] = 0;
} else {
// Key-frames update all buffers, this should be reflected when
// updating state in FrameEncoded().
frame.updated_buffer_mask |= static_cast<uint8_t>(buffer);
}
}
} else {
// Delta frame, update codec specifics with temporal id and sync flag.
vp8_info.temporalIdx = frame_config.packetizer_temporal_idx;
vp8_info.layerSync = frame_config.layer_sync;
}
}
vp8_info.useExplicitDependencies = true;
RTC_DCHECK_EQ(vp8_info.referencedBuffersCount, 0u);
RTC_DCHECK_EQ(vp8_info.updatedBuffersCount, 0u);
GenericFrameInfo& generic_frame_info = info->generic_frame_info.emplace();
for (int i = 0; i < static_cast<int>(Vp8FrameConfig::Buffer::kCount); ++i) {
bool references = false;
bool updates = is_keyframe;
if (!is_keyframe &&
frame_config.References(static_cast<Vp8FrameConfig::Buffer>(i))) {
RTC_DCHECK_LT(vp8_info.referencedBuffersCount,
arraysize(CodecSpecificInfoVP8::referencedBuffers));
references = true;
vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i;
}
if (is_keyframe ||
frame_config.Updates(static_cast<Vp8FrameConfig::Buffer>(i))) {
RTC_DCHECK_LT(vp8_info.updatedBuffersCount,
arraysize(CodecSpecificInfoVP8::updatedBuffers));
updates = true;
vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i;
}
if (references || updates)
generic_frame_info.encoder_buffers.emplace_back(i, references, updates);
}
// The templates are always present on keyframes, and then refered to by
// subsequent frames.
if (is_keyframe) {
info->template_structure = GetTemplateStructure(num_layers_);
generic_frame_info.decode_target_indications =
temporal_pattern_.front().decode_target_indications;
generic_frame_info.temporal_id = 0;
} else {
generic_frame_info.decode_target_indications =
frame.dependency_info.decode_target_indications;
generic_frame_info.temporal_id = frame_config.packetizer_temporal_idx;
}
if (!frame.expired) {
for (Vp8BufferReference buffer : kAllBuffers) {
if (frame.updated_buffer_mask & static_cast<uint8_t>(buffer)) {
frames_since_buffer_refresh_[buffer] = 0;
}
}
}
pending_frames_.erase(pending_frame);
}
void DefaultTemporalLayers::OnFrameDropped(size_t stream_index,
uint32_t rtp_timestamp) {
auto pending_frame = pending_frames_.find(rtp_timestamp);
RTC_DCHECK(pending_frame != pending_frames_.end());
pending_frames_.erase(pending_frame);
}
void DefaultTemporalLayers::OnPacketLossRateUpdate(float packet_loss_rate) {}
void DefaultTemporalLayers::OnRttUpdate(int64_t rtt_ms) {}
void DefaultTemporalLayers::OnLossNotification(
const VideoEncoder::LossNotification& loss_notification) {}
FrameDependencyStructure DefaultTemporalLayers::GetTemplateStructure(
int num_layers) const {
RTC_CHECK_LT(num_layers, 5);
RTC_CHECK_GT(num_layers, 0);
FrameDependencyStructure template_structure;
template_structure.num_decode_targets = num_layers;
using Builder = GenericFrameInfo::Builder;
switch (num_layers) {
case 1: {
template_structure.templates = {
Builder().T(0).Dtis("S").Build(),
Builder().T(0).Dtis("S").Fdiffs({1}).Build(),
};
return template_structure;
}
case 2: {
template_structure.templates = {
Builder().T(0).Dtis("SS").Build(),
Builder().T(0).Dtis("SS").Fdiffs({2}).Build(),
Builder().T(0).Dtis("SR").Fdiffs({2}).Build(),
Builder().T(1).Dtis("-S").Fdiffs({1}).Build(),
Builder().T(1).Dtis("-D").Fdiffs({1, 2}).Build(),
};
return template_structure;
}
case 3: {
if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) {
template_structure.templates = {
Builder().T(0).Dtis("SSS").Build(),
Builder().T(0).Dtis("SSS").Fdiffs({4}).Build(),
Builder().T(1).Dtis("-DR").Fdiffs({2}).Build(),
Builder().T(2).Dtis("--S").Fdiffs({1}).Build(),
Builder().T(2).Dtis("--D").Fdiffs({1, 2}).Build(),
};
} else {
template_structure.templates = {
Builder().T(0).Dtis("SSS").Build(),
Builder().T(0).Dtis("SSS").Fdiffs({4}).Build(),
Builder().T(0).Dtis("SRR").Fdiffs({4}).Build(),
Builder().T(1).Dtis("-SS").Fdiffs({2}).Build(),
Builder().T(1).Dtis("-DS").Fdiffs({2, 4}).Build(),
Builder().T(2).Dtis("--D").Fdiffs({1}).Build(),
Builder().T(2).Dtis("--D").Fdiffs({1, 3}).Build(),
};
}
return template_structure;
}
case 4: {
template_structure.templates = {
Builder().T(0).Dtis("SSSS").Build(),
Builder().T(0).Dtis("SSSS").Fdiffs({8}).Build(),
Builder().T(1).Dtis("-SRR").Fdiffs({4}).Build(),
Builder().T(1).Dtis("-SRR").Fdiffs({4, 8}).Build(),
Builder().T(2).Dtis("--SR").Fdiffs({2}).Build(),
Builder().T(2).Dtis("--SR").Fdiffs({2, 4}).Build(),
Builder().T(3).Dtis("---D").Fdiffs({1}).Build(),
Builder().T(3).Dtis("---D").Fdiffs({1, 3}).Build(),
};
return template_structure;
}
default:
RTC_NOTREACHED();
// To make the compiler happy!
return template_structure;
}
}
// Returns list of temporal dependencies for each frame in the temporal pattern.
// Values are lists of indecies in the pattern.
std::vector<std::set<uint8_t>> GetTemporalDependencies(
int num_temporal_layers) {
switch (num_temporal_layers) {
case 1:
return {{0}};
case 2:
if (!field_trial::IsDisabled("WebRTC-UseShortVP8TL2Pattern")) {
return {{2}, {0}, {0}, {1, 2}};
} else {
return {{6}, {0}, {0}, {1, 2}, {2}, {3, 4}, {4}, {5, 6}};
}
case 3:
if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) {
return {{0}, {0}, {0}, {0, 1, 2}};
} else {
return {{4}, {0}, {0}, {0, 2}, {0}, {2, 4}, {2, 4}, {4, 6}};
}
case 4:
return {{8}, {0}, {0}, {0, 2},
{0}, {0, 2, 4}, {0, 2, 4}, {0, 4, 6},
{0}, {4, 6, 8}, {4, 6, 8}, {4, 8, 10},
{4, 8}, {8, 10, 12}, {8, 10, 12}, {8, 12, 14}};
default:
RTC_NOTREACHED();
return {};
}
}
DefaultTemporalLayersChecker::DefaultTemporalLayersChecker(
int num_temporal_layers)
: TemporalLayersChecker(num_temporal_layers),
num_layers_(std::max(1, num_temporal_layers)),
temporal_ids_(GetTemporalIds(num_layers_)),
temporal_dependencies_(GetTemporalDependencies(num_layers_)),
pattern_idx_(255) {
int i = 0;
while (temporal_ids_.size() < temporal_dependencies_.size()) {
temporal_ids_.push_back(temporal_ids_[i++]);
}
}
DefaultTemporalLayersChecker::~DefaultTemporalLayersChecker() = default;
bool DefaultTemporalLayersChecker::CheckTemporalConfig(
bool frame_is_keyframe,
const Vp8FrameConfig& frame_config) {
if (!TemporalLayersChecker::CheckTemporalConfig(frame_is_keyframe,
frame_config)) {
return false;
}
if (frame_config.drop_frame) {
return true;
}
if (frame_is_keyframe) {
pattern_idx_ = 0;
last_ = BufferState();
golden_ = BufferState();
arf_ = BufferState();
return true;
}
++pattern_idx_;
if (pattern_idx_ == temporal_ids_.size()) {
// All non key-frame buffers should be updated each pattern cycle.
if (!last_.is_keyframe && !last_.is_updated_this_cycle) {
RTC_LOG(LS_ERROR) << "Last buffer was not updated during pattern cycle.";
return false;
}
if (!arf_.is_keyframe && !arf_.is_updated_this_cycle) {
RTC_LOG(LS_ERROR) << "Arf buffer was not updated during pattern cycle.";
return false;
}
if (!golden_.is_keyframe && !golden_.is_updated_this_cycle) {
RTC_LOG(LS_ERROR)
<< "Golden buffer was not updated during pattern cycle.";
return false;
}
last_.is_updated_this_cycle = false;
arf_.is_updated_this_cycle = false;
golden_.is_updated_this_cycle = false;
pattern_idx_ = 0;
}
uint8_t expected_tl_idx = temporal_ids_[pattern_idx_];
if (frame_config.packetizer_temporal_idx != expected_tl_idx) {
RTC_LOG(LS_ERROR) << "Frame has an incorrect temporal index. Expected: "
<< static_cast<int>(expected_tl_idx) << " Actual: "
<< static_cast<int>(frame_config.packetizer_temporal_idx);
return false;
}
bool need_sync = temporal_ids_[pattern_idx_] > 0 &&
temporal_ids_[pattern_idx_] != kNoTemporalIdx;
std::vector<int> dependencies;
if (frame_config.last_buffer_flags & BufferFlags::kReference) {
uint8_t referenced_layer = temporal_ids_[last_.pattern_idx];
if (referenced_layer > 0) {
need_sync = false;
}
if (!last_.is_keyframe) {
dependencies.push_back(last_.pattern_idx);
}
} else if (frame_config.first_reference == Vp8BufferReference::kLast ||
frame_config.second_reference == Vp8BufferReference::kLast) {
RTC_LOG(LS_ERROR)
<< "Last buffer not referenced, but present in search order.";
return false;
}
if (frame_config.arf_buffer_flags & BufferFlags::kReference) {
uint8_t referenced_layer = temporal_ids_[arf_.pattern_idx];
if (referenced_layer > 0) {
need_sync = false;
}
if (!arf_.is_keyframe) {
dependencies.push_back(arf_.pattern_idx);
}
} else if (frame_config.first_reference == Vp8BufferReference::kAltref ||
frame_config.second_reference == Vp8BufferReference::kAltref) {
RTC_LOG(LS_ERROR)
<< "Altret buffer not referenced, but present in search order.";
return false;
}
if (frame_config.golden_buffer_flags & BufferFlags::kReference) {
uint8_t referenced_layer = temporal_ids_[golden_.pattern_idx];
if (referenced_layer > 0) {
need_sync = false;
}
if (!golden_.is_keyframe) {
dependencies.push_back(golden_.pattern_idx);
}
} else if (frame_config.first_reference == Vp8BufferReference::kGolden ||
frame_config.second_reference == Vp8BufferReference::kGolden) {
RTC_LOG(LS_ERROR)
<< "Golden buffer not referenced, but present in search order.";
return false;
}
if (need_sync != frame_config.layer_sync) {
RTC_LOG(LS_ERROR) << "Sync bit is set incorrectly on a frame. Expected: "
<< need_sync << " Actual: " << frame_config.layer_sync;
return false;
}
if (!frame_is_keyframe) {
size_t i;
for (i = 0; i < dependencies.size(); ++i) {
if (temporal_dependencies_[pattern_idx_].find(dependencies[i]) ==
temporal_dependencies_[pattern_idx_].end()) {
RTC_LOG(LS_ERROR)
<< "Illegal temporal dependency out of defined pattern "
"from position "
<< static_cast<int>(pattern_idx_) << " to position "
<< static_cast<int>(dependencies[i]);
return false;
}
}
}
if (frame_config.last_buffer_flags & BufferFlags::kUpdate) {
last_.is_updated_this_cycle = true;
last_.pattern_idx = pattern_idx_;
last_.is_keyframe = false;
}
if (frame_config.arf_buffer_flags & BufferFlags::kUpdate) {
arf_.is_updated_this_cycle = true;
arf_.pattern_idx = pattern_idx_;
arf_.is_keyframe = false;
}
if (frame_config.golden_buffer_flags & BufferFlags::kUpdate) {
golden_.is_updated_this_cycle = true;
golden_.pattern_idx = pattern_idx_;
golden_.is_keyframe = false;
}
return true;
}
} // namespace webrtc

156
modules/video_coding/codecs/vp8/default_temporal_layers.h Normal file
View File

@ -0,0 +1,156 @@
/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
/*
* This file defines classes for doing temporal layers with VP8.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_
#define MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_
#include <stddef.h>
#include <stdint.h>
#include <limits>
#include <map>
#include <memory>
#include <set>
#include <utility>
#include <vector>
#include "absl/types/optional.h"
#include "api/video_codecs/vp8_frame_config.h"
#include "api/video_codecs/vp8_temporal_layers.h"
#include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h"
#include "modules/video_coding/include/video_codec_interface.h"
namespace webrtc {
class DefaultTemporalLayers final : public Vp8FrameBufferController {
public:
explicit DefaultTemporalLayers(int number_of_temporal_layers);
~DefaultTemporalLayers() override;
void SetQpLimits(size_t stream_index, int min_qp, int max_qp) override;
size_t StreamCount() const override;
bool SupportsEncoderFrameDropping(size_t stream_index) const override;
// Returns the recommended VP8 encode flags needed. May refresh the decoder
// and/or update the reference buffers.
Vp8FrameConfig NextFrameConfig(size_t stream_index,
uint32_t timestamp) override;
// New target bitrate, per temporal layer.
void OnRatesUpdated(size_t stream_index,
const std::vector<uint32_t>& bitrates_bps,
int framerate_fps) override;
Vp8EncoderConfig UpdateConfiguration(size_t stream_index) override;
void OnEncodeDone(size_t stream_index,
uint32_t rtp_timestamp,
size_t size_bytes,
bool is_keyframe,
int qp,
CodecSpecificInfo* info) override;
void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) override;
void OnPacketLossRateUpdate(float packet_loss_rate) override;
void OnRttUpdate(int64_t rtt_ms) override;
void OnLossNotification(
const VideoEncoder::LossNotification& loss_notification) override;
private:
struct DependencyInfo {
DependencyInfo() = default;
DependencyInfo(absl::string_view indication_symbols,
Vp8FrameConfig frame_config)
: decode_target_indications(
GenericFrameInfo::DecodeTargetInfo(indication_symbols)),
frame_config(frame_config) {}
absl::InlinedVector<DecodeTargetIndication, 10> decode_target_indications;
Vp8FrameConfig frame_config;
};
static std::vector<DependencyInfo> GetDependencyInfo(size_t num_layers);
bool IsSyncFrame(const Vp8FrameConfig& config) const;
void ValidateReferences(Vp8FrameConfig::BufferFlags* flags,
Vp8FrameConfig::Vp8BufferReference ref) const;
void UpdateSearchOrder(Vp8FrameConfig* config);
const size_t num_layers_;
const std::vector<unsigned int> temporal_ids_;
const std::vector<DependencyInfo> temporal_pattern_;
// Set of buffers that are never updated except by keyframes.
std::set<Vp8FrameConfig::Vp8BufferReference> kf_buffers_;
FrameDependencyStructure GetTemplateStructure(int num_layers) const;
uint8_t pattern_idx_;
// Updated cumulative bitrates, per temporal layer.
absl::optional<std::vector<uint32_t>> new_bitrates_bps_;
struct PendingFrame {
PendingFrame();
PendingFrame(bool expired,
uint8_t updated_buffers_mask,
const DependencyInfo& dependency_info);
// Flag indicating if this frame has expired, ie it belongs to a previous
// iteration of the temporal pattern.
bool expired = false;
// Bitmask of Vp8BufferReference flags, indicating which buffers this frame
// updates.
uint8_t updated_buffer_mask = 0;
// The frame config returned by NextFrameConfig() for this frame.
DependencyInfo dependency_info;
};
// Map from rtp timestamp to pending frame status. Reset on pattern loop.
std::map<uint32_t, PendingFrame> pending_frames_;
// One counter per Vp8BufferReference, indicating number of frames since last
// refresh. For non-base-layer frames (ie golden, altref buffers), this is
// reset when the pattern loops.
std::map<Vp8FrameConfig::Vp8BufferReference, size_t>
frames_since_buffer_refresh_;
// Optional utility used to verify reference validity.
std::unique_ptr<TemporalLayersChecker> checker_;
};
class DefaultTemporalLayersChecker : public TemporalLayersChecker {
public:
explicit DefaultTemporalLayersChecker(int number_of_temporal_layers);
~DefaultTemporalLayersChecker() override;
bool CheckTemporalConfig(bool frame_is_keyframe,
const Vp8FrameConfig& frame_config) override;
private:
struct BufferState {
BufferState()
: is_updated_this_cycle(false), is_keyframe(true), pattern_idx(0) {}
bool is_updated_this_cycle;
bool is_keyframe;
uint8_t pattern_idx;
};
const size_t num_layers_;
std::vector<unsigned int> temporal_ids_;
const std::vector<std::set<uint8_t>> temporal_dependencies_;
BufferState last_;
BufferState arf_;
BufferState golden_;
uint8_t pattern_idx_;
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_

817
modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc Normal file
View File

@ -0,0 +1,817 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/vp8/default_temporal_layers.h"
#include <cstdint>
#include <memory>
#include "api/video/video_bitrate_allocation.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/vp8_frame_config.h"
#include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/utility/simulcast_rate_allocator.h"
#include "test/field_trial.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "vpx/vp8cx.h"
// TODO(bugs.webrtc.org/10582): Test the behavior of UpdateConfiguration().
namespace webrtc {
namespace test {
namespace {
using ::testing::Each;
enum {
kTemporalUpdateLast = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF,
kTemporalUpdateGoldenWithoutDependency =
VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF |
VP8_EFLAG_NO_UPD_LAST,
kTemporalUpdateGolden =
VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
kTemporalUpdateAltrefWithoutDependency =
VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF |
VP8_EFLAG_NO_UPD_LAST,
kTemporalUpdateAltref = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_LAST,
kTemporalUpdateNone = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
kTemporalUpdateNoneNoRefAltRef =
VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
kTemporalUpdateNoneNoRefGolden =
VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
kTemporalUpdateNoneNoRefGoldenAltRef =
VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_REF_ARF |
VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
kTemporalUpdateGoldenWithoutDependencyRefAltRef =
VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
kTemporalUpdateGoldenRefAltRef = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
kTemporalUpdateLastRefAltRef =
VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF,
kTemporalUpdateLastAndGoldenRefAltRef =
VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF,
};
using BufferFlags = Vp8FrameConfig::BufferFlags;
using Vp8BufferReference = Vp8FrameConfig::Vp8BufferReference;
constexpr uint8_t kNone = static_cast<uint8_t>(Vp8BufferReference::kNone);
constexpr uint8_t kLast = static_cast<uint8_t>(Vp8BufferReference::kLast);
constexpr uint8_t kGolden = static_cast<uint8_t>(Vp8BufferReference::kGolden);
constexpr uint8_t kAltref = static_cast<uint8_t>(Vp8BufferReference::kAltref);
constexpr uint8_t kAll = kLast | kGolden | kAltref;
constexpr int ToVp8CodecFlags(uint8_t referenced_buffers,
uint8_t updated_buffers,
bool update_entropy) {
return (((referenced_buffers & kLast) == 0) ? VP8_EFLAG_NO_REF_LAST : 0) |
(((referenced_buffers & kGolden) == 0) ? VP8_EFLAG_NO_REF_GF : 0) |
(((referenced_buffers & kAltref) == 0) ? VP8_EFLAG_NO_REF_ARF : 0) |
(((updated_buffers & kLast) == 0) ? VP8_EFLAG_NO_UPD_LAST : 0) |
(((updated_buffers & kGolden) == 0) ? VP8_EFLAG_NO_UPD_GF : 0) |
(((updated_buffers & kAltref) == 0) ? VP8_EFLAG_NO_UPD_ARF : 0) |
(update_entropy ? 0 : VP8_EFLAG_NO_UPD_ENTROPY);
}
constexpr int kKeyFrameFlags = ToVp8CodecFlags(kNone, kAll, true);
std::vector<uint32_t> GetTemporalLayerRates(int target_bitrate_kbps,
int framerate_fps,
int num_temporal_layers) {
VideoCodec codec;
codec.codecType = VideoCodecType::kVideoCodecVP8;
codec.numberOfSimulcastStreams = 1;
codec.maxBitrate = target_bitrate_kbps;
codec.maxFramerate = framerate_fps;
codec.simulcastStream[0].targetBitrate = target_bitrate_kbps;
codec.simulcastStream[0].maxBitrate = target_bitrate_kbps;
codec.simulcastStream[0].numberOfTemporalLayers = num_temporal_layers;
codec.simulcastStream[0].active = true;
SimulcastRateAllocator allocator(codec);
return allocator
.Allocate(
VideoBitrateAllocationParameters(target_bitrate_kbps, framerate_fps))
.GetTemporalLayerAllocation(0);
}
constexpr int kDefaultBitrateBps = 500;
constexpr int kDefaultFramerate = 30;
constexpr int kDefaultBytesPerFrame =
(kDefaultBitrateBps / 8) / kDefaultFramerate;
constexpr int kDefaultQp = 2;
} // namespace
class TemporalLayersTest : public ::testing::Test {
public:
~TemporalLayersTest() override = default;
CodecSpecificInfo* IgnoredCodecSpecificInfo() {
codec_specific_info_ = std::make_unique<CodecSpecificInfo>();
return codec_specific_info_.get();
}
private:
std::unique_ptr<CodecSpecificInfo> codec_specific_info_;
};
TEST_F(TemporalLayersTest, 2Layers) {
constexpr int kNumLayers = 2;
DefaultTemporalLayers tl(kNumLayers);
DefaultTemporalLayersChecker checker(kNumLayers);
tl.OnRatesUpdated(0,
GetTemporalLayerRates(kDefaultBytesPerFrame,
kDefaultFramerate, kNumLayers),
kDefaultFramerate);
tl.UpdateConfiguration(0);
constexpr size_t kPatternSize = 4;
constexpr size_t kRepetitions = 4;
const int expected_flags[kPatternSize] = {
ToVp8CodecFlags(kLast, kLast, true),
ToVp8CodecFlags(kLast, kGolden, true),
ToVp8CodecFlags(kLast, kLast, true),
ToVp8CodecFlags(kLast | kGolden, kNone, false),
};
const int expected_temporal_idx[kPatternSize] = {0, 1, 0, 1};
const bool expected_layer_sync[kPatternSize] = {false, true, false, false};
uint32_t timestamp = 0;
for (size_t i = 0; i < kPatternSize * kRepetitions; ++i) {
const size_t ind = i % kPatternSize;
const bool is_keyframe = (i == 0);
CodecSpecificInfo info;
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[ind],
LibvpxVp8Encoder::EncodeFlags(tl_config))
<< i;
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe,
kDefaultQp, &info);
EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config));
EXPECT_EQ(expected_temporal_idx[ind], info.codecSpecific.VP8.temporalIdx);
EXPECT_EQ(expected_temporal_idx[ind], tl_config.packetizer_temporal_idx);
EXPECT_EQ(expected_temporal_idx[ind], tl_config.encoder_layer_id);
EXPECT_EQ(is_keyframe || expected_layer_sync[ind],
info.codecSpecific.VP8.layerSync);
EXPECT_EQ(expected_layer_sync[ind], tl_config.layer_sync);
timestamp += 3000;
}
}
TEST_F(TemporalLayersTest, 3Layers) {
constexpr int kNumLayers = 3;
DefaultTemporalLayers tl(kNumLayers);
DefaultTemporalLayersChecker checker(kNumLayers);
tl.OnRatesUpdated(0,
GetTemporalLayerRates(kDefaultBytesPerFrame,
kDefaultFramerate, kNumLayers),
kDefaultFramerate);
tl.UpdateConfiguration(0);
int expected_flags[16] = {
kTemporalUpdateLast,
kTemporalUpdateNoneNoRefGoldenAltRef,
kTemporalUpdateGoldenWithoutDependency,
kTemporalUpdateNoneNoRefAltRef,
kTemporalUpdateLast,
kTemporalUpdateNoneNoRefAltRef,
kTemporalUpdateGolden,
kTemporalUpdateNoneNoRefAltRef,
kTemporalUpdateLast,
kTemporalUpdateNoneNoRefGoldenAltRef,
kTemporalUpdateGoldenWithoutDependency,
kTemporalUpdateNoneNoRefAltRef,
kTemporalUpdateLast,
kTemporalUpdateNoneNoRefAltRef,
kTemporalUpdateGolden,
kTemporalUpdateNoneNoRefAltRef,
};
int expected_temporal_idx[16] = {0, 2, 1, 2, 0, 2, 1, 2,
0, 2, 1, 2, 0, 2, 1, 2};
bool expected_layer_sync[16] = {false, true, true, false, false, false,
false, false, false, true, true, false,
false, false, false, false};
unsigned int timestamp = 0;
for (int i = 0; i < 16; ++i) {
const bool is_keyframe = (i == 0);
CodecSpecificInfo info;
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[i],
LibvpxVp8Encoder::EncodeFlags(tl_config))
<< i;
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe,
kDefaultQp, &info);
EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config));
EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx);
EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
EXPECT_EQ(is_keyframe || expected_layer_sync[i],
info.codecSpecific.VP8.layerSync);
EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
timestamp += 3000;
}
}
TEST_F(TemporalLayersTest, Alternative3Layers) {
constexpr int kNumLayers = 3;
ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
DefaultTemporalLayers tl(kNumLayers);
DefaultTemporalLayersChecker checker(kNumLayers);
tl.OnRatesUpdated(0,
GetTemporalLayerRates(kDefaultBytesPerFrame,
kDefaultFramerate, kNumLayers),
kDefaultFramerate);
tl.UpdateConfiguration(0);
int expected_flags[8] = {kTemporalUpdateLast,
kTemporalUpdateAltrefWithoutDependency,
kTemporalUpdateGoldenWithoutDependency,
kTemporalUpdateNone,
kTemporalUpdateLast,
kTemporalUpdateAltrefWithoutDependency,
kTemporalUpdateGoldenWithoutDependency,
kTemporalUpdateNone};
int expected_temporal_idx[8] = {0, 2, 1, 2, 0, 2, 1, 2};
bool expected_layer_sync[8] = {false, true, true, false,
false, true, true, false};
unsigned int timestamp = 0;
for (int i = 0; i < 8; ++i) {
const bool is_keyframe = (i == 0);
CodecSpecificInfo info;
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[i],
LibvpxVp8Encoder::EncodeFlags(tl_config))
<< i;
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe,
kDefaultQp, &info);
EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config));
EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx);
EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
EXPECT_EQ(is_keyframe || expected_layer_sync[i],
info.codecSpecific.VP8.layerSync);
EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
timestamp += 3000;
}
}
TEST_F(TemporalLayersTest, SearchOrder) {
constexpr int kNumLayers = 3;
ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
DefaultTemporalLayers tl(kNumLayers);
DefaultTemporalLayersChecker checker(kNumLayers);
tl.OnRatesUpdated(0,
GetTemporalLayerRates(kDefaultBytesPerFrame,
kDefaultFramerate, kNumLayers),
kDefaultFramerate);
tl.UpdateConfiguration(0);
// Use a repeating pattern of tl 0, 2, 1, 2.
// Tl 0, 1, 2 update last, golden, altref respectively.
// Start with a key-frame. tl_config flags can be ignored.
uint32_t timestamp = 0;
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
IgnoredCodecSpecificInfo());
// TL2 frame. First one only references TL0. Updates altref.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
// TL1 frame. Can only reference TL0. Updated golden.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
// TL2 frame. Can reference all three buffers. Golden was the last to be
// updated, the next to last was altref.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kGolden);
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kAltref);
}
TEST_F(TemporalLayersTest, SearchOrderWithDrop) {
constexpr int kNumLayers = 3;
ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
DefaultTemporalLayers tl(kNumLayers);
DefaultTemporalLayersChecker checker(kNumLayers);
tl.OnRatesUpdated(0,
GetTemporalLayerRates(kDefaultBytesPerFrame,
kDefaultFramerate, kNumLayers),
kDefaultFramerate);
tl.UpdateConfiguration(0);
// Use a repeating pattern of tl 0, 2, 1, 2.
// Tl 0, 1, 2 update last, golden, altref respectively.
// Start with a key-frame. tl_config flags can be ignored.
uint32_t timestamp = 0;
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
IgnoredCodecSpecificInfo());
// TL2 frame. First one only references TL0. Updates altref.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
// Dropped TL1 frame. Can only reference TL0. Should have updated golden.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
// TL2 frame. Can normally reference all three buffers, but golden has not
// been populated this cycle. Altref was last to be updated, before that last.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kAltref);
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kLast);
}
TEST_F(TemporalLayersTest, 4Layers) {
constexpr int kNumLayers = 4;
DefaultTemporalLayers tl(kNumLayers);
DefaultTemporalLayersChecker checker(kNumLayers);
tl.OnRatesUpdated(0,
GetTemporalLayerRates(kDefaultBytesPerFrame,
kDefaultFramerate, kNumLayers),
kDefaultFramerate);
tl.UpdateConfiguration(0);
int expected_flags[16] = {
kTemporalUpdateLast,
kTemporalUpdateNoneNoRefGoldenAltRef,
kTemporalUpdateAltrefWithoutDependency,
kTemporalUpdateNoneNoRefGolden,
kTemporalUpdateGoldenWithoutDependency,
kTemporalUpdateNone,
kTemporalUpdateAltref,
kTemporalUpdateNone,
kTemporalUpdateLast,
kTemporalUpdateNone,
kTemporalUpdateAltref,
kTemporalUpdateNone,
kTemporalUpdateGolden,
kTemporalUpdateNone,
kTemporalUpdateAltref,
kTemporalUpdateNone,
};
int expected_temporal_idx[16] = {0, 3, 2, 3, 1, 3, 2, 3,
0, 3, 2, 3, 1, 3, 2, 3};
bool expected_layer_sync[16] = {false, true, true, false, true, false,
false, false, false, false, false, false,
false, false, false, false};
uint32_t timestamp = 0;
for (int i = 0; i < 16; ++i) {
const bool is_keyframe = (i == 0);
CodecSpecificInfo info;
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[i],
LibvpxVp8Encoder::EncodeFlags(tl_config))
<< i;
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe,
kDefaultQp, &info);
EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config));
EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx);
EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
EXPECT_EQ(is_keyframe || expected_layer_sync[i],
info.codecSpecific.VP8.layerSync);
EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
timestamp += 3000;
}
}
TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) {
constexpr int kNumLayers = 3;
// Use a repeating pattern of tl 0, 2, 1, 2.
// Tl 0, 1, 2 update last, golden, altref respectively.
ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
DefaultTemporalLayers tl(kNumLayers);
DefaultTemporalLayersChecker checker(kNumLayers);
tl.OnRatesUpdated(0,
GetTemporalLayerRates(kDefaultBytesPerFrame,
kDefaultFramerate, kNumLayers),
kDefaultFramerate);
tl.UpdateConfiguration(0);
// Start with a keyframe.
uint32_t timestamp = 0;
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
IgnoredCodecSpecificInfo());
// Dropped TL2 frame.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
// Dropped TL1 frame.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
// TL2 frame. Can reference all three buffers, valid since golden and altref
// both contain the last keyframe.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference);
// Restart of cycle!
// TL0 base layer frame, updating and referencing last.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
// TL2 frame, updating altref.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
// TL1 frame, updating golden.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
// TL2 frame. Can still reference all buffer since they have been update this
// cycle.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference);
// Restart of cycle!
// TL0 base layer frame, updating and referencing last.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
// Dropped TL2 frame.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
// Dropped TL1 frame.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
// TL2 frame. This time golden and altref contain data from the previous cycle
// and cannot be referenced.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference);
EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
}
TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExist) {
constexpr int kNumLayers = 3;
// Use a repeating pattern of tl 0, 2, 1, 2.
// Tl 0, 1 updates last, golden respectively. Altref is always last keyframe.
DefaultTemporalLayers tl(kNumLayers);
DefaultTemporalLayersChecker checker(kNumLayers);
tl.OnRatesUpdated(0,
GetTemporalLayerRates(kDefaultBytesPerFrame,
kDefaultFramerate, kNumLayers),
kDefaultFramerate);
tl.UpdateConfiguration(0);
// Start with a keyframe.
uint32_t timestamp = 0;
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
IgnoredCodecSpecificInfo());
// Do a full cycle of the pattern.
for (int i = 0; i < 7; ++i) {
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
}
// TL0 base layer frame, starting the cycle over.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
// TL2 frame.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
// Encoder has a hiccup and builds a queue, so frame encoding is delayed.
// TL1 frame, updating golden.
tl_config = tl.NextFrameConfig(0, ++timestamp);
// TL2 frame, that should be referencing golden, but we can't be certain it's
// not going to be dropped, so that is not allowed.
tl_config = tl.NextFrameConfig(0, timestamp + 1);
EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference);
EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
// TL0 base layer frame.
tl_config = tl.NextFrameConfig(0, timestamp + 2);
// The previous four enqueued frames finally get encoded, and the updated
// buffers are now OK to reference.
// Enqueued TL1 frame ready.
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
// Enqueued TL2 frame.
tl.OnEncodeDone(0, ++timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
// Enqueued TL0 frame.
tl.OnEncodeDone(0, ++timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
// TL2 frame, all buffers are now in a known good state, OK to reference.
tl_config = tl.NextFrameConfig(0, ++timestamp + 1);
EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
}
TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExistLongDelay) {
constexpr int kNumLayers = 3;
// Use a repeating pattern of tl 0, 2, 1, 2.
// Tl 0, 1 updates last, golden, altref respectively.
ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
DefaultTemporalLayers tl(kNumLayers);
DefaultTemporalLayersChecker checker(kNumLayers);
tl.OnRatesUpdated(0,
GetTemporalLayerRates(kDefaultBytesPerFrame,
kDefaultFramerate, kNumLayers),
kDefaultFramerate);
tl.UpdateConfiguration(0);
// Start with a keyframe.
uint32_t timestamp = 0;
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
IgnoredCodecSpecificInfo());
// Do a full cycle of the pattern.
for (int i = 0; i < 3; ++i) {
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
}
// TL0 base layer frame, starting the cycle over.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
// TL2 frame.
tl_config = tl.NextFrameConfig(0, ++timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
// Encoder has a hiccup and builds a queue, so frame encoding is delayed.
// Encoded, but delayed frames in TL 1, 2.
tl_config = tl.NextFrameConfig(0, timestamp + 1);
tl_config = tl.NextFrameConfig(0, timestamp + 2);
// Restart of the pattern!
// Encoded, but delayed frames in TL 2, 1.
tl_config = tl.NextFrameConfig(0, timestamp + 3);
tl_config = tl.NextFrameConfig(0, timestamp + 4);
// TL1 frame from last cycle is ready.
tl.OnEncodeDone(0, timestamp + 1, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
// TL2 frame from last cycle is ready.
tl.OnEncodeDone(0, timestamp + 2, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
// TL2 frame, that should be referencing all buffers, but altref and golden
// haven not been updated this cycle. (Don't be fooled by the late frames from
// the last cycle!)
tl_config = tl.NextFrameConfig(0, timestamp + 5);
EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference);
EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
}
TEST_F(TemporalLayersTest, KeyFrame) {
constexpr int kNumLayers = 3;
DefaultTemporalLayers tl(kNumLayers);
DefaultTemporalLayersChecker checker(kNumLayers);
tl.OnRatesUpdated(0,
GetTemporalLayerRates(kDefaultBytesPerFrame,
kDefaultFramerate, kNumLayers),
kDefaultFramerate);
tl.UpdateConfiguration(0);
int expected_flags[8] = {
kTemporalUpdateLastRefAltRef,
kTemporalUpdateNoneNoRefGoldenAltRef,
kTemporalUpdateGoldenWithoutDependency,
kTemporalUpdateNoneNoRefAltRef,
kTemporalUpdateLast,
kTemporalUpdateNoneNoRefAltRef,
kTemporalUpdateGolden,
kTemporalUpdateNone,
};
int expected_temporal_idx[8] = {0, 2, 1, 2, 0, 2, 1, 2};
bool expected_layer_sync[8] = {true, true, true, false,
false, false, false, false};
uint32_t timestamp = 0;
for (int i = 0; i < 7; ++i) {
// Temporal pattern starts from 0 after key frame. Let the first |i| - 1
// frames be delta frames, and the |i|th one key frame.
for (int j = 1; j <= i; ++j) {
// Since last frame was always a keyframe and thus index 0 in the pattern,
// this loop starts at index 1.
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
EXPECT_EQ(expected_flags[j], LibvpxVp8Encoder::EncodeFlags(tl_config))
<< j;
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfo());
EXPECT_TRUE(checker.CheckTemporalConfig(false, tl_config));
EXPECT_EQ(expected_temporal_idx[j], tl_config.packetizer_temporal_idx);
EXPECT_EQ(expected_temporal_idx[j], tl_config.encoder_layer_id);
EXPECT_EQ(expected_layer_sync[j], tl_config.layer_sync);
timestamp += 3000;
}
CodecSpecificInfo info;
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
&info);
EXPECT_TRUE(info.codecSpecific.VP8.layerSync)
<< "Key frame should be marked layer sync.";
EXPECT_EQ(0, info.codecSpecific.VP8.temporalIdx)
<< "Key frame should always be packetized as layer 0";
EXPECT_EQ(0, info.generic_frame_info->temporal_id)
<< "Key frame should always be packetized as layer 0";
EXPECT_THAT(info.generic_frame_info->decode_target_indications,
Each(DecodeTargetIndication::kSwitch))
<< "Key frame is universal switch";
EXPECT_TRUE(checker.CheckTemporalConfig(true, tl_config));
}
}
class TemporalLayersReferenceTest : public TemporalLayersTest,
public ::testing::WithParamInterface<int> {
public:
TemporalLayersReferenceTest()
: timestamp_(1),
last_sync_timestamp_(timestamp_),
tl0_reference_(nullptr) {}
virtual ~TemporalLayersReferenceTest() {}
protected:
static const int kMaxPatternLength = 32;
struct BufferState {
BufferState() : BufferState(-1, 0, false) {}
BufferState(int temporal_idx, uint32_t timestamp, bool sync)
: temporal_idx(temporal_idx), timestamp(timestamp), sync(sync) {}
int temporal_idx;
uint32_t timestamp;
bool sync;
};
bool UpdateSyncRefState(const BufferFlags& flags, BufferState* buffer_state) {
if (flags & BufferFlags::kReference) {
if (buffer_state->temporal_idx == -1)
return true; // References key-frame.
if (buffer_state->temporal_idx == 0) {
// No more than one reference to TL0 frame.
EXPECT_EQ(nullptr, tl0_reference_);
tl0_reference_ = buffer_state;
return true;
}
return false; // References higher layer.
}
return true; // No reference, does not affect sync frame status.
}
void ValidateReference(const BufferFlags& flags,
const BufferState& buffer_state,
int temporal_layer) {
if (flags & BufferFlags::kReference) {
if (temporal_layer > 0 && buffer_state.timestamp > 0) {
// Check that high layer reference does not go past last sync frame.
EXPECT_GE(buffer_state.timestamp, last_sync_timestamp_);
}
// No reference to buffer in higher layer.
EXPECT_LE(buffer_state.temporal_idx, temporal_layer);
}
}
uint32_t timestamp_ = 1;
uint32_t last_sync_timestamp_ = timestamp_;
BufferState* tl0_reference_;
BufferState last_state;
BufferState golden_state;
BufferState altref_state;
};
INSTANTIATE_TEST_SUITE_P(DefaultTemporalLayersTest,
TemporalLayersReferenceTest,
::testing::Range(1, kMaxTemporalStreams + 1));
TEST_P(TemporalLayersReferenceTest, ValidFrameConfigs) {
const int num_layers = GetParam();
DefaultTemporalLayers tl(num_layers);
tl.OnRatesUpdated(
0, GetTemporalLayerRates(kDefaultBytesPerFrame, kDefaultFramerate, 1),
kDefaultFramerate);
tl.UpdateConfiguration(0);
// Run through the pattern and store the frame dependencies, plus keep track
// of the buffer state; which buffers references which temporal layers (if
// (any). If a given buffer is never updated, it is legal to reference it
// even for sync frames. In order to be general, don't assume TL0 always
// updates |last|.
std::vector<Vp8FrameConfig> tl_configs(kMaxPatternLength);
for (int i = 0; i < kMaxPatternLength; ++i) {
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp_);
tl.OnEncodeDone(0, timestamp_, kDefaultBytesPerFrame, i == 0, kDefaultQp,
IgnoredCodecSpecificInfo());
++timestamp_;
EXPECT_FALSE(tl_config.drop_frame);
tl_configs.push_back(tl_config);
int temporal_idx = tl_config.encoder_layer_id;
// For the default layers, always keep encoder and rtp layers in sync.
EXPECT_EQ(tl_config.packetizer_temporal_idx, temporal_idx);
// Determine if this frame is in a higher layer but references only TL0
// or untouched buffers, if so verify it is marked as a layer sync.
bool is_sync_frame = true;
tl0_reference_ = nullptr;
if (temporal_idx <= 0) {
is_sync_frame = false; // TL0 by definition not a sync frame.
} else if (!UpdateSyncRefState(tl_config.last_buffer_flags, &last_state)) {
is_sync_frame = false;
} else if (!UpdateSyncRefState(tl_config.golden_buffer_flags,
&golden_state)) {
is_sync_frame = false;
} else if (!UpdateSyncRefState(tl_config.arf_buffer_flags, &altref_state)) {
is_sync_frame = false;
}
if (is_sync_frame) {
// Cache timestamp for last found sync frame, so that we can verify no
// references back past this frame.
ASSERT_TRUE(tl0_reference_);
last_sync_timestamp_ = tl0_reference_->timestamp;
}
EXPECT_EQ(tl_config.layer_sync, is_sync_frame);
// Validate no reference from lower to high temporal layer, or backwards
// past last reference frame.
ValidateReference(tl_config.last_buffer_flags, last_state, temporal_idx);
ValidateReference(tl_config.golden_buffer_flags, golden_state,
temporal_idx);
ValidateReference(tl_config.arf_buffer_flags, altref_state, temporal_idx);
// Update the current layer state.
BufferState state = {temporal_idx, timestamp_, is_sync_frame};
if (tl_config.last_buffer_flags & BufferFlags::kUpdate)
last_state = state;
if (tl_config.golden_buffer_flags & BufferFlags::kUpdate)
golden_state = state;
if (tl_config.arf_buffer_flags & BufferFlags::kUpdate)
altref_state = state;
}
}
} // namespace test
} // namespace webrtc

63
modules/video_coding/codecs/vp8/include/temporal_layers_checker.h Normal file
View File

@ -0,0 +1,63 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_TEMPORAL_LAYERS_CHECKER_H_
#define MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_TEMPORAL_LAYERS_CHECKER_H_
#include <stdint.h>
#include <memory>
#include "api/video_codecs/vp8_frame_config.h"
#include "api/video_codecs/vp8_temporal_layers.h"
namespace webrtc {
// Interface for a class that verifies correctness of temporal layer
// configurations (dependencies, sync flag, etc).
// Intended to be used in tests as well as with real apps in debug mode.
class TemporalLayersChecker {
public:
explicit TemporalLayersChecker(int num_temporal_layers);
virtual ~TemporalLayersChecker() {}
virtual bool CheckTemporalConfig(bool frame_is_keyframe,
const Vp8FrameConfig& frame_config);
static std::unique_ptr<TemporalLayersChecker> CreateTemporalLayersChecker(
Vp8TemporalLayersType type,
int num_temporal_layers);
private:
struct BufferState {
BufferState() : is_keyframe(true), temporal_layer(0), sequence_number(0) {}
bool is_keyframe;
uint8_t temporal_layer;
uint32_t sequence_number;
};
bool CheckAndUpdateBufferState(BufferState* state,
bool* need_sync,
bool frame_is_keyframe,
uint8_t temporal_layer,
Vp8FrameConfig::BufferFlags flags,
uint32_t sequence_number,
uint32_t* lowest_sequence_referenced);
BufferState last_;
BufferState arf_;
BufferState golden_;
int num_temporal_layers_;
uint32_t sequence_number_;
uint32_t last_sync_sequence_number_;
uint32_t last_tl0_sequence_number_;
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_TEMPORAL_LAYERS_CHECKER_H_

55
modules/video_coding/codecs/vp8/include/vp8.h Normal file
View File

@ -0,0 +1,55 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_H_
#define MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_H_
#include <memory>
#include <vector>
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/vp8_frame_buffer_controller.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "rtc_base/deprecation.h"
namespace webrtc {
// TODO(brandtr): Move these interfaces to the api/ folder.
class VP8Encoder {
public:
struct Settings {
// Allows for overriding the Vp8FrameBufferController used by the encoder.
// If unset, a default Vp8FrameBufferController will be instantiated
// internally.
std::unique_ptr<Vp8FrameBufferControllerFactory>
frame_buffer_controller_factory = nullptr;
// Allows for overriding the resolution/bitrate limits exposed through
// VideoEncoder::GetEncoderInfo(). No override is done if empty.
std::vector<VideoEncoder::ResolutionBitrateLimits>
resolution_bitrate_limits = {};
};
static std::unique_ptr<VideoEncoder> Create();
static std::unique_ptr<VideoEncoder> Create(Settings settings);
RTC_DEPRECATED static std::unique_ptr<VideoEncoder> Create(
std::unique_ptr<Vp8FrameBufferControllerFactory>
frame_buffer_controller_factory);
};
class VP8Decoder {
public:
static std::unique_ptr<VideoDecoder> Create();
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_H_

49
modules/video_coding/codecs/vp8/include/vp8_globals.h Normal file
View File

@ -0,0 +1,49 @@
/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
// This file contains codec dependent definitions that are needed in
// order to compile the WebRTC codebase, even if this codec is not used.
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_GLOBALS_H_
#define MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_GLOBALS_H_
#include "modules/video_coding/codecs/interface/common_constants.h"
namespace webrtc {
struct RTPVideoHeaderVP8 {
void InitRTPVideoHeaderVP8() {
nonReference = false;
pictureId = kNoPictureId;
tl0PicIdx = kNoTl0PicIdx;
temporalIdx = kNoTemporalIdx;
layerSync = false;
keyIdx = kNoKeyIdx;
partitionId = 0;
beginningOfPartition = false;
}
bool nonReference; // Frame is discardable.
int16_t pictureId; // Picture ID index, 15 bits;
// kNoPictureId if PictureID does not exist.
int16_t tl0PicIdx; // TL0PIC_IDX, 8 bits;
// kNoTl0PicIdx means no value provided.
uint8_t temporalIdx; // Temporal layer index, or kNoTemporalIdx.
bool layerSync; // This frame is a layer sync frame.
// Disabled if temporalIdx == kNoTemporalIdx.
int keyIdx; // 5 bits; kNoKeyIdx means not used.
int partitionId; // VP8 partition ID
bool beginningOfPartition; // True if this packet is the first
// in a VP8 partition. Otherwise false
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_GLOBALS_H_

206
modules/video_coding/codecs/vp8/libvpx_interface.cc Normal file
View File

@ -0,0 +1,206 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/vp8/libvpx_interface.h"
#include <memory>
#include "rtc_base/checks.h"
namespace webrtc {
namespace {
class LibvpxVp8Facade : public LibvpxInterface {
public:
LibvpxVp8Facade() = default;
~LibvpxVp8Facade() override = default;
vpx_image_t* img_alloc(vpx_image_t* img,
vpx_img_fmt_t fmt,
unsigned int d_w,
unsigned int d_h,
unsigned int align) const override {
return ::vpx_img_alloc(img, fmt, d_w, d_h, align);
}
vpx_image_t* img_wrap(vpx_image_t* img,
vpx_img_fmt_t fmt,
unsigned int d_w,
unsigned int d_h,
unsigned int stride_align,
unsigned char* img_data) const override {
return ::vpx_img_wrap(img, fmt, d_w, d_h, stride_align, img_data);
}
void img_free(vpx_image_t* img) const override { ::vpx_img_free(img); }
vpx_codec_err_t codec_enc_config_set(
vpx_codec_ctx_t* ctx,
const vpx_codec_enc_cfg_t* cfg) const override {
return ::vpx_codec_enc_config_set(ctx, cfg);
}
vpx_codec_err_t codec_enc_config_default(vpx_codec_iface_t* iface,
vpx_codec_enc_cfg_t* cfg,
unsigned int usage) const override {
return ::vpx_codec_enc_config_default(iface, cfg, usage);
}
vpx_codec_err_t codec_enc_init(vpx_codec_ctx_t* ctx,
vpx_codec_iface_t* iface,
const vpx_codec_enc_cfg_t* cfg,
vpx_codec_flags_t flags) const override {
return ::vpx_codec_enc_init(ctx, iface, cfg, flags);
}
vpx_codec_err_t codec_enc_init_multi(vpx_codec_ctx_t* ctx,
vpx_codec_iface_t* iface,
vpx_codec_enc_cfg_t* cfg,
int num_enc,
vpx_codec_flags_t flags,
vpx_rational_t* dsf) const override {
return ::vpx_codec_enc_init_multi(ctx, iface, cfg, num_enc, flags, dsf);
}
vpx_codec_err_t codec_destroy(vpx_codec_ctx_t* ctx) const override {
return ::vpx_codec_destroy(ctx);
}
// For types related to these parameters, see section
// "VP8 encoder control function parameter type" in vpx/vp8cx.h.
vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx,
vp8e_enc_control_id ctrl_id,
uint32_t param) const override {
// We need an explicit call for each type since vpx_codec_control is a
// macro that gets expanded into another call based on the parameter name.
switch (ctrl_id) {
case VP8E_SET_ENABLEAUTOALTREF:
return vpx_codec_control(ctx, VP8E_SET_ENABLEAUTOALTREF, param);
case VP8E_SET_NOISE_SENSITIVITY:
return vpx_codec_control(ctx, VP8E_SET_NOISE_SENSITIVITY, param);
case VP8E_SET_SHARPNESS:
return vpx_codec_control(ctx, VP8E_SET_SHARPNESS, param);
case VP8E_SET_STATIC_THRESHOLD:
return vpx_codec_control(ctx, VP8E_SET_STATIC_THRESHOLD, param);
case VP8E_SET_ARNR_MAXFRAMES:
return vpx_codec_control(ctx, VP8E_SET_ARNR_MAXFRAMES, param);
case VP8E_SET_ARNR_STRENGTH:
return vpx_codec_control(ctx, VP8E_SET_ARNR_STRENGTH, param);
case VP8E_SET_ARNR_TYPE:
RTC_NOTREACHED() << "VP8E_SET_ARNR_TYPE is deprecated.";
return VPX_CODEC_UNSUP_FEATURE;
case VP8E_SET_CQ_LEVEL:
return vpx_codec_control(ctx, VP8E_SET_CQ_LEVEL, param);
case VP8E_SET_MAX_INTRA_BITRATE_PCT:
return vpx_codec_control(ctx, VP8E_SET_MAX_INTRA_BITRATE_PCT, param);
case VP8E_SET_GF_CBR_BOOST_PCT:
return vpx_codec_control(ctx, VP8E_SET_GF_CBR_BOOST_PCT, param);
case VP8E_SET_SCREEN_CONTENT_MODE:
return vpx_codec_control(ctx, VP8E_SET_SCREEN_CONTENT_MODE, param);
default:
RTC_NOTREACHED() << "Unsupported libvpx ctrl_id: " << ctrl_id;
}
return VPX_CODEC_ERROR;
}
vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx,
vp8e_enc_control_id ctrl_id,
int param) const override {
switch (ctrl_id) {
case VP8E_SET_FRAME_FLAGS:
return vpx_codec_control(ctx, VP8E_SET_FRAME_FLAGS, param);
case VP8E_SET_TEMPORAL_LAYER_ID:
return vpx_codec_control(ctx, VP8E_SET_TEMPORAL_LAYER_ID, param);
case VP8E_SET_CPUUSED:
return vpx_codec_control(ctx, VP8E_SET_CPUUSED, param);
case VP8E_SET_TOKEN_PARTITIONS:
return vpx_codec_control(ctx, VP8E_SET_TOKEN_PARTITIONS, param);
case VP8E_SET_TUNING:
return vpx_codec_control(ctx, VP8E_SET_TUNING, param);
default:
RTC_NOTREACHED() << "Unsupported libvpx ctrl_id: " << ctrl_id;
}
return VPX_CODEC_ERROR;
}
vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx,
vp8e_enc_control_id ctrl_id,
int* param) const override {
switch (ctrl_id) {
case VP8E_GET_LAST_QUANTIZER:
return vpx_codec_control(ctx, VP8E_GET_LAST_QUANTIZER, param);
case VP8E_GET_LAST_QUANTIZER_64:
return vpx_codec_control(ctx, VP8E_GET_LAST_QUANTIZER_64, param);
default:
RTC_NOTREACHED() << "Unsupported libvpx ctrl_id: " << ctrl_id;
}
return VPX_CODEC_ERROR;
}
vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx,
vp8e_enc_control_id ctrl_id,
vpx_roi_map* param) const override {
switch (ctrl_id) {
case VP8E_SET_ROI_MAP:
return vpx_codec_control(ctx, VP8E_SET_ROI_MAP, param);
default:
RTC_NOTREACHED() << "Unsupported libvpx ctrl_id: " << ctrl_id;
}
return VPX_CODEC_ERROR;
}
vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx,
vp8e_enc_control_id ctrl_id,
vpx_active_map* param) const override {
switch (ctrl_id) {
case VP8E_SET_ACTIVEMAP:
return vpx_codec_control(ctx, VP8E_SET_ACTIVEMAP, param);
default:
RTC_NOTREACHED() << "Unsupported libvpx ctrl_id: " << ctrl_id;
}
return VPX_CODEC_ERROR;
}
vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx,
vp8e_enc_control_id ctrl_id,
vpx_scaling_mode* param) const override {
switch (ctrl_id) {
case VP8E_SET_SCALEMODE:
return vpx_codec_control(ctx, VP8E_SET_SCALEMODE, param);
default:
RTC_NOTREACHED() << "Unsupported libvpx ctrl_id: " << ctrl_id;
}
return VPX_CODEC_ERROR;
}
vpx_codec_err_t codec_encode(vpx_codec_ctx_t* ctx,
const vpx_image_t* img,
vpx_codec_pts_t pts,
uint64_t duration,
vpx_enc_frame_flags_t flags,
uint64_t deadline) const override {
return ::vpx_codec_encode(ctx, img, pts, duration, flags, deadline);
}
const vpx_codec_cx_pkt_t* codec_get_cx_data(
vpx_codec_ctx_t* ctx,
vpx_codec_iter_t* iter) const override {
return ::vpx_codec_get_cx_data(ctx, iter);
}
};
} // namespace
std::unique_ptr<LibvpxInterface> LibvpxInterface::CreateEncoder() {
return std::make_unique<LibvpxVp8Facade>();
}
} // namespace webrtc

102
modules/video_coding/codecs/vp8/libvpx_interface.h Normal file
View File

@ -0,0 +1,102 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_INTERFACE_H_
#define MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_INTERFACE_H_
#include <stdint.h>
#include <memory>
#include "vpx/vp8cx.h"
#include "vpx/vpx_codec.h"
#include "vpx/vpx_encoder.h"
#include "vpx/vpx_image.h"
namespace webrtc {
// This interface is a proxy to to the static libvpx functions, so that they
// can be mocked for testing. Currently supports VP8 encoder functions.
// TODO(sprang): Extend this to VP8 decoder and VP9 encoder/decoder too.
class LibvpxInterface {
public:
LibvpxInterface() = default;
virtual ~LibvpxInterface() = default;
virtual vpx_image_t* img_alloc(vpx_image_t* img,
vpx_img_fmt_t fmt,
unsigned int d_w,
unsigned int d_h,
unsigned int align) const = 0;
virtual vpx_image_t* img_wrap(vpx_image_t* img,
vpx_img_fmt_t fmt,
unsigned int d_w,
unsigned int d_h,
unsigned int stride_align,
unsigned char* img_data) const = 0;
virtual void img_free(vpx_image_t* img) const = 0;
virtual vpx_codec_err_t codec_enc_config_set(
vpx_codec_ctx_t* ctx,
const vpx_codec_enc_cfg_t* cfg) const = 0;
virtual vpx_codec_err_t codec_enc_config_default(
vpx_codec_iface_t* iface,
vpx_codec_enc_cfg_t* cfg,
unsigned int usage) const = 0;
virtual vpx_codec_err_t codec_enc_init(vpx_codec_ctx_t* ctx,
vpx_codec_iface_t* iface,
const vpx_codec_enc_cfg_t* cfg,
vpx_codec_flags_t flags) const = 0;
virtual vpx_codec_err_t codec_enc_init_multi(vpx_codec_ctx_t* ctx,
vpx_codec_iface_t* iface,
vpx_codec_enc_cfg_t* cfg,
int num_enc,
vpx_codec_flags_t flags,
vpx_rational_t* dsf) const = 0;
virtual vpx_codec_err_t codec_destroy(vpx_codec_ctx_t* ctx) const = 0;
virtual vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx,
vp8e_enc_control_id ctrl_id,
uint32_t param) const = 0;
virtual vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx,
vp8e_enc_control_id ctrl_id,
int param) const = 0;
virtual vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx,
vp8e_enc_control_id ctrl_id,
int* param) const = 0;
virtual vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx,
vp8e_enc_control_id ctrl_id,
vpx_roi_map* param) const = 0;
virtual vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx,
vp8e_enc_control_id ctrl_id,
vpx_active_map* param) const = 0;
virtual vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx,
vp8e_enc_control_id ctrl_id,
vpx_scaling_mode* param) const = 0;
virtual vpx_codec_err_t codec_encode(vpx_codec_ctx_t* ctx,
const vpx_image_t* img,
vpx_codec_pts_t pts,
uint64_t duration,
vpx_enc_frame_flags_t flags,
uint64_t deadline) const = 0;
virtual const vpx_codec_cx_pkt_t* codec_get_cx_data(
vpx_codec_ctx_t* ctx,
vpx_codec_iter_t* iter) const = 0;
// Returns interface wrapping the actual libvpx functions.
static std::unique_ptr<LibvpxInterface> CreateEncoder();
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_INTERFACE_H_

363
modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc Normal file
View File

@ -0,0 +1,363 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h"
#include <stdio.h>
#include <string.h>
#include <algorithm>
#include <memory>
#include <string>
#include "absl/types/optional.h"
#include "api/scoped_refptr.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_frame.h"
#include "api/video/video_frame_buffer.h"
#include "api/video/video_rotation.h"
#include "modules/video_coding/codecs/vp8/include/vp8.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/checks.h"
#include "rtc_base/numerics/exp_filter.h"
#include "rtc_base/time_utils.h"
#include "system_wrappers/include/field_trial.h"
#include "system_wrappers/include/metrics.h"
#include "third_party/libyuv/include/libyuv/convert.h"
#include "vpx/vp8.h"
#include "vpx/vp8dx.h"
#include "vpx/vpx_decoder.h"
namespace webrtc {
namespace {
constexpr int kVp8ErrorPropagationTh = 30;
// vpx_decoder.h documentation indicates decode deadline is time in us, with
// "Set to zero for unlimited.", but actual implementation requires this to be
// a mode with 0 meaning allow delay and 1 not allowing it.
constexpr long kDecodeDeadlineRealtime = 1; // NOLINT
const char kVp8PostProcArmFieldTrial[] = "WebRTC-VP8-Postproc-Config-Arm";
void GetPostProcParamsFromFieldTrialGroup(
LibvpxVp8Decoder::DeblockParams* deblock_params) {
std::string group =
webrtc::field_trial::FindFullName(kVp8PostProcArmFieldTrial);
if (group.empty())
return;
LibvpxVp8Decoder::DeblockParams params;
if (sscanf(group.c_str(), "Enabled-%d,%d,%d", &params.max_level,
&params.min_qp, &params.degrade_qp) != 3)
return;
if (params.max_level < 0 || params.max_level > 16)
return;
if (params.min_qp < 0 || params.degrade_qp <= params.min_qp)
return;
*deblock_params = params;
}
} // namespace
std::unique_ptr<VideoDecoder> VP8Decoder::Create() {
return std::make_unique<LibvpxVp8Decoder>();
}
class LibvpxVp8Decoder::QpSmoother {
public:
QpSmoother() : last_sample_ms_(rtc::TimeMillis()), smoother_(kAlpha) {}
int GetAvg() const {
float value = smoother_.filtered();
return (value == rtc::ExpFilter::kValueUndefined) ? 0
: static_cast<int>(value);
}
void Add(float sample) {
int64_t now_ms = rtc::TimeMillis();
smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample);
last_sample_ms_ = now_ms;
}
void Reset() { smoother_.Reset(kAlpha); }
private:
const float kAlpha = 0.95f;
int64_t last_sample_ms_;
rtc::ExpFilter smoother_;
};
LibvpxVp8Decoder::LibvpxVp8Decoder()
: use_postproc_arm_(
webrtc::field_trial::IsEnabled(kVp8PostProcArmFieldTrial)),
buffer_pool_(false, 300 /* max_number_of_buffers*/),
decode_complete_callback_(NULL),
inited_(false),
decoder_(NULL),
propagation_cnt_(-1),
last_frame_width_(0),
last_frame_height_(0),
key_frame_required_(true),
qp_smoother_(use_postproc_arm_ ? new QpSmoother() : nullptr) {
if (use_postproc_arm_)
GetPostProcParamsFromFieldTrialGroup(&deblock_);
}
LibvpxVp8Decoder::~LibvpxVp8Decoder() {
inited_ = true; // in order to do the actual release
Release();
}
int LibvpxVp8Decoder::InitDecode(const VideoCodec* inst, int number_of_cores) {
int ret_val = Release();
if (ret_val < 0) {
return ret_val;
}
if (decoder_ == NULL) {
decoder_ = new vpx_codec_ctx_t;
memset(decoder_, 0, sizeof(*decoder_));
}
vpx_codec_dec_cfg_t cfg;
// Setting number of threads to a constant value (1)
cfg.threads = 1;
cfg.h = cfg.w = 0; // set after decode
#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
defined(WEBRTC_ANDROID)
vpx_codec_flags_t flags = use_postproc_arm_ ? VPX_CODEC_USE_POSTPROC : 0;
#else
vpx_codec_flags_t flags = VPX_CODEC_USE_POSTPROC;
#endif
if (vpx_codec_dec_init(decoder_, vpx_codec_vp8_dx(), &cfg, flags)) {
delete decoder_;
decoder_ = nullptr;
return WEBRTC_VIDEO_CODEC_MEMORY;
}
propagation_cnt_ = -1;
inited_ = true;
// Always start with a complete key frame.
key_frame_required_ = true;
if (inst && inst->buffer_pool_size) {
if (!buffer_pool_.Resize(*inst->buffer_pool_size)) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
}
return WEBRTC_VIDEO_CODEC_OK;
}
int LibvpxVp8Decoder::Decode(const EncodedImage& input_image,
bool missing_frames,
int64_t /*render_time_ms*/) {
if (!inited_) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
if (decode_complete_callback_ == NULL) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
if (input_image.data() == NULL && input_image.size() > 0) {
// Reset to avoid requesting key frames too often.
if (propagation_cnt_ > 0)
propagation_cnt_ = 0;
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
// Post process configurations.
#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
defined(WEBRTC_ANDROID)
if (use_postproc_arm_) {
vp8_postproc_cfg_t ppcfg;
ppcfg.post_proc_flag = VP8_MFQE;
// For low resolutions, use stronger deblocking filter.
int last_width_x_height = last_frame_width_ * last_frame_height_;
if (last_width_x_height > 0 && last_width_x_height <= 320 * 240) {
// Enable the deblock and demacroblocker based on qp thresholds.
RTC_DCHECK(qp_smoother_);
int qp = qp_smoother_->GetAvg();
if (qp > deblock_.min_qp) {
int level = deblock_.max_level;
if (qp < deblock_.degrade_qp) {
// Use lower level.
level = deblock_.max_level * (qp - deblock_.min_qp) /
(deblock_.degrade_qp - deblock_.min_qp);
}
// Deblocking level only affects VP8_DEMACROBLOCK.
ppcfg.deblocking_level = std::max(level, 1);
ppcfg.post_proc_flag |= VP8_DEBLOCK | VP8_DEMACROBLOCK;
}
}
vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg);
}
#else
vp8_postproc_cfg_t ppcfg;
// MFQE enabled to reduce key frame popping.
ppcfg.post_proc_flag = VP8_MFQE | VP8_DEBLOCK;
// For VGA resolutions and lower, enable the demacroblocker postproc.
if (last_frame_width_ * last_frame_height_ <= 640 * 360) {
ppcfg.post_proc_flag |= VP8_DEMACROBLOCK;
}
// Strength of deblocking filter. Valid range:[0,16]
ppcfg.deblocking_level = 3;
vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg);
#endif
// Always start with a complete key frame.
if (key_frame_required_) {
if (input_image._frameType != VideoFrameType::kVideoFrameKey)
return WEBRTC_VIDEO_CODEC_ERROR;
// We have a key frame - is it complete?
if (input_image._completeFrame) {
key_frame_required_ = false;
} else {
return WEBRTC_VIDEO_CODEC_ERROR;
}
}
// Restrict error propagation using key frame requests.
// Reset on a key frame refresh.
if (input_image._frameType == VideoFrameType::kVideoFrameKey &&
input_image._completeFrame) {
propagation_cnt_ = -1;
// Start count on first loss.
} else if ((!input_image._completeFrame || missing_frames) &&
propagation_cnt_ == -1) {
propagation_cnt_ = 0;
}
if (propagation_cnt_ >= 0) {
propagation_cnt_++;
}
vpx_codec_iter_t iter = NULL;
vpx_image_t* img;
int ret;
// Check for missing frames.
if (missing_frames) {
// Call decoder with zero data length to signal missing frames.
if (vpx_codec_decode(decoder_, NULL, 0, 0, kDecodeDeadlineRealtime)) {
// Reset to avoid requesting key frames too often.
if (propagation_cnt_ > 0)
propagation_cnt_ = 0;
return WEBRTC_VIDEO_CODEC_ERROR;
}
img = vpx_codec_get_frame(decoder_, &iter);
iter = NULL;
}
const uint8_t* buffer = input_image.data();
if (input_image.size() == 0) {
buffer = NULL; // Triggers full frame concealment.
}
if (vpx_codec_decode(decoder_, buffer, input_image.size(), 0,
kDecodeDeadlineRealtime)) {
// Reset to avoid requesting key frames too often.
if (propagation_cnt_ > 0) {
propagation_cnt_ = 0;
}
return WEBRTC_VIDEO_CODEC_ERROR;
}
img = vpx_codec_get_frame(decoder_, &iter);
int qp;
vpx_codec_err_t vpx_ret =
vpx_codec_control(decoder_, VPXD_GET_LAST_QUANTIZER, &qp);
RTC_DCHECK_EQ(vpx_ret, VPX_CODEC_OK);
ret = ReturnFrame(img, input_image.Timestamp(), qp, input_image.ColorSpace());
if (ret != 0) {
// Reset to avoid requesting key frames too often.
if (ret < 0 && propagation_cnt_ > 0)
propagation_cnt_ = 0;
return ret;
}
// Check Vs. threshold
if (propagation_cnt_ > kVp8ErrorPropagationTh) {
// Reset to avoid requesting key frames too often.
propagation_cnt_ = 0;
return WEBRTC_VIDEO_CODEC_ERROR;
}
return WEBRTC_VIDEO_CODEC_OK;
}
int LibvpxVp8Decoder::ReturnFrame(
const vpx_image_t* img,
uint32_t timestamp,
int qp,
const webrtc::ColorSpace* explicit_color_space) {
if (img == NULL) {
// Decoder OK and NULL image => No show frame
return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
}
if (qp_smoother_) {
if (last_frame_width_ != static_cast<int>(img->d_w) ||
last_frame_height_ != static_cast<int>(img->d_h)) {
qp_smoother_->Reset();
}
qp_smoother_->Add(qp);
}
last_frame_width_ = img->d_w;
last_frame_height_ = img->d_h;
// Allocate memory for decoded image.
rtc::scoped_refptr<I420Buffer> buffer =
buffer_pool_.CreateBuffer(img->d_w, img->d_h);
if (!buffer.get()) {
// Pool has too many pending frames.
RTC_HISTOGRAM_BOOLEAN("WebRTC.Video.LibvpxVp8Decoder.TooManyPendingFrames",
1);
return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
}
libyuv::I420Copy(img->planes[VPX_PLANE_Y], img->stride[VPX_PLANE_Y],
img->planes[VPX_PLANE_U], img->stride[VPX_PLANE_U],
img->planes[VPX_PLANE_V], img->stride[VPX_PLANE_V],
buffer->MutableDataY(), buffer->StrideY(),
buffer->MutableDataU(), buffer->StrideU(),
buffer->MutableDataV(), buffer->StrideV(), img->d_w,
img->d_h);
VideoFrame decoded_image = VideoFrame::Builder()
.set_video_frame_buffer(buffer)
.set_timestamp_rtp(timestamp)
.set_color_space(explicit_color_space)
.build();
decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp);
return WEBRTC_VIDEO_CODEC_OK;
}
int LibvpxVp8Decoder::RegisterDecodeCompleteCallback(
DecodedImageCallback* callback) {
decode_complete_callback_ = callback;
return WEBRTC_VIDEO_CODEC_OK;
}
int LibvpxVp8Decoder::Release() {
int ret_val = WEBRTC_VIDEO_CODEC_OK;
if (decoder_ != NULL) {
if (inited_) {
if (vpx_codec_destroy(decoder_)) {
ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
}
}
delete decoder_;
decoder_ = NULL;
}
buffer_pool_.Release();
inited_ = false;
return ret_val;
}
const char* LibvpxVp8Decoder::ImplementationName() const {
return "libvpx";
}
} // namespace webrtc

70
modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h Normal file
View File

@ -0,0 +1,70 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_DECODER_H_
#define MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_DECODER_H_
#include <memory>
#include "api/video/encoded_image.h"
#include "api/video_codecs/video_decoder.h"
#include "common_video/include/i420_buffer_pool.h"
#include "modules/video_coding/codecs/vp8/include/vp8.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "vpx/vp8dx.h"
#include "vpx/vpx_decoder.h"
namespace webrtc {
class LibvpxVp8Decoder : public VideoDecoder {
public:
LibvpxVp8Decoder();
~LibvpxVp8Decoder() override;
int InitDecode(const VideoCodec* inst, int number_of_cores) override;
int Decode(const EncodedImage& input_image,
bool missing_frames,
int64_t /*render_time_ms*/) override;
int RegisterDecodeCompleteCallback(DecodedImageCallback* callback) override;
int Release() override;
const char* ImplementationName() const override;
struct DeblockParams {
int max_level = 6; // Deblocking strength: [0, 16].
int degrade_qp = 1; // If QP value is below, start lowering |max_level|.
int min_qp = 0; // If QP value is below, turn off deblocking.
};
private:
class QpSmoother;
int ReturnFrame(const vpx_image_t* img,
uint32_t timeStamp,
int qp,
const webrtc::ColorSpace* explicit_color_space);
const bool use_postproc_arm_;
I420BufferPool buffer_pool_;
DecodedImageCallback* decode_complete_callback_;
bool inited_;
vpx_codec_ctx_t* decoder_;
int propagation_cnt_;
int last_frame_width_;
int last_frame_height_;
bool key_frame_required_;
DeblockParams deblock_;
const std::unique_ptr<QpSmoother> qp_smoother_;
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_DECODER_H_

1306
modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc Normal file
View File

File diff suppressed because it is too large Load Diff

147
modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h Normal file
View File

@ -0,0 +1,147 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_ENCODER_H_
#define MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_ENCODER_H_
#include <memory>
#include <string>
#include <vector>
#include "api/fec_controller_override.h"
#include "api/video/encoded_image.h"
#include "api/video/video_frame.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/vp8_frame_buffer_controller.h"
#include "api/video_codecs/vp8_frame_config.h"
#include "modules/video_coding/codecs/vp8/include/vp8.h"
#include "modules/video_coding/codecs/vp8/libvpx_interface.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/utility/framerate_controller.h"
#include "rtc_base/experiments/cpu_speed_experiment.h"
#include "rtc_base/experiments/rate_control_settings.h"
#include "vpx/vp8cx.h"
#include "vpx/vpx_encoder.h"
namespace webrtc {
class LibvpxVp8Encoder : public VideoEncoder {
public:
LibvpxVp8Encoder(std::unique_ptr<LibvpxInterface> interface,
VP8Encoder::Settings settings);
~LibvpxVp8Encoder() override;
int Release() override;
void SetFecControllerOverride(
FecControllerOverride* fec_controller_override) override;
int InitEncode(const VideoCodec* codec_settings,
const VideoEncoder::Settings& settings) override;
int Encode(const VideoFrame& input_image,
const std::vector<VideoFrameType>* frame_types) override;
int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override;
void SetRates(const RateControlParameters& parameters) override;
void OnPacketLossRateUpdate(float packet_loss_rate) override;
void OnRttUpdate(int64_t rtt_ms) override;
void OnLossNotification(const LossNotification& loss_notification) override;
EncoderInfo GetEncoderInfo() const override;
static vpx_enc_frame_flags_t EncodeFlags(const Vp8FrameConfig& references);
private:
// Get the cpu_speed setting for encoder based on resolution and/or platform.
int GetCpuSpeed(int width, int height);
// Determine number of encoder threads to use.
int NumberOfThreads(int width, int height, int number_of_cores);
// Call encoder initialize function and set control settings.
int InitAndSetControlSettings();
void PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
const vpx_codec_cx_pkt& pkt,
int stream_idx,
int encoder_idx,
uint32_t timestamp);
int GetEncodedPartitions(const VideoFrame& input_image,
bool retransmission_allowed);
// Set the stream state for stream |stream_idx|.
void SetStreamState(bool send_stream, int stream_idx);
uint32_t MaxIntraTarget(uint32_t optimal_buffer_size);
uint32_t FrameDropThreshold(size_t spatial_idx) const;
size_t SteadyStateSize(int sid, int tid);
bool UpdateVpxConfiguration(size_t stream_index);
const std::unique_ptr<LibvpxInterface> libvpx_;
const absl::optional<std::vector<CpuSpeedExperiment::Config>>
experimental_cpu_speed_config_arm_;
const RateControlSettings rate_control_settings_;
EncodedImageCallback* encoded_complete_callback_ = nullptr;
VideoCodec codec_;
bool inited_ = false;
int64_t timestamp_ = 0;
int qp_max_ = 56;
int cpu_speed_default_ = -6;
int number_of_cores_ = 0;
uint32_t rc_max_intra_target_ = 0;
int num_active_streams_ = 0;
const std::unique_ptr<Vp8FrameBufferControllerFactory>
frame_buffer_controller_factory_;
std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_;
const std::vector<VideoEncoder::ResolutionBitrateLimits>
resolution_bitrate_limits_;
std::vector<bool> key_frame_request_;
std::vector<bool> send_stream_;
std::vector<int> cpu_speed_;
std::vector<vpx_image_t> raw_images_;
std::vector<EncodedImage> encoded_images_;
std::vector<vpx_codec_ctx_t> encoders_;
std::vector<vpx_codec_enc_cfg_t> vpx_configs_;
std::vector<Vp8EncoderConfig> config_overrides_;
std::vector<vpx_rational_t> downsampling_factors_;
// Variable frame-rate screencast related fields and methods.
const struct VariableFramerateExperiment {
bool enabled = false;
// Framerate is limited to this value in steady state.
float framerate_limit = 5.0;
// This qp or below is considered a steady state.
int steady_state_qp = 15;
// Frames of at least this percentage below ideal for configured bitrate are
// considered in a steady state.
int steady_state_undershoot_percentage = 30;
} variable_framerate_experiment_;
static VariableFramerateExperiment ParseVariableFramerateConfig(
std::string group_name);
FramerateController framerate_controller_;
int num_steady_state_frames_ = 0;
FecControllerOverride* fec_controller_override_ = nullptr;
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_ENCODER_H_

108
modules/video_coding/codecs/vp8/libvpx_vp8_simulcast_test.cc Normal file
View File

@ -0,0 +1,108 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <memory>
#include "api/test/create_simulcast_test_fixture.h"
#include "api/test/simulcast_test_fixture.h"
#include "api/test/video/function_video_decoder_factory.h"
#include "api/test/video/function_video_encoder_factory.h"
#include "modules/video_coding/codecs/vp8/include/vp8.h"
#include "test/gtest.h"
namespace webrtc {
namespace test {
namespace {
std::unique_ptr<SimulcastTestFixture> CreateSpecificSimulcastTestFixture() {
std::unique_ptr<VideoEncoderFactory> encoder_factory =
std::make_unique<FunctionVideoEncoderFactory>(
[]() { return VP8Encoder::Create(); });
std::unique_ptr<VideoDecoderFactory> decoder_factory =
std::make_unique<FunctionVideoDecoderFactory>(
[]() { return VP8Decoder::Create(); });
return CreateSimulcastTestFixture(std::move(encoder_factory),
std::move(decoder_factory),
SdpVideoFormat("VP8"));
}
} // namespace
TEST(LibvpxVp8SimulcastTest, TestKeyFrameRequestsOnAllStreams) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestKeyFrameRequestsOnAllStreams();
}
TEST(LibvpxVp8SimulcastTest, TestPaddingAllStreams) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestPaddingAllStreams();
}
TEST(LibvpxVp8SimulcastTest, TestPaddingTwoStreams) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestPaddingTwoStreams();
}
TEST(LibvpxVp8SimulcastTest, TestPaddingTwoStreamsOneMaxedOut) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestPaddingTwoStreamsOneMaxedOut();
}
TEST(LibvpxVp8SimulcastTest, TestPaddingOneStream) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestPaddingOneStream();
}
TEST(LibvpxVp8SimulcastTest, TestPaddingOneStreamTwoMaxedOut) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestPaddingOneStreamTwoMaxedOut();
}
TEST(LibvpxVp8SimulcastTest, TestSendAllStreams) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestSendAllStreams();
}
TEST(LibvpxVp8SimulcastTest, TestDisablingStreams) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestDisablingStreams();
}
TEST(LibvpxVp8SimulcastTest, TestActiveStreams) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestActiveStreams();
}
TEST(LibvpxVp8SimulcastTest, TestSwitchingToOneStream) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestSwitchingToOneStream();
}
TEST(LibvpxVp8SimulcastTest, TestSwitchingToOneOddStream) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestSwitchingToOneOddStream();
}
TEST(LibvpxVp8SimulcastTest, TestSwitchingToOneSmallStream) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestSwitchingToOneSmallStream();
}
TEST(LibvpxVp8SimulcastTest, TestSpatioTemporalLayers333PatternEncoder) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestSpatioTemporalLayers333PatternEncoder();
}
TEST(LibvpxVp8SimulcastTest, TestStrideEncodeDecode) {
auto fixture = CreateSpecificSimulcastTestFixture();
fixture->TestStrideEncodeDecode();
}
} // namespace test
} // namespace webrtc

625
modules/video_coding/codecs/vp8/screenshare_layers.cc Normal file
View File

@ -0,0 +1,625 @@
/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/vp8/screenshare_layers.h"
#include <stdlib.h>
#include <algorithm>
#include <memory>
#include "modules/video_coding/include/video_codec_interface.h"
#include "rtc_base/arraysize.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/time_utils.h"
#include "system_wrappers/include/metrics.h"
namespace webrtc {
namespace {
using BufferFlags = Vp8FrameConfig::BufferFlags;
constexpr BufferFlags kNone = Vp8FrameConfig::BufferFlags::kNone;
constexpr BufferFlags kReference = Vp8FrameConfig::BufferFlags::kReference;
constexpr BufferFlags kUpdate = Vp8FrameConfig::BufferFlags::kUpdate;
constexpr BufferFlags kReferenceAndUpdate =
Vp8FrameConfig::BufferFlags::kReferenceAndUpdate;
constexpr int kOneSecond90Khz = 90000;
constexpr int kMinTimeBetweenSyncs = kOneSecond90Khz * 2;
constexpr int kMaxTimeBetweenSyncs = kOneSecond90Khz * 4;
constexpr int kQpDeltaThresholdForSync = 8;
constexpr int kMinBitrateKbpsForQpBoost = 500;
} // namespace
const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5;
const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0;
constexpr int ScreenshareLayers::kMaxNumTemporalLayers;
// Always emit a frame with certain interval, even if bitrate targets have
// been exceeded. This prevents needless keyframe requests.
const int ScreenshareLayers::kMaxFrameIntervalMs = 2750;
ScreenshareLayers::ScreenshareLayers(int num_temporal_layers)
: number_of_temporal_layers_(
std::min(kMaxNumTemporalLayers, num_temporal_layers)),
active_layer_(-1),
last_timestamp_(-1),
last_sync_timestamp_(-1),
last_emitted_tl0_timestamp_(-1),
last_frame_time_ms_(-1),
max_debt_bytes_(0),
encode_framerate_(1000.0f, 1000.0f), // 1 second window, second scale.
bitrate_updated_(false),
checker_(TemporalLayersChecker::CreateTemporalLayersChecker(
Vp8TemporalLayersType::kBitrateDynamic,
num_temporal_layers)) {
RTC_CHECK_GT(number_of_temporal_layers_, 0);
RTC_CHECK_LE(number_of_temporal_layers_, kMaxNumTemporalLayers);
}
ScreenshareLayers::~ScreenshareLayers() {
UpdateHistograms();
}
void ScreenshareLayers::SetQpLimits(size_t stream_index,
int min_qp,
int max_qp) {
RTC_DCHECK_LT(stream_index, StreamCount());
// 0 < min_qp <= max_qp
RTC_DCHECK_LT(0, min_qp);
RTC_DCHECK_LE(min_qp, max_qp);
RTC_DCHECK_EQ(min_qp_.has_value(), max_qp_.has_value());
if (!min_qp_.has_value()) {
min_qp_ = min_qp;
max_qp_ = max_qp;
} else {
RTC_DCHECK_EQ(min_qp, min_qp_.value());
RTC_DCHECK_EQ(max_qp, max_qp_.value());
}
}
size_t ScreenshareLayers::StreamCount() const {
return 1;
}
bool ScreenshareLayers::SupportsEncoderFrameDropping(
size_t stream_index) const {
RTC_DCHECK_LT(stream_index, StreamCount());
// Frame dropping is handled internally by this class.
return false;
}
Vp8FrameConfig ScreenshareLayers::NextFrameConfig(size_t stream_index,
uint32_t timestamp) {
RTC_DCHECK_LT(stream_index, StreamCount());
auto it = pending_frame_configs_.find(timestamp);
if (it != pending_frame_configs_.end()) {
// Drop and re-encode, reuse the previous config.
return it->second.frame_config;
}
if (number_of_temporal_layers_ <= 1) {
// No flags needed for 1 layer screenshare.
// TODO(pbos): Consider updating only last, and not all buffers.
DependencyInfo dependency_info{
"S", {kReferenceAndUpdate, kReferenceAndUpdate, kReferenceAndUpdate}};
pending_frame_configs_[timestamp] = dependency_info;
return dependency_info.frame_config;
}
const int64_t now_ms = rtc::TimeMillis();
int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
int64_t ts_diff;
if (last_timestamp_ == -1) {
ts_diff = kOneSecond90Khz / capture_framerate_.value_or(*target_framerate_);
} else {
ts_diff = unwrapped_timestamp - last_timestamp_;
}
if (target_framerate_) {
// If input frame rate exceeds target frame rate, either over a one second
// averaging window, or if frame interval is below 90% of desired value,
// drop frame.
if (encode_framerate_.Rate(now_ms).value_or(0) > *target_framerate_)
return Vp8FrameConfig(kNone, kNone, kNone);
// Primarily check if frame interval is too short using frame timestamps,
// as if they are correct they won't be affected by queuing in webrtc.
const int64_t expected_frame_interval_90khz =
kOneSecond90Khz / *target_framerate_;
if (last_timestamp_ != -1 && ts_diff > 0) {
if (ts_diff < 85 * expected_frame_interval_90khz / 100) {
return Vp8FrameConfig(kNone, kNone, kNone);
}
} else {
// Timestamps looks off, use realtime clock here instead.
const int64_t expected_frame_interval_ms = 1000 / *target_framerate_;
if (last_frame_time_ms_ != -1 &&
now_ms - last_frame_time_ms_ <
(85 * expected_frame_interval_ms) / 100) {
return Vp8FrameConfig(kNone, kNone, kNone);
}
}
}
if (stats_.first_frame_time_ms_ == -1)
stats_.first_frame_time_ms_ = now_ms;
// Make sure both frame droppers leak out bits.
layers_[0].UpdateDebt(ts_diff / 90);
layers_[1].UpdateDebt(ts_diff / 90);
last_timestamp_ = timestamp;
last_frame_time_ms_ = now_ms;
TemporalLayerState layer_state = TemporalLayerState::kDrop;
if (active_layer_ == -1 ||
layers_[active_layer_].state != TemporalLayer::State::kDropped) {
if (last_emitted_tl0_timestamp_ != -1 &&
(unwrapped_timestamp - last_emitted_tl0_timestamp_) / 90 >
kMaxFrameIntervalMs) {
// Too long time has passed since the last frame was emitted, cancel
// enough debt to allow a single frame.
layers_[0].debt_bytes_ = max_debt_bytes_ - 1;
}
if (layers_[0].debt_bytes_ > max_debt_bytes_) {
// Must drop TL0, encode TL1 instead.
if (layers_[1].debt_bytes_ > max_debt_bytes_) {
// Must drop both TL0 and TL1.
active_layer_ = -1;
} else {
active_layer_ = 1;
}
} else {
active_layer_ = 0;
}
}
switch (active_layer_) {
case 0:
layer_state = TemporalLayerState::kTl0;
last_emitted_tl0_timestamp_ = unwrapped_timestamp;
break;
case 1:
if (layers_[1].state != TemporalLayer::State::kDropped) {
if (TimeToSync(unwrapped_timestamp) ||
layers_[1].state == TemporalLayer::State::kKeyFrame) {
last_sync_timestamp_ = unwrapped_timestamp;
layer_state = TemporalLayerState::kTl1Sync;
} else {
layer_state = TemporalLayerState::kTl1;
}
} else {
layer_state = last_sync_timestamp_ == unwrapped_timestamp
? TemporalLayerState::kTl1Sync
: TemporalLayerState::kTl1;
}
break;
case -1:
layer_state = TemporalLayerState::kDrop;
++stats_.num_dropped_frames_;
break;
default:
RTC_NOTREACHED();
}
DependencyInfo dependency_info;
// TODO(pbos): Consider referencing but not updating the 'alt' buffer for all
// layers.
switch (layer_state) {
case TemporalLayerState::kDrop:
dependency_info = {"", {kNone, kNone, kNone}};
break;
case TemporalLayerState::kTl0:
// TL0 only references and updates 'last'.
dependency_info = {"SS", {kReferenceAndUpdate, kNone, kNone}};
dependency_info.frame_config.packetizer_temporal_idx = 0;
break;
case TemporalLayerState::kTl1:
// TL1 references both 'last' and 'golden' but only updates 'golden'.
dependency_info = {"-R", {kReference, kReferenceAndUpdate, kNone}};
dependency_info.frame_config.packetizer_temporal_idx = 1;
break;
case TemporalLayerState::kTl1Sync:
// Predict from only TL0 to allow participants to switch to the high
// bitrate stream. Updates 'golden' so that TL1 can continue to refer to
// and update 'golden' from this point on.
dependency_info = {"-S", {kReference, kUpdate, kNone}};
dependency_info.frame_config.packetizer_temporal_idx = 1;
dependency_info.frame_config.layer_sync = true;
break;
}
pending_frame_configs_[timestamp] = dependency_info;
return dependency_info.frame_config;
}
void ScreenshareLayers::OnRatesUpdated(
size_t stream_index,
const std::vector<uint32_t>& bitrates_bps,
int framerate_fps) {
RTC_DCHECK_LT(stream_index, StreamCount());
RTC_DCHECK_GT(framerate_fps, 0);
RTC_DCHECK_GE(bitrates_bps.size(), 1);
RTC_DCHECK_LE(bitrates_bps.size(), 2);
// |bitrates_bps| uses individual rates per layer, but we want to use the
// accumulated rate here.
uint32_t tl0_kbps = bitrates_bps[0] / 1000;
uint32_t tl1_kbps = tl0_kbps;
if (bitrates_bps.size() > 1) {
tl1_kbps += bitrates_bps[1] / 1000;
}
if (!target_framerate_) {
// First OnRatesUpdated() is called during construction, with the
// configured targets as parameters.
target_framerate_ = framerate_fps;
capture_framerate_ = target_framerate_;
bitrate_updated_ = true;
} else {
if ((capture_framerate_ &&
framerate_fps != static_cast<int>(*capture_framerate_)) ||
(tl0_kbps != layers_[0].target_rate_kbps_) ||
(tl1_kbps != layers_[1].target_rate_kbps_)) {
bitrate_updated_ = true;
}
if (framerate_fps < 0) {
capture_framerate_.reset();
} else {
capture_framerate_ = framerate_fps;
}
}
layers_[0].target_rate_kbps_ = tl0_kbps;
layers_[1].target_rate_kbps_ = tl1_kbps;
}
void ScreenshareLayers::OnEncodeDone(size_t stream_index,
uint32_t rtp_timestamp,
size_t size_bytes,
bool is_keyframe,
int qp,
CodecSpecificInfo* info) {
RTC_DCHECK_LT(stream_index, StreamCount());
if (size_bytes == 0) {
RTC_LOG(LS_WARNING) << "Empty frame; treating as dropped.";
OnFrameDropped(stream_index, rtp_timestamp);
return;
}
absl::optional<DependencyInfo> dependency_info;
auto it = pending_frame_configs_.find(rtp_timestamp);
if (it != pending_frame_configs_.end()) {
dependency_info = it->second;
pending_frame_configs_.erase(it);
if (checker_) {
RTC_DCHECK(checker_->CheckTemporalConfig(is_keyframe,
dependency_info->frame_config));
}
}
CodecSpecificInfoVP8& vp8_info = info->codecSpecific.VP8;
GenericFrameInfo& generic_frame_info = info->generic_frame_info.emplace();
if (number_of_temporal_layers_ == 1) {
vp8_info.temporalIdx = kNoTemporalIdx;
vp8_info.layerSync = false;
generic_frame_info.decode_target_indications =
GenericFrameInfo::DecodeTargetInfo("S");
generic_frame_info.encoder_buffers.emplace_back(
0, /*referenced=*/!is_keyframe, /*updated=*/true);
} else {
int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(rtp_timestamp);
if (dependency_info) {
vp8_info.temporalIdx =
dependency_info->frame_config.packetizer_temporal_idx;
vp8_info.layerSync = dependency_info->frame_config.layer_sync;
generic_frame_info.decode_target_indications =
dependency_info->decode_target_indications;
} else {
RTC_DCHECK(is_keyframe);
}
if (is_keyframe) {
vp8_info.temporalIdx = 0;
last_sync_timestamp_ = unwrapped_timestamp;
vp8_info.layerSync = true;
layers_[0].state = TemporalLayer::State::kKeyFrame;
layers_[1].state = TemporalLayer::State::kKeyFrame;
active_layer_ = 1;
info->template_structure =
GetTemplateStructure(number_of_temporal_layers_);
generic_frame_info.decode_target_indications =
GenericFrameInfo::DecodeTargetInfo("SS");
} else if (active_layer_ >= 0 && layers_[active_layer_].state ==
TemporalLayer::State::kKeyFrame) {
layers_[active_layer_].state = TemporalLayer::State::kNormal;
}
vp8_info.useExplicitDependencies = true;
RTC_DCHECK_EQ(vp8_info.referencedBuffersCount, 0u);
RTC_DCHECK_EQ(vp8_info.updatedBuffersCount, 0u);
// Note that |frame_config| is not derefernced if |is_keyframe|,
// meaning it's never dereferenced if the optional may be unset.
for (int i = 0; i < static_cast<int>(Vp8FrameConfig::Buffer::kCount); ++i) {
bool references = false;
bool updates = is_keyframe;
if (!is_keyframe && dependency_info->frame_config.References(
static_cast<Vp8FrameConfig::Buffer>(i))) {
RTC_DCHECK_LT(vp8_info.referencedBuffersCount,
arraysize(CodecSpecificInfoVP8::referencedBuffers));
references = true;
vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i;
}
if (is_keyframe || dependency_info->frame_config.Updates(
static_cast<Vp8FrameConfig::Buffer>(i))) {
RTC_DCHECK_LT(vp8_info.updatedBuffersCount,
arraysize(CodecSpecificInfoVP8::updatedBuffers));
updates = true;
vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i;
}
if (references || updates)
generic_frame_info.encoder_buffers.emplace_back(i, references, updates);
}
}
encode_framerate_.Update(1, rtc::TimeMillis());
if (number_of_temporal_layers_ == 1)
return;
RTC_DCHECK_NE(-1, active_layer_);
if (layers_[active_layer_].state == TemporalLayer::State::kDropped) {
layers_[active_layer_].state = TemporalLayer::State::kQualityBoost;
}
if (qp != -1)
layers_[active_layer_].last_qp = qp;
if (active_layer_ == 0) {
layers_[0].debt_bytes_ += size_bytes;
layers_[1].debt_bytes_ += size_bytes;
++stats_.num_tl0_frames_;
stats_.tl0_target_bitrate_sum_ += layers_[0].target_rate_kbps_;
stats_.tl0_qp_sum_ += qp;
} else if (active_layer_ == 1) {
layers_[1].debt_bytes_ += size_bytes;
++stats_.num_tl1_frames_;
stats_.tl1_target_bitrate_sum_ += layers_[1].target_rate_kbps_;
stats_.tl1_qp_sum_ += qp;
}
}
void ScreenshareLayers::OnFrameDropped(size_t stream_index,
uint32_t rtp_timestamp) {
layers_[active_layer_].state = TemporalLayer::State::kDropped;
++stats_.num_overshoots_;
}
void ScreenshareLayers::OnPacketLossRateUpdate(float packet_loss_rate) {}
void ScreenshareLayers::OnRttUpdate(int64_t rtt_ms) {}
void ScreenshareLayers::OnLossNotification(
const VideoEncoder::LossNotification& loss_notification) {}
FrameDependencyStructure ScreenshareLayers::GetTemplateStructure(
int num_layers) const {
RTC_CHECK_LT(num_layers, 3);
RTC_CHECK_GT(num_layers, 0);
FrameDependencyStructure template_structure;
template_structure.num_decode_targets = num_layers;
using Builder = GenericFrameInfo::Builder;
switch (num_layers) {
case 1: {
template_structure.templates = {
Builder().T(0).Dtis("S").Build(),
Builder().T(0).Dtis("S").Fdiffs({1}).Build(),
};
return template_structure;
}
case 2: {
template_structure.templates = {
Builder().T(0).Dtis("SS").Build(),
Builder().T(0).Dtis("SS").Fdiffs({1}).Build(),
Builder().T(1).Dtis("-S").Fdiffs({1}).Build(),
};
return template_structure;
}
default:
RTC_NOTREACHED();
// To make the compiler happy!
return template_structure;
}
}
bool ScreenshareLayers::TimeToSync(int64_t timestamp) const {
RTC_DCHECK_EQ(1, active_layer_);
RTC_DCHECK_NE(-1, layers_[0].last_qp);
if (layers_[1].last_qp == -1) {
// First frame in TL1 should only depend on TL0 since there are no
// previous frames in TL1.
return true;
}
RTC_DCHECK_NE(-1, last_sync_timestamp_);
int64_t timestamp_diff = timestamp - last_sync_timestamp_;
if (timestamp_diff > kMaxTimeBetweenSyncs) {
// After a certain time, force a sync frame.
return true;
} else if (timestamp_diff < kMinTimeBetweenSyncs) {
// If too soon from previous sync frame, don't issue a new one.
return false;
}
// Issue a sync frame if difference in quality between TL0 and TL1 isn't too
// large.
if (layers_[0].last_qp - layers_[1].last_qp < kQpDeltaThresholdForSync)
return true;
return false;
}
uint32_t ScreenshareLayers::GetCodecTargetBitrateKbps() const {
uint32_t target_bitrate_kbps = layers_[0].target_rate_kbps_;
if (number_of_temporal_layers_ > 1) {
// Calculate a codec target bitrate. This may be higher than TL0, gaining
// quality at the expense of frame rate at TL0. Constraints:
// - TL0 frame rate no less than framerate / kMaxTL0FpsReduction.
// - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate.
target_bitrate_kbps =
std::min(layers_[0].target_rate_kbps_ * kMaxTL0FpsReduction,
layers_[1].target_rate_kbps_ / kAcceptableTargetOvershoot);
}
return std::max(layers_[0].target_rate_kbps_, target_bitrate_kbps);
}
Vp8EncoderConfig ScreenshareLayers::UpdateConfiguration(size_t stream_index) {
RTC_DCHECK_LT(stream_index, StreamCount());
RTC_DCHECK(min_qp_.has_value());
RTC_DCHECK(max_qp_.has_value());
const uint32_t target_bitrate_kbps = GetCodecTargetBitrateKbps();
// TODO(sprang): We _really_ need to make an overhaul of this class. :(
// If we're dropping frames in order to meet a target framerate, adjust the
// bitrate assigned to the encoder so the total average bitrate is correct.
float encoder_config_bitrate_kbps = target_bitrate_kbps;
if (target_framerate_ && capture_framerate_ &&
*target_framerate_ < *capture_framerate_) {
encoder_config_bitrate_kbps *=
static_cast<float>(*capture_framerate_) / *target_framerate_;
}
if (bitrate_updated_ ||
encoder_config_.rc_target_bitrate !=
absl::make_optional(encoder_config_bitrate_kbps)) {
encoder_config_.rc_target_bitrate = encoder_config_bitrate_kbps;
// Don't reconfigure qp limits during quality boost frames.
if (active_layer_ == -1 ||
layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) {
const int min_qp = min_qp_.value();
const int max_qp = max_qp_.value();
// After a dropped frame, a frame with max qp will be encoded and the
// quality will then ramp up from there. To boost the speed of recovery,
// encode the next frame with lower max qp, if there is sufficient
// bandwidth to do so without causing excessive delay.
// TL0 is the most important to improve since the errors in this layer
// will propagate to TL1.
// Currently, reduce max qp by 20% for TL0 and 15% for TL1.
if (layers_[1].target_rate_kbps_ >= kMinBitrateKbpsForQpBoost) {
layers_[0].enhanced_max_qp = min_qp + (((max_qp - min_qp) * 80) / 100);
layers_[1].enhanced_max_qp = min_qp + (((max_qp - min_qp) * 85) / 100);
} else {
layers_[0].enhanced_max_qp = -1;
layers_[1].enhanced_max_qp = -1;
}
}
if (capture_framerate_) {
int avg_frame_size =
(target_bitrate_kbps * 1000) / (8 * *capture_framerate_);
// Allow max debt to be the size of a single optimal frame.
// TODO(sprang): Determine if this needs to be adjusted by some factor.
// (Lower values may cause more frame drops, higher may lead to queuing
// delays.)
max_debt_bytes_ = avg_frame_size;
}
bitrate_updated_ = false;
}
// Don't try to update boosts state if not active yet.
if (active_layer_ == -1)
return encoder_config_;
if (number_of_temporal_layers_ <= 1)
return encoder_config_;
// If layer is in the quality boost state (following a dropped frame), update
// the configuration with the adjusted (lower) qp and set the state back to
// normal.
unsigned int adjusted_max_qp = max_qp_.value(); // Set the normal max qp.
if (layers_[active_layer_].state == TemporalLayer::State::kQualityBoost) {
if (layers_[active_layer_].enhanced_max_qp != -1) {
// Bitrate is high enough for quality boost, update max qp.
adjusted_max_qp = layers_[active_layer_].enhanced_max_qp;
}
// Regardless of qp, reset the boost state for the next frame.
layers_[active_layer_].state = TemporalLayer::State::kNormal;
}
encoder_config_.rc_max_quantizer = adjusted_max_qp;
return encoder_config_;
}
void ScreenshareLayers::TemporalLayer::UpdateDebt(int64_t delta_ms) {
uint32_t debt_reduction_bytes = target_rate_kbps_ * delta_ms / 8;
if (debt_reduction_bytes >= debt_bytes_) {
debt_bytes_ = 0;
} else {
debt_bytes_ -= debt_reduction_bytes;
}
}
void ScreenshareLayers::UpdateHistograms() {
if (stats_.first_frame_time_ms_ == -1)
return;
int64_t duration_sec =
(rtc::TimeMillis() - stats_.first_frame_time_ms_ + 500) / 1000;
if (duration_sec >= metrics::kMinRunTimeInSeconds) {
RTC_HISTOGRAM_COUNTS_10000(
"WebRTC.Video.Screenshare.Layer0.FrameRate",
(stats_.num_tl0_frames_ + (duration_sec / 2)) / duration_sec);
RTC_HISTOGRAM_COUNTS_10000(
"WebRTC.Video.Screenshare.Layer1.FrameRate",
(stats_.num_tl1_frames_ + (duration_sec / 2)) / duration_sec);
int total_frames = stats_.num_tl0_frames_ + stats_.num_tl1_frames_;
RTC_HISTOGRAM_COUNTS_10000(
"WebRTC.Video.Screenshare.FramesPerDrop",
(stats_.num_dropped_frames_ == 0
? 0
: total_frames / stats_.num_dropped_frames_));
RTC_HISTOGRAM_COUNTS_10000(
"WebRTC.Video.Screenshare.FramesPerOvershoot",
(stats_.num_overshoots_ == 0 ? 0
: total_frames / stats_.num_overshoots_));
if (stats_.num_tl0_frames_ > 0) {
RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.Layer0.Qp",
stats_.tl0_qp_sum_ / stats_.num_tl0_frames_);
RTC_HISTOGRAM_COUNTS_10000(
"WebRTC.Video.Screenshare.Layer0.TargetBitrate",
stats_.tl0_target_bitrate_sum_ / stats_.num_tl0_frames_);
}
if (stats_.num_tl1_frames_ > 0) {
RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.Layer1.Qp",
stats_.tl1_qp_sum_ / stats_.num_tl1_frames_);
RTC_HISTOGRAM_COUNTS_10000(
"WebRTC.Video.Screenshare.Layer1.TargetBitrate",
stats_.tl1_target_bitrate_sum_ / stats_.num_tl1_frames_);
}
}
}
} // namespace webrtc

164
modules/video_coding/codecs/vp8/screenshare_layers.h Normal file
View File

@ -0,0 +1,164 @@
/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_
#define MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_
#include <map>
#include <memory>
#include <utility>
#include <vector>
#include "api/video_codecs/vp8_frame_config.h"
#include "api/video_codecs/vp8_temporal_layers.h"
#include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/utility/frame_dropper.h"
#include "rtc_base/rate_statistics.h"
#include "rtc_base/time_utils.h"
namespace webrtc {
struct CodecSpecificInfoVP8;
class Clock;
class ScreenshareLayers final : public Vp8FrameBufferController {
public:
static const double kMaxTL0FpsReduction;
static const double kAcceptableTargetOvershoot;
static const int kMaxFrameIntervalMs;
explicit ScreenshareLayers(int num_temporal_layers);
~ScreenshareLayers() override;
void SetQpLimits(size_t stream_index, int min_qp, int max_qp) override;
size_t StreamCount() const override;
bool SupportsEncoderFrameDropping(size_t stream_index) const override;
// Returns the recommended VP8 encode flags needed. May refresh the decoder
// and/or update the reference buffers.
Vp8FrameConfig NextFrameConfig(size_t stream_index,
uint32_t rtp_timestamp) override;
// New target bitrate, per temporal layer.
void OnRatesUpdated(size_t stream_index,
const std::vector<uint32_t>& bitrates_bps,
int framerate_fps) override;
Vp8EncoderConfig UpdateConfiguration(size_t stream_index) override;
void OnEncodeDone(size_t stream_index,
uint32_t rtp_timestamp,
size_t size_bytes,
bool is_keyframe,
int qp,
CodecSpecificInfo* info) override;
void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) override;
void OnPacketLossRateUpdate(float packet_loss_rate) override;
void OnRttUpdate(int64_t rtt_ms) override;
void OnLossNotification(
const VideoEncoder::LossNotification& loss_notification) override;
private:
enum class TemporalLayerState : int { kDrop, kTl0, kTl1, kTl1Sync };
struct DependencyInfo {
DependencyInfo() = default;
DependencyInfo(absl::string_view indication_symbols,
Vp8FrameConfig frame_config)
: decode_target_indications(
GenericFrameInfo::DecodeTargetInfo(indication_symbols)),
frame_config(frame_config) {}
absl::InlinedVector<DecodeTargetIndication, 10> decode_target_indications;
Vp8FrameConfig frame_config;
};
bool TimeToSync(int64_t timestamp) const;
uint32_t GetCodecTargetBitrateKbps() const;
const int number_of_temporal_layers_;
// TODO(eladalon/sprang): These should be made into const-int set in the ctor.
absl::optional<int> min_qp_;
absl::optional<int> max_qp_;
int active_layer_;
int64_t last_timestamp_;
int64_t last_sync_timestamp_;
int64_t last_emitted_tl0_timestamp_;
int64_t last_frame_time_ms_;
rtc::TimestampWrapAroundHandler time_wrap_handler_;
uint32_t max_debt_bytes_;
std::map<uint32_t, DependencyInfo> pending_frame_configs_;
// Configured max framerate.
absl::optional<uint32_t> target_framerate_;
// Incoming framerate from capturer.
absl::optional<uint32_t> capture_framerate_;
// Tracks what framerate we actually encode, and drops frames on overshoot.
RateStatistics encode_framerate_;
bool bitrate_updated_;
static constexpr int kMaxNumTemporalLayers = 2;
struct TemporalLayer {
TemporalLayer()
: state(State::kNormal),
enhanced_max_qp(-1),
last_qp(-1),
debt_bytes_(0),
target_rate_kbps_(0) {}
enum class State {
kNormal,
kDropped,
kReencoded,
kQualityBoost,
kKeyFrame
} state;
int enhanced_max_qp;
int last_qp;
uint32_t debt_bytes_;
uint32_t target_rate_kbps_;
void UpdateDebt(int64_t delta_ms);
} layers_[kMaxNumTemporalLayers];
void UpdateHistograms();
FrameDependencyStructure GetTemplateStructure(int num_layers) const;
// Data for histogram statistics.
struct Stats {
int64_t first_frame_time_ms_ = -1;
int64_t num_tl0_frames_ = 0;
int64_t num_tl1_frames_ = 0;
int64_t num_dropped_frames_ = 0;
int64_t num_overshoots_ = 0;
int64_t tl0_qp_sum_ = 0;
int64_t tl1_qp_sum_ = 0;
int64_t tl0_target_bitrate_sum_ = 0;
int64_t tl1_target_bitrate_sum_ = 0;
} stats_;
Vp8EncoderConfig encoder_config_;
// Optional utility used to verify reference validity.
std::unique_ptr<TemporalLayersChecker> checker_;
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_

784
modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc Normal file
View File

@ -0,0 +1,784 @@
/*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/vp8/screenshare_layers.h"
#include <stdlib.h>
#include <string.h>
#include <cstdint>
#include <memory>
#include <vector>
#include "api/video_codecs/vp8_frame_config.h"
#include "modules/video_coding/codecs/interface/common_constants.h"
#include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "rtc_base/checks.h"
#include "rtc_base/fake_clock.h"
#include "system_wrappers/include/metrics.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "vpx/vp8cx.h"
using ::testing::_;
using ::testing::ElementsAre;
using ::testing::NiceMock;
namespace webrtc {
namespace {
// 5 frames per second at 90 kHz.
const uint32_t kTimestampDelta5Fps = 90000 / 5;
const int kDefaultQp = 54;
const int kDefaultTl0BitrateKbps = 200;
const int kDefaultTl1BitrateKbps = 2000;
const int kFrameRate = 5;
const int kSyncPeriodSeconds = 2;
const int kMaxSyncPeriodSeconds = 4;
// Expected flags for corresponding temporal layers.
const int kTl0Flags = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF;
const int kTl1Flags =
VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST;
const int kTl1SyncFlags = VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF |
VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST;
const std::vector<uint32_t> kDefault2TlBitratesBps = {
kDefaultTl0BitrateKbps * 1000,
(kDefaultTl1BitrateKbps - kDefaultTl0BitrateKbps) * 1000};
} // namespace
class ScreenshareLayerTest : public ::testing::Test {
protected:
ScreenshareLayerTest()
: min_qp_(2),
max_qp_(kDefaultQp),
frame_size_(-1),
timestamp_(90),
config_updated_(false) {}
virtual ~ScreenshareLayerTest() {}
void SetUp() override {
layers_.reset(new ScreenshareLayers(2));
cfg_ = ConfigureBitrates();
}
int EncodeFrame(bool base_sync, CodecSpecificInfo* info = nullptr) {
CodecSpecificInfo ignored_info;
if (!info) {
info = &ignored_info;
}
int flags = ConfigureFrame(base_sync);
if (flags != -1)
layers_->OnEncodeDone(0, timestamp_, frame_size_, base_sync, kDefaultQp,
info);
return flags;
}
int ConfigureFrame(bool key_frame) {
tl_config_ = NextFrameConfig(0, timestamp_);
EXPECT_EQ(0, tl_config_.encoder_layer_id)
<< "ScreenshareLayers always encodes using the bitrate allocator for "
"layer 0, but may reference different buffers and packetize "
"differently.";
if (tl_config_.drop_frame) {
return -1;
}
const uint32_t prev_rc_target_bitrate = cfg_.rc_target_bitrate.value_or(-1);
const uint32_t prev_rc_max_quantizer = cfg_.rc_max_quantizer.value_or(-1);
cfg_ = layers_->UpdateConfiguration(0);
config_updated_ =
cfg_.temporal_layer_config.has_value() ||
(cfg_.rc_target_bitrate.has_value() &&
cfg_.rc_target_bitrate.value() != prev_rc_target_bitrate) ||
(cfg_.rc_max_quantizer.has_value() &&
cfg_.rc_max_quantizer.value() != prev_rc_max_quantizer) ||
cfg_.g_error_resilient.has_value();
int flags = LibvpxVp8Encoder::EncodeFlags(tl_config_);
EXPECT_NE(-1, frame_size_);
return flags;
}
Vp8FrameConfig NextFrameConfig(size_t stream_index, uint32_t timestamp) {
int64_t timestamp_ms = timestamp / 90;
clock_.AdvanceTime(TimeDelta::Millis(timestamp_ms - rtc::TimeMillis()));
return layers_->NextFrameConfig(stream_index, timestamp);
}
int FrameSizeForBitrate(int bitrate_kbps) {
return ((bitrate_kbps * 1000) / 8) / kFrameRate;
}
Vp8EncoderConfig ConfigureBitrates() {
layers_->SetQpLimits(0, min_qp_, max_qp_);
layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, kFrameRate);
const Vp8EncoderConfig vp8_cfg = layers_->UpdateConfiguration(0);
EXPECT_TRUE(vp8_cfg.rc_target_bitrate.has_value());
frame_size_ = FrameSizeForBitrate(vp8_cfg.rc_target_bitrate.value());
return vp8_cfg;
}
void WithQpLimits(int min_qp, int max_qp) {
min_qp_ = min_qp;
max_qp_ = max_qp;
}
// Runs a few initial frames and makes sure we have seen frames on both
// temporal layers, including sync and non-sync frames.
bool RunGracePeriod() {
bool got_tl0 = false;
bool got_tl1 = false;
bool got_tl1_sync = false;
for (int i = 0; i < 10; ++i) {
CodecSpecificInfo info;
EXPECT_NE(-1, EncodeFrame(false, &info));
timestamp_ += kTimestampDelta5Fps;
if (info.codecSpecific.VP8.temporalIdx == 0) {
got_tl0 = true;
} else if (info.codecSpecific.VP8.layerSync) {
got_tl1_sync = true;
} else {
got_tl1 = true;
}
if (got_tl0 && got_tl1 && got_tl1_sync)
return true;
}
return false;
}
// Adds frames until we get one in the specified temporal layer. The last
// FrameEncoded() call will be omitted and needs to be done by the caller.
// Returns the flags for the last frame.
int SkipUntilTl(int layer) {
return SkipUntilTlAndSync(layer, absl::nullopt);
}
// Same as SkipUntilTl, but also waits until the sync bit condition is met.
int SkipUntilTlAndSync(int layer, absl::optional<bool> sync) {
int flags = 0;
const int kMaxFramesToSkip =
1 + (sync.value_or(false) ? kMaxSyncPeriodSeconds : 1) * kFrameRate;
for (int i = 0; i < kMaxFramesToSkip; ++i) {
flags = ConfigureFrame(false);
if (tl_config_.packetizer_temporal_idx != layer ||
(sync && *sync != tl_config_.layer_sync)) {
if (flags != -1) {
// If flags do not request a frame drop, report some default values
// for frame size etc.
CodecSpecificInfo info;
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
&info);
}
timestamp_ += kTimestampDelta5Fps;
} else {
// Found frame from sought after layer.
return flags;
}
}
ADD_FAILURE() << "Did not get a frame of TL" << layer << " in time.";
return -1;
}
int min_qp_;
uint32_t max_qp_;
int frame_size_;
rtc::ScopedFakeClock clock_;
std::unique_ptr<ScreenshareLayers> layers_;
uint32_t timestamp_;
Vp8FrameConfig tl_config_;
Vp8EncoderConfig cfg_;
bool config_updated_;
CodecSpecificInfo* IgnoredCodecSpecificInfo() {
ignored_codec_specific_info_ = std::make_unique<CodecSpecificInfo>();
return ignored_codec_specific_info_.get();
}
private:
std::unique_ptr<CodecSpecificInfo> ignored_codec_specific_info_;
};
TEST_F(ScreenshareLayerTest, 1Layer) {
layers_.reset(new ScreenshareLayers(1));
ConfigureBitrates();
// One layer screenshare should not use the frame dropper as all frames will
// belong to the base layer.
const int kSingleLayerFlags = 0;
auto info = std::make_unique<CodecSpecificInfo>();
int flags = EncodeFrame(false, info.get());
timestamp_ += kTimestampDelta5Fps;
EXPECT_EQ(static_cast<uint8_t>(kNoTemporalIdx),
info->codecSpecific.VP8.temporalIdx);
EXPECT_FALSE(info->codecSpecific.VP8.layerSync);
info = std::make_unique<CodecSpecificInfo>();
flags = EncodeFrame(false, info.get());
EXPECT_EQ(kSingleLayerFlags, flags);
EXPECT_EQ(static_cast<uint8_t>(kNoTemporalIdx),
info->codecSpecific.VP8.temporalIdx);
EXPECT_FALSE(info->codecSpecific.VP8.layerSync);
}
TEST_F(ScreenshareLayerTest, 2LayersPeriodicSync) {
std::vector<int> sync_times;
const int kNumFrames = kSyncPeriodSeconds * kFrameRate * 2 - 1;
for (int i = 0; i < kNumFrames; ++i) {
CodecSpecificInfo info;
EncodeFrame(false, &info);
timestamp_ += kTimestampDelta5Fps;
if (info.codecSpecific.VP8.temporalIdx == 1 &&
info.codecSpecific.VP8.layerSync) {
sync_times.push_back(timestamp_);
}
}
ASSERT_EQ(2u, sync_times.size());
EXPECT_GE(sync_times[1] - sync_times[0], 90000 * kSyncPeriodSeconds);
}
TEST_F(ScreenshareLayerTest, 2LayersSyncAfterTimeout) {
std::vector<int> sync_times;
const int kNumFrames = kMaxSyncPeriodSeconds * kFrameRate * 2 - 1;
for (int i = 0; i < kNumFrames; ++i) {
CodecSpecificInfo info;
tl_config_ = NextFrameConfig(0, timestamp_);
cfg_ = layers_->UpdateConfiguration(0);
// Simulate TL1 being at least 8 qp steps better.
if (tl_config_.packetizer_temporal_idx == 0) {
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
&info);
} else {
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp - 8,
&info);
}
if (info.codecSpecific.VP8.temporalIdx == 1 &&
info.codecSpecific.VP8.layerSync)
sync_times.push_back(timestamp_);
timestamp_ += kTimestampDelta5Fps;
}
ASSERT_EQ(2u, sync_times.size());
EXPECT_GE(sync_times[1] - sync_times[0], 90000 * kMaxSyncPeriodSeconds);
}
TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) {
std::vector<int> sync_times;
const int kNumFrames = (kSyncPeriodSeconds +
((kMaxSyncPeriodSeconds - kSyncPeriodSeconds) / 2)) *
kFrameRate;
for (int i = 0; i < kNumFrames; ++i) {
CodecSpecificInfo info;
ConfigureFrame(false);
// Simulate TL1 being at least 8 qp steps better.
if (tl_config_.packetizer_temporal_idx == 0) {
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
&info);
} else {
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp - 8,
&info);
}
if (info.codecSpecific.VP8.temporalIdx == 1 &&
info.codecSpecific.VP8.layerSync)
sync_times.push_back(timestamp_);
timestamp_ += kTimestampDelta5Fps;
}
ASSERT_EQ(1u, sync_times.size());
bool bumped_tl0_quality = false;
for (int i = 0; i < 3; ++i) {
CodecSpecificInfo info;
int flags = ConfigureFrame(false);
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp - 8,
&info);
if (info.codecSpecific.VP8.temporalIdx == 0) {
// Bump TL0 to same quality as TL1.
bumped_tl0_quality = true;
} else {
if (bumped_tl0_quality) {
EXPECT_TRUE(info.codecSpecific.VP8.layerSync);
EXPECT_EQ(kTl1SyncFlags, flags);
return;
}
}
timestamp_ += kTimestampDelta5Fps;
}
ADD_FAILURE() << "No TL1 frame arrived within time limit.";
}
TEST_F(ScreenshareLayerTest, 2LayersToggling) {
EXPECT_TRUE(RunGracePeriod());
// Insert 50 frames. 2/5 should be TL0.
int tl0_frames = 0;
int tl1_frames = 0;
for (int i = 0; i < 50; ++i) {
CodecSpecificInfo info;
EncodeFrame(false, &info);
timestamp_ += kTimestampDelta5Fps;
switch (info.codecSpecific.VP8.temporalIdx) {
case 0:
++tl0_frames;
break;
case 1:
++tl1_frames;
break;
default:
abort();
}
}
EXPECT_EQ(20, tl0_frames);
EXPECT_EQ(30, tl1_frames);
}
TEST_F(ScreenshareLayerTest, AllFitsLayer0) {
frame_size_ = FrameSizeForBitrate(kDefaultTl0BitrateKbps);
// Insert 50 frames, small enough that all fits in TL0.
for (int i = 0; i < 50; ++i) {
CodecSpecificInfo info;
int flags = EncodeFrame(false, &info);
timestamp_ += kTimestampDelta5Fps;
EXPECT_EQ(kTl0Flags, flags);
EXPECT_EQ(0, info.codecSpecific.VP8.temporalIdx);
}
}
TEST_F(ScreenshareLayerTest, TooHighBitrate) {
frame_size_ = 2 * FrameSizeForBitrate(kDefaultTl1BitrateKbps);
// Insert 100 frames. Half should be dropped.
int tl0_frames = 0;
int tl1_frames = 0;
int dropped_frames = 0;
for (int i = 0; i < 100; ++i) {
CodecSpecificInfo info;
int flags = EncodeFrame(false, &info);
timestamp_ += kTimestampDelta5Fps;
if (flags == -1) {
++dropped_frames;
} else {
switch (info.codecSpecific.VP8.temporalIdx) {
case 0:
++tl0_frames;
break;
case 1:
++tl1_frames;
break;
default:
ADD_FAILURE() << "Unexpected temporal id";
}
}
}
EXPECT_NEAR(50, tl0_frames + tl1_frames, 1);
EXPECT_NEAR(50, dropped_frames, 1);
}
TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL0) {
const int kTl0_kbps = 100;
const int kTl1_kbps = 1000;
const std::vector<uint32_t> layer_rates = {kTl0_kbps * 1000,
(kTl1_kbps - kTl0_kbps) * 1000};
layers_->OnRatesUpdated(0, layer_rates, kFrameRate);
cfg_ = layers_->UpdateConfiguration(0);
EXPECT_EQ(static_cast<unsigned int>(
ScreenshareLayers::kMaxTL0FpsReduction * kTl0_kbps + 0.5),
cfg_.rc_target_bitrate);
}
TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL1) {
const int kTl0_kbps = 100;
const int kTl1_kbps = 450;
const std::vector<uint32_t> layer_rates = {kTl0_kbps * 1000,
(kTl1_kbps - kTl0_kbps) * 1000};
layers_->OnRatesUpdated(0, layer_rates, kFrameRate);
cfg_ = layers_->UpdateConfiguration(0);
EXPECT_EQ(static_cast<unsigned int>(
kTl1_kbps / ScreenshareLayers::kAcceptableTargetOvershoot),
cfg_.rc_target_bitrate);
}
TEST_F(ScreenshareLayerTest, TargetBitrateBelowTL0) {
const int kTl0_kbps = 100;
const std::vector<uint32_t> layer_rates = {kTl0_kbps * 1000};
layers_->OnRatesUpdated(0, layer_rates, kFrameRate);
cfg_ = layers_->UpdateConfiguration(0);
EXPECT_EQ(static_cast<uint32_t>(kTl0_kbps), cfg_.rc_target_bitrate);
}
TEST_F(ScreenshareLayerTest, EncoderDrop) {
EXPECT_TRUE(RunGracePeriod());
SkipUntilTl(0);
// Size 0 indicates dropped frame.
layers_->OnEncodeDone(0, timestamp_, 0, false, 0, IgnoredCodecSpecificInfo());
// Re-encode frame (so don't advance timestamp).
int flags = EncodeFrame(false);
timestamp_ += kTimestampDelta5Fps;
EXPECT_FALSE(config_updated_);
EXPECT_EQ(kTl0Flags, flags);
// Next frame should have boosted quality...
SkipUntilTl(0);
EXPECT_TRUE(config_updated_);
EXPECT_LT(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
IgnoredCodecSpecificInfo());
timestamp_ += kTimestampDelta5Fps;
// ...then back to standard setup.
SkipUntilTl(0);
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
IgnoredCodecSpecificInfo());
timestamp_ += kTimestampDelta5Fps;
EXPECT_EQ(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
// Next drop in TL1.
SkipUntilTl(1);
layers_->OnEncodeDone(0, timestamp_, 0, false, 0, IgnoredCodecSpecificInfo());
// Re-encode frame (so don't advance timestamp).
flags = EncodeFrame(false);
timestamp_ += kTimestampDelta5Fps;
EXPECT_FALSE(config_updated_);
EXPECT_EQ(kTl1Flags, flags);
// Next frame should have boosted QP.
SkipUntilTl(1);
EXPECT_TRUE(config_updated_);
EXPECT_LT(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
IgnoredCodecSpecificInfo());
timestamp_ += kTimestampDelta5Fps;
// ...and back to normal.
SkipUntilTl(1);
EXPECT_EQ(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
IgnoredCodecSpecificInfo());
timestamp_ += kTimestampDelta5Fps;
}
TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) {
const int kLowBitrateKbps = 50;
const int kLargeFrameSizeBytes = 100000;
const uint32_t kStartTimestamp = 1234;
const std::vector<uint32_t> layer_rates = {kLowBitrateKbps * 1000};
layers_->OnRatesUpdated(0, layer_rates, kFrameRate);
cfg_ = layers_->UpdateConfiguration(0);
EXPECT_EQ(kTl0Flags,
LibvpxVp8Encoder::EncodeFlags(NextFrameConfig(0, kStartTimestamp)));
layers_->OnEncodeDone(0, kStartTimestamp, kLargeFrameSizeBytes, false,
kDefaultQp, IgnoredCodecSpecificInfo());
const uint32_t kTwoSecondsLater =
kStartTimestamp + (ScreenshareLayers::kMaxFrameIntervalMs * 90);
// Sanity check, repayment time should exceed kMaxFrameIntervalMs.
ASSERT_GT(kStartTimestamp + 90 * (kLargeFrameSizeBytes * 8) / kLowBitrateKbps,
kStartTimestamp + (ScreenshareLayers::kMaxFrameIntervalMs * 90));
// Expect drop one frame interval before the two second timeout. If we try
// any later, the frame will be dropped anyway by the frame rate throttling
// logic.
EXPECT_TRUE(
NextFrameConfig(0, kTwoSecondsLater - kTimestampDelta5Fps).drop_frame);
// More than two seconds has passed since last frame, one should be emitted
// even if bitrate target is then exceeded.
EXPECT_EQ(kTl0Flags, LibvpxVp8Encoder::EncodeFlags(
NextFrameConfig(0, kTwoSecondsLater + 90)));
}
TEST_F(ScreenshareLayerTest, UpdatesHistograms) {
metrics::Reset();
bool trigger_drop = false;
bool dropped_frame = false;
bool overshoot = false;
const int kTl0Qp = 35;
const int kTl1Qp = 30;
for (int64_t timestamp = 0;
timestamp < kTimestampDelta5Fps * 5 * metrics::kMinRunTimeInSeconds;
timestamp += kTimestampDelta5Fps) {
tl_config_ = NextFrameConfig(0, timestamp);
if (tl_config_.drop_frame) {
dropped_frame = true;
continue;
}
int flags = LibvpxVp8Encoder::EncodeFlags(tl_config_);
if (flags != -1)
cfg_ = layers_->UpdateConfiguration(0);
if (timestamp >= kTimestampDelta5Fps * 5 && !overshoot && flags != -1) {
// Simulate one overshoot.
layers_->OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
overshoot = true;
}
if (flags == kTl0Flags) {
if (timestamp >= kTimestampDelta5Fps * 20 && !trigger_drop) {
// Simulate a too large frame, to cause frame drop.
layers_->OnEncodeDone(0, timestamp, frame_size_ * 10, false, kTl0Qp,
IgnoredCodecSpecificInfo());
trigger_drop = true;
} else {
layers_->OnEncodeDone(0, timestamp, frame_size_, false, kTl0Qp,
IgnoredCodecSpecificInfo());
}
} else if (flags == kTl1Flags || flags == kTl1SyncFlags) {
layers_->OnEncodeDone(0, timestamp, frame_size_, false, kTl1Qp,
IgnoredCodecSpecificInfo());
} else if (flags == -1) {
dropped_frame = true;
} else {
RTC_NOTREACHED() << "Unexpected flags";
}
clock_.AdvanceTime(TimeDelta::Millis(1000 / 5));
}
EXPECT_TRUE(overshoot);
EXPECT_TRUE(dropped_frame);
layers_.reset(); // Histograms are reported on destruction.
EXPECT_METRIC_EQ(
1, metrics::NumSamples("WebRTC.Video.Screenshare.Layer0.FrameRate"));
EXPECT_METRIC_EQ(
1, metrics::NumSamples("WebRTC.Video.Screenshare.Layer1.FrameRate"));
EXPECT_METRIC_EQ(
1, metrics::NumSamples("WebRTC.Video.Screenshare.FramesPerDrop"));
EXPECT_METRIC_EQ(
1, metrics::NumSamples("WebRTC.Video.Screenshare.FramesPerOvershoot"));
EXPECT_METRIC_EQ(1,
metrics::NumSamples("WebRTC.Video.Screenshare.Layer0.Qp"));
EXPECT_METRIC_EQ(1,
metrics::NumSamples("WebRTC.Video.Screenshare.Layer1.Qp"));
EXPECT_METRIC_EQ(
1, metrics::NumSamples("WebRTC.Video.Screenshare.Layer0.TargetBitrate"));
EXPECT_METRIC_EQ(
1, metrics::NumSamples("WebRTC.Video.Screenshare.Layer1.TargetBitrate"));
EXPECT_METRIC_GT(
metrics::MinSample("WebRTC.Video.Screenshare.Layer0.FrameRate"), 1);
EXPECT_METRIC_GT(
metrics::MinSample("WebRTC.Video.Screenshare.Layer1.FrameRate"), 1);
EXPECT_METRIC_GT(metrics::MinSample("WebRTC.Video.Screenshare.FramesPerDrop"),
1);
EXPECT_METRIC_GT(
metrics::MinSample("WebRTC.Video.Screenshare.FramesPerOvershoot"), 1);
EXPECT_METRIC_EQ(
1, metrics::NumEvents("WebRTC.Video.Screenshare.Layer0.Qp", kTl0Qp));
EXPECT_METRIC_EQ(
1, metrics::NumEvents("WebRTC.Video.Screenshare.Layer1.Qp", kTl1Qp));
EXPECT_METRIC_EQ(
1, metrics::NumEvents("WebRTC.Video.Screenshare.Layer0.TargetBitrate",
kDefaultTl0BitrateKbps));
EXPECT_METRIC_EQ(
1, metrics::NumEvents("WebRTC.Video.Screenshare.Layer1.TargetBitrate",
kDefaultTl1BitrateKbps));
}
TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) {
int64_t kTestSpanMs = 2000;
int64_t kFrameIntervalsMs = 1000 / kFrameRate;
uint32_t timestamp = 1234;
int num_input_frames = 0;
int num_discarded_frames = 0;
// Send at regular rate - no drops expected.
for (int64_t i = 0; i < kTestSpanMs; i += kFrameIntervalsMs) {
if (NextFrameConfig(0, timestamp).drop_frame) {
++num_discarded_frames;
} else {
size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8;
layers_->OnEncodeDone(0, timestamp, frame_size_bytes, false, kDefaultQp,
IgnoredCodecSpecificInfo());
}
timestamp += kFrameIntervalsMs * 90;
clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalsMs));
++num_input_frames;
}
EXPECT_EQ(0, num_discarded_frames);
// Send at twice the configured rate - drop every other frame.
num_input_frames = 0;
num_discarded_frames = 0;
for (int64_t i = 0; i < kTestSpanMs; i += kFrameIntervalsMs / 2) {
if (NextFrameConfig(0, timestamp).drop_frame) {
++num_discarded_frames;
} else {
size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8;
layers_->OnEncodeDone(0, timestamp, frame_size_bytes, false, kDefaultQp,
IgnoredCodecSpecificInfo());
}
timestamp += kFrameIntervalsMs * 90 / 2;
clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalsMs));
++num_input_frames;
}
// Allow for some rounding errors in the measurements.
EXPECT_NEAR(num_discarded_frames, num_input_frames / 2, 2);
}
TEST_F(ScreenshareLayerTest, 2LayersSyncAtOvershootDrop) {
// Run grace period so we have existing frames in both TL0 and Tl1.
EXPECT_TRUE(RunGracePeriod());
// Move ahead until we have a sync frame in TL1.
EXPECT_EQ(kTl1SyncFlags, SkipUntilTlAndSync(1, true));
ASSERT_TRUE(tl_config_.layer_sync);
// Simulate overshoot of this frame.
layers_->OnEncodeDone(0, timestamp_, 0, false, 0, nullptr);
cfg_ = layers_->UpdateConfiguration(0);
EXPECT_EQ(kTl1SyncFlags, LibvpxVp8Encoder::EncodeFlags(tl_config_));
CodecSpecificInfo new_info;
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
&new_info);
EXPECT_TRUE(new_info.codecSpecific.VP8.layerSync);
}
TEST_F(ScreenshareLayerTest, DropOnTooShortFrameInterval) {
// Run grace period so we have existing frames in both TL0 and Tl1.
EXPECT_TRUE(RunGracePeriod());
// Add a large gap, so there's plenty of room in the rate tracker.
timestamp_ += kTimestampDelta5Fps * 3;
EXPECT_FALSE(NextFrameConfig(0, timestamp_).drop_frame);
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
IgnoredCodecSpecificInfo());
// Frame interval below 90% if desired time is not allowed, try inserting
// frame just before this limit.
const int64_t kMinFrameInterval = (kTimestampDelta5Fps * 85) / 100;
timestamp_ += kMinFrameInterval - 90;
EXPECT_TRUE(NextFrameConfig(0, timestamp_).drop_frame);
// Try again at the limit, now it should pass.
timestamp_ += 90;
EXPECT_FALSE(NextFrameConfig(0, timestamp_).drop_frame);
}
TEST_F(ScreenshareLayerTest, AdjustsBitrateWhenDroppingFrames) {
const uint32_t kTimestampDelta10Fps = kTimestampDelta5Fps / 2;
const int kNumFrames = 30;
ASSERT_TRUE(cfg_.rc_target_bitrate.has_value());
const uint32_t default_bitrate = cfg_.rc_target_bitrate.value();
layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, 10);
int num_dropped_frames = 0;
for (int i = 0; i < kNumFrames; ++i) {
if (EncodeFrame(false) == -1)
++num_dropped_frames;
timestamp_ += kTimestampDelta10Fps;
}
cfg_ = layers_->UpdateConfiguration(0);
EXPECT_EQ(num_dropped_frames, kNumFrames / 2);
EXPECT_EQ(cfg_.rc_target_bitrate, default_bitrate * 2);
}
TEST_F(ScreenshareLayerTest, UpdatesConfigurationAfterRateChange) {
// Set inital rate again, no need to update configuration.
layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, kFrameRate);
cfg_ = layers_->UpdateConfiguration(0);
// Rate changed, now update config.
std::vector<uint32_t> bitrates = kDefault2TlBitratesBps;
bitrates[1] -= 100000;
layers_->OnRatesUpdated(0, bitrates, 5);
cfg_ = layers_->UpdateConfiguration(0);
// Changed rate, but then set changed rate again before trying to update
// configuration, update should still apply.
bitrates[1] -= 100000;
layers_->OnRatesUpdated(0, bitrates, 5);
layers_->OnRatesUpdated(0, bitrates, 5);
cfg_ = layers_->UpdateConfiguration(0);
}
TEST_F(ScreenshareLayerTest, MaxQpRestoredAfterDoubleDrop) {
// Run grace period so we have existing frames in both TL0 and Tl1.
EXPECT_TRUE(RunGracePeriod());
// Move ahead until we have a sync frame in TL1.
EXPECT_EQ(kTl1SyncFlags, SkipUntilTlAndSync(1, true));
ASSERT_TRUE(tl_config_.layer_sync);
// Simulate overshoot of this frame.
layers_->OnEncodeDone(0, timestamp_, 0, false, -1, nullptr);
// Simulate re-encoded frame.
layers_->OnEncodeDone(0, timestamp_, 1, false, max_qp_,
IgnoredCodecSpecificInfo());
// Next frame, expect boosted quality.
// Slightly alter bitrate between each frame.
std::vector<uint32_t> kDefault2TlBitratesBpsAlt = kDefault2TlBitratesBps;
kDefault2TlBitratesBpsAlt[1] += 4000;
layers_->OnRatesUpdated(0, kDefault2TlBitratesBpsAlt, kFrameRate);
EXPECT_EQ(kTl1Flags, SkipUntilTlAndSync(1, false));
EXPECT_TRUE(config_updated_);
EXPECT_LT(cfg_.rc_max_quantizer, max_qp_);
ASSERT_TRUE(cfg_.rc_max_quantizer.has_value());
const uint32_t adjusted_qp = cfg_.rc_max_quantizer.value();
// Simulate overshoot of this frame.
layers_->OnEncodeDone(0, timestamp_, 0, false, -1, nullptr);
// Simulate re-encoded frame.
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, max_qp_,
IgnoredCodecSpecificInfo());
// A third frame, expect boosted quality.
layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, kFrameRate);
EXPECT_EQ(kTl1Flags, SkipUntilTlAndSync(1, false));
EXPECT_TRUE(config_updated_);
EXPECT_LT(cfg_.rc_max_quantizer, max_qp_);
EXPECT_EQ(adjusted_qp, cfg_.rc_max_quantizer);
// Frame encoded.
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, max_qp_,
IgnoredCodecSpecificInfo());
// A fourth frame, max qp should be restored.
layers_->OnRatesUpdated(0, kDefault2TlBitratesBpsAlt, kFrameRate);
EXPECT_EQ(kTl1Flags, SkipUntilTlAndSync(1, false));
EXPECT_EQ(cfg_.rc_max_quantizer, max_qp_);
}
} // namespace webrtc

17
modules/video_coding/codecs/vp8/temporal_layers.h Normal file
View File

@ -0,0 +1,17 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_
#define MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_
// TODO(webrtc:9012) Remove this file when downstream projects have updated.
#include "api/video_codecs/vp8_temporal_layers.h"
#endif // MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_

145
modules/video_coding/codecs/vp8/temporal_layers_checker.cc Normal file
View File

@ -0,0 +1,145 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h"
#include <memory>
#include "modules/video_coding/codecs/interface/common_constants.h"
#include "modules/video_coding/codecs/vp8/default_temporal_layers.h"
#include "rtc_base/logging.h"
namespace webrtc {
std::unique_ptr<TemporalLayersChecker>
TemporalLayersChecker::CreateTemporalLayersChecker(Vp8TemporalLayersType type,
int num_temporal_layers) {
switch (type) {
case Vp8TemporalLayersType::kFixedPattern:
return std::make_unique<DefaultTemporalLayersChecker>(
num_temporal_layers);
case Vp8TemporalLayersType::kBitrateDynamic:
// Conference mode temporal layering for screen content in base stream.
return std::make_unique<TemporalLayersChecker>(num_temporal_layers);
}
}
TemporalLayersChecker::TemporalLayersChecker(int num_temporal_layers)
: num_temporal_layers_(num_temporal_layers),
sequence_number_(0),
last_sync_sequence_number_(0),
last_tl0_sequence_number_(0) {}
bool TemporalLayersChecker::CheckAndUpdateBufferState(
BufferState* state,
bool* need_sync,
bool frame_is_keyframe,
uint8_t temporal_layer,
Vp8FrameConfig::BufferFlags flags,
uint32_t sequence_number,
uint32_t* lowest_sequence_referenced) {
if (flags & Vp8FrameConfig::BufferFlags::kReference) {
if (state->temporal_layer > 0 && !state->is_keyframe) {
*need_sync = false;
}
if (!state->is_keyframe && !frame_is_keyframe &&
state->sequence_number < *lowest_sequence_referenced) {
*lowest_sequence_referenced = state->sequence_number;
}
if (!frame_is_keyframe && !state->is_keyframe &&
state->temporal_layer > temporal_layer) {
RTC_LOG(LS_ERROR) << "Frame is referencing higher temporal layer.";
return false;
}
}
if ((flags & Vp8FrameConfig::BufferFlags::kUpdate)) {
state->temporal_layer = temporal_layer;
state->sequence_number = sequence_number;
state->is_keyframe = frame_is_keyframe;
}
if (frame_is_keyframe)
state->is_keyframe = true;
return true;
}
bool TemporalLayersChecker::CheckTemporalConfig(
bool frame_is_keyframe,
const Vp8FrameConfig& frame_config) {
if (frame_config.drop_frame ||
frame_config.packetizer_temporal_idx == kNoTemporalIdx) {
return true;
}
++sequence_number_;
if (frame_config.packetizer_temporal_idx >= num_temporal_layers_ ||
(frame_config.packetizer_temporal_idx == kNoTemporalIdx &&
num_temporal_layers_ > 1)) {
RTC_LOG(LS_ERROR) << "Incorrect temporal layer set for frame: "
<< frame_config.packetizer_temporal_idx
<< " num_temporal_layers: " << num_temporal_layers_;
return false;
}
uint32_t lowest_sequence_referenced = sequence_number_;
bool need_sync = frame_config.packetizer_temporal_idx > 0 &&
frame_config.packetizer_temporal_idx != kNoTemporalIdx;
if (!CheckAndUpdateBufferState(
&last_, &need_sync, frame_is_keyframe,
frame_config.packetizer_temporal_idx, frame_config.last_buffer_flags,
sequence_number_, &lowest_sequence_referenced)) {
RTC_LOG(LS_ERROR) << "Error in the Last buffer";
return false;
}
if (!CheckAndUpdateBufferState(&golden_, &need_sync, frame_is_keyframe,
frame_config.packetizer_temporal_idx,
frame_config.golden_buffer_flags,
sequence_number_,
&lowest_sequence_referenced)) {
RTC_LOG(LS_ERROR) << "Error in the Golden buffer";
return false;
}
if (!CheckAndUpdateBufferState(
&arf_, &need_sync, frame_is_keyframe,
frame_config.packetizer_temporal_idx, frame_config.arf_buffer_flags,
sequence_number_, &lowest_sequence_referenced)) {
RTC_LOG(LS_ERROR) << "Error in the Arf buffer";
return false;
}
if (lowest_sequence_referenced < last_sync_sequence_number_ &&
!frame_is_keyframe) {
RTC_LOG(LS_ERROR) << "Reference past the last sync frame. Referenced "
<< lowest_sequence_referenced << ", but sync was at "
<< last_sync_sequence_number_;
return false;
}
if (frame_config.packetizer_temporal_idx == 0) {
last_tl0_sequence_number_ = sequence_number_;
}
if (frame_is_keyframe) {
last_sync_sequence_number_ = sequence_number_;
}
if (need_sync) {
last_sync_sequence_number_ = last_tl0_sequence_number_;
}
// Ignore sync flag on key-frames as it really doesn't matter.
if (need_sync != frame_config.layer_sync && !frame_is_keyframe) {
RTC_LOG(LS_ERROR) << "Sync bit is set incorrectly on a frame. Expected: "
<< need_sync << " Actual: " << frame_config.layer_sync;
return false;
}
return true;
}
} // namespace webrtc

94
modules/video_coding/codecs/vp8/test/mock_libvpx_interface.h Normal file
View File

@ -0,0 +1,94 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_TEST_MOCK_LIBVPX_INTERFACE_H_
#define MODULES_VIDEO_CODING_CODECS_VP8_TEST_MOCK_LIBVPX_INTERFACE_H_
#include "modules/video_coding/codecs/vp8/libvpx_interface.h"
#include "test/gmock.h"
#include "test/gtest.h"
namespace webrtc {
class MockLibvpxVp8Interface : public LibvpxInterface {
public:
MOCK_CONST_METHOD5(img_alloc,
vpx_image_t*(vpx_image_t*,
vpx_img_fmt_t,
unsigned int,
unsigned int,
unsigned int));
MOCK_CONST_METHOD6(img_wrap,
vpx_image_t*(vpx_image_t*,
vpx_img_fmt_t,
unsigned int,
unsigned int,
unsigned int,
unsigned char*));
MOCK_CONST_METHOD1(img_free, void(vpx_image_t* img));
MOCK_CONST_METHOD2(codec_enc_config_set,
vpx_codec_err_t(vpx_codec_ctx_t*,
const vpx_codec_enc_cfg_t*));
MOCK_CONST_METHOD3(codec_enc_config_default,
vpx_codec_err_t(vpx_codec_iface_t*,
vpx_codec_enc_cfg_t*,
unsigned int));
MOCK_CONST_METHOD4(codec_enc_init,
vpx_codec_err_t(vpx_codec_ctx_t*,
vpx_codec_iface_t*,
const vpx_codec_enc_cfg_t*,
vpx_codec_flags_t));
MOCK_CONST_METHOD6(codec_enc_init_multi,
vpx_codec_err_t(vpx_codec_ctx_t*,
vpx_codec_iface_t*,
vpx_codec_enc_cfg_t*,
int,
vpx_codec_flags_t,
vpx_rational_t*));
MOCK_CONST_METHOD1(codec_destroy, vpx_codec_err_t(vpx_codec_ctx_t*));
MOCK_CONST_METHOD3(codec_control,
vpx_codec_err_t(vpx_codec_ctx_t*,
vp8e_enc_control_id,
uint32_t));
MOCK_CONST_METHOD3(codec_control,
vpx_codec_err_t(vpx_codec_ctx_t*,
vp8e_enc_control_id,
int));
MOCK_CONST_METHOD3(codec_control,
vpx_codec_err_t(vpx_codec_ctx_t*,
vp8e_enc_control_id,
int*));
MOCK_CONST_METHOD3(codec_control,
vpx_codec_err_t(vpx_codec_ctx_t*,
vp8e_enc_control_id,
vpx_roi_map*));
MOCK_CONST_METHOD3(codec_control,
vpx_codec_err_t(vpx_codec_ctx_t*,
vp8e_enc_control_id,
vpx_active_map*));
MOCK_CONST_METHOD3(codec_control,
vpx_codec_err_t(vpx_codec_ctx_t*,
vp8e_enc_control_id,
vpx_scaling_mode*));
MOCK_CONST_METHOD6(codec_encode,
vpx_codec_err_t(vpx_codec_ctx_t*,
const vpx_image_t*,
vpx_codec_pts_t,
uint64_t,
vpx_enc_frame_flags_t,
uint64_t));
MOCK_CONST_METHOD2(codec_get_cx_data,
const vpx_codec_cx_pkt_t*(vpx_codec_ctx_t*,
vpx_codec_iter_t*));
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_VP8_TEST_MOCK_LIBVPX_INTERFACE_H_

719
modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc Normal file
View File

@ -0,0 +1,719 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <stdio.h>
#include <memory>
#include "api/test/create_frame_generator.h"
#include "api/test/frame_generator_interface.h"
#include "api/test/mock_video_decoder.h"
#include "api/test/mock_video_encoder.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/vp8_temporal_layers.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "common_video/test/utilities.h"
#include "modules/video_coding/codecs/test/video_codec_unittest.h"
#include "modules/video_coding/codecs/vp8/include/vp8.h"
#include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h"
#include "modules/video_coding/codecs/vp8/test/mock_libvpx_interface.h"
#include "modules/video_coding/utility/vp8_header_parser.h"
#include "rtc_base/time_utils.h"
#include "test/field_trial.h"
#include "test/video_codec_settings.h"
namespace webrtc {
using ::testing::_;
using ::testing::AllOf;
using ::testing::ElementsAreArray;
using ::testing::Field;
using ::testing::Invoke;
using ::testing::NiceMock;
using ::testing::Return;
using EncoderInfo = webrtc::VideoEncoder::EncoderInfo;
using FramerateFractions =
absl::InlinedVector<uint8_t, webrtc::kMaxTemporalStreams>;
namespace {
constexpr uint32_t kLegacyScreenshareTl0BitrateKbps = 200;
constexpr uint32_t kLegacyScreenshareTl1BitrateKbps = 1000;
constexpr uint32_t kInitialTimestampRtp = 123;
constexpr int64_t kTestNtpTimeMs = 456;
constexpr int64_t kInitialTimestampMs = 789;
constexpr int kNumCores = 1;
constexpr size_t kMaxPayloadSize = 1440;
constexpr int kWidth = 172;
constexpr int kHeight = 144;
constexpr float kFramerateFps = 30;
const VideoEncoder::Capabilities kCapabilities(false);
const VideoEncoder::Settings kSettings(kCapabilities,
kNumCores,
kMaxPayloadSize);
} // namespace
class TestVp8Impl : public VideoCodecUnitTest {
protected:
std::unique_ptr<VideoEncoder> CreateEncoder() override {
return VP8Encoder::Create();
}
std::unique_ptr<VideoDecoder> CreateDecoder() override {
return VP8Decoder::Create();
}
void ModifyCodecSettings(VideoCodec* codec_settings) override {
webrtc::test::CodecSettings(kVideoCodecVP8, codec_settings);
codec_settings->width = kWidth;
codec_settings->height = kHeight;
codec_settings->VP8()->denoisingOn = true;
codec_settings->VP8()->frameDroppingOn = false;
codec_settings->VP8()->automaticResizeOn = false;
codec_settings->VP8()->complexity = VideoCodecComplexity::kComplexityNormal;
}
void EncodeAndWaitForFrame(const VideoFrame& input_frame,
EncodedImage* encoded_frame,
CodecSpecificInfo* codec_specific_info,
bool keyframe = false) {
std::vector<VideoFrameType> frame_types;
if (keyframe) {
frame_types.emplace_back(VideoFrameType::kVideoFrameKey);
} else {
frame_types.emplace_back(VideoFrameType::kVideoFrameDelta);
}
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->Encode(input_frame, &frame_types));
ASSERT_TRUE(WaitForEncodedFrame(encoded_frame, codec_specific_info));
VerifyQpParser(*encoded_frame);
EXPECT_EQ(kVideoCodecVP8, codec_specific_info->codecType);
EXPECT_EQ(0, encoded_frame->SpatialIndex());
}
void EncodeAndExpectFrameWith(const VideoFrame& input_frame,
uint8_t temporal_idx,
bool keyframe = false) {
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
EncodeAndWaitForFrame(input_frame, &encoded_frame, &codec_specific_info,
keyframe);
EXPECT_EQ(temporal_idx, codec_specific_info.codecSpecific.VP8.temporalIdx);
}
void VerifyQpParser(const EncodedImage& encoded_frame) const {
int qp;
EXPECT_GT(encoded_frame.size(), 0u);
ASSERT_TRUE(vp8::GetQp(encoded_frame.data(), encoded_frame.size(), &qp));
EXPECT_EQ(encoded_frame.qp_, qp) << "Encoder QP != parsed bitstream QP.";
}
};
TEST_F(TestVp8Impl, ErrorResilienceDisabledForNoTemporalLayers) {
codec_settings_.simulcastStream[0].numberOfTemporalLayers = 1;
auto* const vpx = new NiceMock<MockLibvpxVp8Interface>();
LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
VP8Encoder::Settings());
EXPECT_CALL(*vpx,
codec_enc_init(
_, _, Field(&vpx_codec_enc_cfg_t::g_error_resilient, 0), _));
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder.InitEncode(&codec_settings_, kSettings));
}
TEST_F(TestVp8Impl, DefaultErrorResilienceEnabledForTemporalLayers) {
codec_settings_.simulcastStream[0].numberOfTemporalLayers = 2;
codec_settings_.VP8()->numberOfTemporalLayers = 2;
auto* const vpx = new NiceMock<MockLibvpxVp8Interface>();
LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
VP8Encoder::Settings());
EXPECT_CALL(*vpx,
codec_enc_init(_, _,
Field(&vpx_codec_enc_cfg_t::g_error_resilient,
VPX_ERROR_RESILIENT_DEFAULT),
_));
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder.InitEncode(&codec_settings_, kSettings));
}
TEST_F(TestVp8Impl,
PartitionErrorResilienceEnabledForTemporalLayersWithFieldTrial) {
test::ScopedFieldTrials field_trials(
"WebRTC-VP8-ForcePartitionResilience/Enabled/");
codec_settings_.simulcastStream[0].numberOfTemporalLayers = 2;
codec_settings_.VP8()->numberOfTemporalLayers = 2;
auto* const vpx = new NiceMock<MockLibvpxVp8Interface>();
LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
VP8Encoder::Settings());
EXPECT_CALL(*vpx,
codec_enc_init(_, _,
Field(&vpx_codec_enc_cfg_t::g_error_resilient,
VPX_ERROR_RESILIENT_PARTITIONS),
_));
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder.InitEncode(&codec_settings_, kSettings));
}
TEST_F(TestVp8Impl, SetRates) {
auto* const vpx = new NiceMock<MockLibvpxVp8Interface>();
LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
VP8Encoder::Settings());
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder.InitEncode(&codec_settings_,
VideoEncoder::Settings(kCapabilities, 1, 1000)));
const uint32_t kBitrateBps = 300000;
VideoBitrateAllocation bitrate_allocation;
bitrate_allocation.SetBitrate(0, 0, kBitrateBps);
EXPECT_CALL(
*vpx,
codec_enc_config_set(
_, AllOf(Field(&vpx_codec_enc_cfg_t::rc_target_bitrate,
kBitrateBps / 1000),
Field(&vpx_codec_enc_cfg_t::rc_undershoot_pct, 100u),
Field(&vpx_codec_enc_cfg_t::rc_overshoot_pct, 15u),
Field(&vpx_codec_enc_cfg_t::rc_buf_sz, 1000u),
Field(&vpx_codec_enc_cfg_t::rc_buf_optimal_sz, 600u),
Field(&vpx_codec_enc_cfg_t::rc_dropframe_thresh, 30u))))
.WillOnce(Return(VPX_CODEC_OK));
encoder.SetRates(VideoEncoder::RateControlParameters(
bitrate_allocation, static_cast<double>(codec_settings_.maxFramerate)));
}
TEST_F(TestVp8Impl, DynamicSetRates) {
test::ScopedFieldTrials field_trials(
"WebRTC-VideoRateControl/vp8_dynamic_rate:true/");
auto* const vpx = new NiceMock<MockLibvpxVp8Interface>();
LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
VP8Encoder::Settings());
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder.InitEncode(&codec_settings_,
VideoEncoder::Settings(kCapabilities, 1, 1000)));
const uint32_t kBitrateBps = 300000;
VideoEncoder::RateControlParameters rate_settings;
rate_settings.bitrate.SetBitrate(0, 0, kBitrateBps);
rate_settings.framerate_fps =
static_cast<double>(codec_settings_.maxFramerate);
// Set rates with no headroom.
rate_settings.bandwidth_allocation = DataRate::BitsPerSec(kBitrateBps);
EXPECT_CALL(
*vpx,
codec_enc_config_set(
_, AllOf(Field(&vpx_codec_enc_cfg_t::rc_target_bitrate,
kBitrateBps / 1000),
Field(&vpx_codec_enc_cfg_t::rc_undershoot_pct, 1000u),
Field(&vpx_codec_enc_cfg_t::rc_overshoot_pct, 0u),
Field(&vpx_codec_enc_cfg_t::rc_buf_sz, 100u),
Field(&vpx_codec_enc_cfg_t::rc_buf_optimal_sz, 30u),
Field(&vpx_codec_enc_cfg_t::rc_dropframe_thresh, 40u))))
.WillOnce(Return(VPX_CODEC_OK));
encoder.SetRates(rate_settings);
// Set rates with max headroom.
rate_settings.bandwidth_allocation = DataRate::BitsPerSec(kBitrateBps * 2);
EXPECT_CALL(
*vpx, codec_enc_config_set(
_, AllOf(Field(&vpx_codec_enc_cfg_t::rc_target_bitrate,
kBitrateBps / 1000),
Field(&vpx_codec_enc_cfg_t::rc_undershoot_pct, 100u),
Field(&vpx_codec_enc_cfg_t::rc_overshoot_pct, 15u),
Field(&vpx_codec_enc_cfg_t::rc_buf_sz, 1000u),
Field(&vpx_codec_enc_cfg_t::rc_buf_optimal_sz, 600u),
Field(&vpx_codec_enc_cfg_t::rc_dropframe_thresh, 5u))))
.WillOnce(Return(VPX_CODEC_OK));
encoder.SetRates(rate_settings);
// Set rates with headroom half way.
rate_settings.bandwidth_allocation =
DataRate::BitsPerSec((3 * kBitrateBps) / 2);
EXPECT_CALL(
*vpx,
codec_enc_config_set(
_, AllOf(Field(&vpx_codec_enc_cfg_t::rc_target_bitrate,
kBitrateBps / 1000),
Field(&vpx_codec_enc_cfg_t::rc_undershoot_pct, 550u),
Field(&vpx_codec_enc_cfg_t::rc_overshoot_pct, 8u),
Field(&vpx_codec_enc_cfg_t::rc_buf_sz, 550u),
Field(&vpx_codec_enc_cfg_t::rc_buf_optimal_sz, 315u),
Field(&vpx_codec_enc_cfg_t::rc_dropframe_thresh, 23u))))
.WillOnce(Return(VPX_CODEC_OK));
encoder.SetRates(rate_settings);
}
TEST_F(TestVp8Impl, EncodeFrameAndRelease) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kSettings));
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
EncodeAndWaitForFrame(NextInputFrame(), &encoded_frame, &codec_specific_info);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
EXPECT_EQ(WEBRTC_VIDEO_CODEC_UNINITIALIZED,
encoder_->Encode(NextInputFrame(), nullptr));
}
TEST_F(TestVp8Impl, InitDecode) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release());
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
decoder_->InitDecode(&codec_settings_, kNumCores));
}
TEST_F(TestVp8Impl, OnEncodedImageReportsInfo) {
VideoFrame input_frame = NextInputFrame();
input_frame.set_timestamp(kInitialTimestampRtp);
input_frame.set_timestamp_us(kInitialTimestampMs *
rtc::kNumMicrosecsPerMillisec);
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
EncodeAndWaitForFrame(input_frame, &encoded_frame, &codec_specific_info);
EXPECT_EQ(kInitialTimestampRtp, encoded_frame.Timestamp());
EXPECT_EQ(kWidth, static_cast<int>(encoded_frame._encodedWidth));
EXPECT_EQ(kHeight, static_cast<int>(encoded_frame._encodedHeight));
}
TEST_F(TestVp8Impl, DecodedQpEqualsEncodedQp) {
VideoFrame input_frame = NextInputFrame();
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
EncodeAndWaitForFrame(input_frame, &encoded_frame, &codec_specific_info);
// First frame should be a key frame.
encoded_frame._frameType = VideoFrameType::kVideoFrameKey;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, false, -1));
std::unique_ptr<VideoFrame> decoded_frame;
absl::optional<uint8_t> decoded_qp;
ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
ASSERT_TRUE(decoded_frame);
ASSERT_TRUE(decoded_qp);
EXPECT_GT(I420PSNR(&input_frame, decoded_frame.get()), 36);
EXPECT_EQ(encoded_frame.qp_, *decoded_qp);
}
TEST_F(TestVp8Impl, ChecksSimulcastSettings) {
codec_settings_.numberOfSimulcastStreams = 2;
// Resolutions are not in ascending order, temporal layers do not match.
codec_settings_.simulcastStream[0] = {kWidth, kHeight, kFramerateFps, 2,
4000, 3000, 2000, 80};
codec_settings_.simulcastStream[1] = {kWidth / 2, kHeight / 2, 30, 3,
4000, 3000, 2000, 80};
EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED,
encoder_->InitEncode(&codec_settings_, kSettings));
codec_settings_.numberOfSimulcastStreams = 3;
// Resolutions are not in ascending order.
codec_settings_.simulcastStream[0] = {
kWidth / 2, kHeight / 2, kFramerateFps, 1, 4000, 3000, 2000, 80};
codec_settings_.simulcastStream[1] = {
kWidth / 2 - 1, kHeight / 2 - 1, kFramerateFps, 1, 4000, 3000, 2000, 80};
codec_settings_.simulcastStream[2] = {kWidth, kHeight, 30, 1,
4000, 3000, 2000, 80};
EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED,
encoder_->InitEncode(&codec_settings_, kSettings));
// Resolutions are not in ascending order.
codec_settings_.simulcastStream[0] = {kWidth, kHeight, kFramerateFps, 1,
4000, 3000, 2000, 80};
codec_settings_.simulcastStream[1] = {kWidth, kHeight, kFramerateFps, 1,
4000, 3000, 2000, 80};
codec_settings_.simulcastStream[2] = {
kWidth - 1, kHeight - 1, kFramerateFps, 1, 4000, 3000, 2000, 80};
EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED,
encoder_->InitEncode(&codec_settings_, kSettings));
// Temporal layers do not match.
codec_settings_.simulcastStream[0] = {
kWidth / 4, kHeight / 4, kFramerateFps, 1, 4000, 3000, 2000, 80};
codec_settings_.simulcastStream[1] = {
kWidth / 2, kHeight / 2, kFramerateFps, 2, 4000, 3000, 2000, 80};
codec_settings_.simulcastStream[2] = {kWidth, kHeight, kFramerateFps, 3,
4000, 3000, 2000, 80};
EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED,
encoder_->InitEncode(&codec_settings_, kSettings));
// Resolutions do not match codec config.
codec_settings_.simulcastStream[0] = {
kWidth / 4 + 1, kHeight / 4 + 1, kFramerateFps, 1, 4000, 3000, 2000, 80};
codec_settings_.simulcastStream[1] = {
kWidth / 2 + 2, kHeight / 2 + 2, kFramerateFps, 1, 4000, 3000, 2000, 80};
codec_settings_.simulcastStream[2] = {
kWidth + 4, kHeight + 4, kFramerateFps, 1, 4000, 3000, 2000, 80};
EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED,
encoder_->InitEncode(&codec_settings_, kSettings));
// Everything fine: scaling by 2, top resolution matches video, temporal
// settings are the same for all layers.
codec_settings_.simulcastStream[0] = {
kWidth / 4, kHeight / 4, kFramerateFps, 1, 4000, 3000, 2000, 80};
codec_settings_.simulcastStream[1] = {
kWidth / 2, kHeight / 2, kFramerateFps, 1, 4000, 3000, 2000, 80};
codec_settings_.simulcastStream[2] = {kWidth, kHeight, kFramerateFps, 1,
4000, 3000, 2000, 80};
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kSettings));
// Everything fine: custom scaling, top resolution matches video, temporal
// settings are the same for all layers.
codec_settings_.simulcastStream[0] = {
kWidth / 4, kHeight / 4, kFramerateFps, 1, 4000, 3000, 2000, 80};
codec_settings_.simulcastStream[1] = {kWidth, kHeight, kFramerateFps, 1,
4000, 3000, 2000, 80};
codec_settings_.simulcastStream[2] = {kWidth, kHeight, kFramerateFps, 1,
4000, 3000, 2000, 80};
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kSettings));
}
#if defined(WEBRTC_ANDROID)
#define MAYBE_AlignedStrideEncodeDecode DISABLED_AlignedStrideEncodeDecode
#else
#define MAYBE_AlignedStrideEncodeDecode AlignedStrideEncodeDecode
#endif
TEST_F(TestVp8Impl, MAYBE_AlignedStrideEncodeDecode) {
VideoFrame input_frame = NextInputFrame();
input_frame.set_timestamp(kInitialTimestampRtp);
input_frame.set_timestamp_us(kInitialTimestampMs *
rtc::kNumMicrosecsPerMillisec);
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
EncodeAndWaitForFrame(input_frame, &encoded_frame, &codec_specific_info);
// First frame should be a key frame.
encoded_frame._frameType = VideoFrameType::kVideoFrameKey;
encoded_frame.ntp_time_ms_ = kTestNtpTimeMs;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, false, -1));
std::unique_ptr<VideoFrame> decoded_frame;
absl::optional<uint8_t> decoded_qp;
ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
ASSERT_TRUE(decoded_frame);
// Compute PSNR on all planes (faster than SSIM).
EXPECT_GT(I420PSNR(&input_frame, decoded_frame.get()), 36);
EXPECT_EQ(kInitialTimestampRtp, decoded_frame->timestamp());
}
#if defined(WEBRTC_ANDROID)
#define MAYBE_DecodeWithACompleteKeyFrame DISABLED_DecodeWithACompleteKeyFrame
#else
#define MAYBE_DecodeWithACompleteKeyFrame DecodeWithACompleteKeyFrame
#endif
TEST_F(TestVp8Impl, MAYBE_DecodeWithACompleteKeyFrame) {
VideoFrame input_frame = NextInputFrame();
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
EncodeAndWaitForFrame(input_frame, &encoded_frame, &codec_specific_info);
// Setting complete to false -> should return an error.
encoded_frame._completeFrame = false;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERROR,
decoder_->Decode(encoded_frame, false, -1));
// Setting complete back to true. Forcing a delta frame.
encoded_frame._frameType = VideoFrameType::kVideoFrameDelta;
encoded_frame._completeFrame = true;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERROR,
decoder_->Decode(encoded_frame, false, -1));
// Now setting a key frame.
encoded_frame._frameType = VideoFrameType::kVideoFrameKey;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, false, -1));
std::unique_ptr<VideoFrame> decoded_frame;
absl::optional<uint8_t> decoded_qp;
ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
ASSERT_TRUE(decoded_frame);
EXPECT_GT(I420PSNR(&input_frame, decoded_frame.get()), 36);
}
TEST_F(TestVp8Impl, EncoderWith2TemporalLayers) {
codec_settings_.VP8()->numberOfTemporalLayers = 2;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kSettings));
// Temporal layer 0.
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
EncodeAndWaitForFrame(NextInputFrame(), &encoded_frame, &codec_specific_info);
EXPECT_EQ(0, codec_specific_info.codecSpecific.VP8.temporalIdx);
// Temporal layer 1.
EncodeAndExpectFrameWith(NextInputFrame(), 1);
// Temporal layer 0.
EncodeAndExpectFrameWith(NextInputFrame(), 0);
// Temporal layer 1.
EncodeAndExpectFrameWith(NextInputFrame(), 1);
}
TEST_F(TestVp8Impl, ScalingDisabledIfAutomaticResizeOff) {
codec_settings_.VP8()->frameDroppingOn = true;
codec_settings_.VP8()->automaticResizeOn = false;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kSettings));
VideoEncoder::ScalingSettings settings =
encoder_->GetEncoderInfo().scaling_settings;
EXPECT_FALSE(settings.thresholds.has_value());
}
TEST_F(TestVp8Impl, ScalingEnabledIfAutomaticResizeOn) {
codec_settings_.VP8()->frameDroppingOn = true;
codec_settings_.VP8()->automaticResizeOn = true;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kSettings));
VideoEncoder::ScalingSettings settings =
encoder_->GetEncoderInfo().scaling_settings;
EXPECT_TRUE(settings.thresholds.has_value());
EXPECT_EQ(kDefaultMinPixelsPerFrame, settings.min_pixels_per_frame);
}
TEST_F(TestVp8Impl, DontDropKeyframes) {
// Set very high resolution to trigger overuse more easily.
const int kScreenWidth = 1920;
const int kScreenHeight = 1080;
codec_settings_.width = kScreenWidth;
codec_settings_.height = kScreenHeight;
// Screensharing has the internal frame dropper off, and instead per frame
// asks ScreenshareLayers to decide if it should be dropped or not.
codec_settings_.VP8()->frameDroppingOn = false;
codec_settings_.mode = VideoCodecMode::kScreensharing;
// ScreenshareLayers triggers on 2 temporal layers and 1000kbps max bitrate.
codec_settings_.VP8()->numberOfTemporalLayers = 2;
codec_settings_.maxBitrate = 1000;
// Reset the frame generator with large number of squares, leading to lots of
// details and high probability of overshoot.
input_frame_generator_ = test::CreateSquareFrameGenerator(
codec_settings_.width, codec_settings_.height,
test::FrameGeneratorInterface::OutputType::kI420,
/* num_squares = */ absl::optional<int>(300));
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kSettings));
VideoBitrateAllocation bitrate_allocation;
// Bitrate only enough for TL0.
bitrate_allocation.SetBitrate(0, 0, 200000);
encoder_->SetRates(
VideoEncoder::RateControlParameters(bitrate_allocation, 5.0));
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
EncodeAndWaitForFrame(NextInputFrame(), &encoded_frame, &codec_specific_info,
true);
EncodeAndExpectFrameWith(NextInputFrame(), 0, true);
EncodeAndExpectFrameWith(NextInputFrame(), 0, true);
EncodeAndExpectFrameWith(NextInputFrame(), 0, true);
}
TEST_F(TestVp8Impl, KeepsTimestampOnReencode) {
auto* const vpx = new NiceMock<MockLibvpxVp8Interface>();
LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
VP8Encoder::Settings());
// Settings needed to trigger ScreenshareLayers usage, which is required for
// overshoot-drop-reencode logic.
codec_settings_.maxBitrate = 1000;
codec_settings_.mode = VideoCodecMode::kScreensharing;
codec_settings_.VP8()->numberOfTemporalLayers = 2;
EXPECT_CALL(*vpx, img_wrap(_, _, _, _, _, _))
.WillOnce(Invoke([](vpx_image_t* img, vpx_img_fmt_t fmt, unsigned int d_w,
unsigned int d_h, unsigned int stride_align,
unsigned char* img_data) {
img->fmt = fmt;
img->d_w = d_w;
img->d_h = d_h;
img->img_data = img_data;
return img;
}));
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder.InitEncode(&codec_settings_,
VideoEncoder::Settings(kCapabilities, 1, 1000)));
MockEncodedImageCallback callback;
encoder.RegisterEncodeCompleteCallback(&callback);
// Simulate overshoot drop, re-encode: encode function will be called twice
// with the same parameters. codec_get_cx_data() will by default return no
// image data and be interpreted as drop.
EXPECT_CALL(*vpx, codec_encode(_, _, /* pts = */ 0, _, _, _))
.Times(2)
.WillRepeatedly(Return(vpx_codec_err_t::VPX_CODEC_OK));
auto delta_frame =
std::vector<VideoFrameType>{VideoFrameType::kVideoFrameDelta};
encoder.Encode(NextInputFrame(), &delta_frame);
}
TEST(LibvpxVp8EncoderTest, GetEncoderInfoReturnsStaticInformation) {
auto* const vpx = new NiceMock<MockLibvpxVp8Interface>();
LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
VP8Encoder::Settings());
const auto info = encoder.GetEncoderInfo();
EXPECT_FALSE(info.supports_native_handle);
EXPECT_FALSE(info.is_hardware_accelerated);
EXPECT_FALSE(info.has_internal_source);
EXPECT_TRUE(info.supports_simulcast);
EXPECT_EQ(info.implementation_name, "libvpx");
}
TEST(LibvpxVp8EncoderTest,
GetEncoderInfoReturnsEmptyResolutionBitrateLimitsByDefault) {
auto* const vpx = new NiceMock<MockLibvpxVp8Interface>();
LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
VP8Encoder::Settings());
const auto info = encoder.GetEncoderInfo();
EXPECT_TRUE(info.resolution_bitrate_limits.empty());
}
TEST(LibvpxVp8EncoderTest,
GetEncoderInfoReturnsResolutionBitrateLimitsAsConfigured) {
std::vector<VideoEncoder::ResolutionBitrateLimits> resolution_bitrate_limits =
{VideoEncoder::ResolutionBitrateLimits(/*frame_size_pixels=*/640 * 360,
/*min_start_bitrate_bps=*/300,
/*min_bitrate_bps=*/100,
/*max_bitrate_bps=*/1000),
VideoEncoder::ResolutionBitrateLimits(320 * 180, 100, 30, 500)};
VP8Encoder::Settings settings;
settings.resolution_bitrate_limits = resolution_bitrate_limits;
auto* const vpx = new NiceMock<MockLibvpxVp8Interface>();
LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
std::move(settings));
const auto info = encoder.GetEncoderInfo();
EXPECT_EQ(info.resolution_bitrate_limits, resolution_bitrate_limits);
}
TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationNoLayers) {
FramerateFractions expected_fps_allocation[kMaxSpatialLayers] = {
FramerateFractions(1, EncoderInfo::kMaxFramerateFraction)};
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationTwoTemporalLayers) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
codec_settings_.numberOfSimulcastStreams = 1;
codec_settings_.simulcastStream[0].active = true;
codec_settings_.simulcastStream[0].targetBitrate = 100;
codec_settings_.simulcastStream[0].maxBitrate = 100;
codec_settings_.simulcastStream[0].numberOfTemporalLayers = 2;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kSettings));
FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction);
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationThreeTemporalLayers) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
codec_settings_.numberOfSimulcastStreams = 1;
codec_settings_.simulcastStream[0].active = true;
codec_settings_.simulcastStream[0].targetBitrate = 100;
codec_settings_.simulcastStream[0].maxBitrate = 100;
codec_settings_.simulcastStream[0].numberOfTemporalLayers = 3;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kSettings));
FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 4);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction);
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationScreenshareLayers) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
codec_settings_.numberOfSimulcastStreams = 1;
codec_settings_.mode = VideoCodecMode::kScreensharing;
codec_settings_.simulcastStream[0].active = true;
codec_settings_.simulcastStream[0].minBitrate = 30;
codec_settings_.simulcastStream[0].targetBitrate =
kLegacyScreenshareTl0BitrateKbps;
codec_settings_.simulcastStream[0].maxBitrate =
kLegacyScreenshareTl1BitrateKbps;
codec_settings_.simulcastStream[0].numberOfTemporalLayers = 2;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kSettings));
// Expect empty vector, since this mode doesn't have a fixed framerate.
FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationSimulcastVideo) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
// Set up three simulcast streams with three temporal layers each.
codec_settings_.numberOfSimulcastStreams = 3;
for (int i = 0; i < codec_settings_.numberOfSimulcastStreams; ++i) {
codec_settings_.simulcastStream[i].active = true;
codec_settings_.simulcastStream[i].minBitrate = 30;
codec_settings_.simulcastStream[i].targetBitrate = 30;
codec_settings_.simulcastStream[i].maxBitrate = 30;
codec_settings_.simulcastStream[i].numberOfTemporalLayers = 3;
codec_settings_.simulcastStream[i].width =
codec_settings_.width >>
(codec_settings_.numberOfSimulcastStreams - i - 1);
codec_settings_.simulcastStream[i].height =
codec_settings_.height >>
(codec_settings_.numberOfSimulcastStreams - i - 1);
}
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kSettings));
FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 4);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction);
expected_fps_allocation[1] = expected_fps_allocation[0];
expected_fps_allocation[2] = expected_fps_allocation[0];
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
// Release encoder and re-init without temporal layers.
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
// Sanity check fps allocation when not inited.
FramerateFractions default_fps_fraction[kMaxSpatialLayers];
default_fps_fraction[0].push_back(EncoderInfo::kMaxFramerateFraction);
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(default_fps_fraction));
for (int i = 0; i < codec_settings_.numberOfSimulcastStreams; ++i) {
codec_settings_.simulcastStream[i].numberOfTemporalLayers = 1;
}
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kSettings));
for (size_t i = 0; i < 3; ++i) {
expected_fps_allocation[i].clear();
expected_fps_allocation[i].push_back(EncoderInfo::kMaxFramerateFraction);
}
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
} // namespace webrtc

3
modules/video_coding/codecs/vp9/DEPS Normal file
View File

@ -0,0 +1,3 @@
include_rules = [
"+media/base",
]

47
modules/video_coding/codecs/vp9/include/vp9.h Normal file
View File

@ -0,0 +1,47 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#ifndef MODULES_VIDEO_CODING_CODECS_VP9_INCLUDE_VP9_H_
#define MODULES_VIDEO_CODING_CODECS_VP9_INCLUDE_VP9_H_
#include <memory>
#include <vector>
#include "api/video_codecs/sdp_video_format.h"
#include "media/base/codec.h"
#include "modules/video_coding/include/video_codec_interface.h"
namespace webrtc {
// Returns a vector with all supported internal VP9 profiles that we can
// negotiate in SDP, in order of preference.
std::vector<SdpVideoFormat> SupportedVP9Codecs();
class VP9Encoder : public VideoEncoder {
public:
// Deprecated. Returns default implementation using VP9 Profile 0.
// TODO(emircan): Remove once this is no longer used.
static std::unique_ptr<VP9Encoder> Create();
// Parses VP9 Profile from |codec| and returns the appropriate implementation.
static std::unique_ptr<VP9Encoder> Create(const cricket::VideoCodec& codec);
~VP9Encoder() override {}
};
class VP9Decoder : public VideoDecoder {
public:
static std::unique_ptr<VP9Decoder> Create();
~VP9Decoder() override {}
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_VP9_INCLUDE_VP9_H_

223
modules/video_coding/codecs/vp9/include/vp9_globals.h Normal file
View File

@ -0,0 +1,223 @@
/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
// This file contains codec dependent definitions that are needed in
// order to compile the WebRTC codebase, even if this codec is not used.
#ifndef MODULES_VIDEO_CODING_CODECS_VP9_INCLUDE_VP9_GLOBALS_H_
#define MODULES_VIDEO_CODING_CODECS_VP9_INCLUDE_VP9_GLOBALS_H_
#include <assert.h>
#include <stdint.h>
#include "modules/video_coding/codecs/interface/common_constants.h"
namespace webrtc {
const int16_t kMaxOneBytePictureId = 0x7F; // 7 bits
const int16_t kMaxTwoBytePictureId = 0x7FFF; // 15 bits
const uint8_t kNoSpatialIdx = 0xFF;
const uint8_t kNoGofIdx = 0xFF;
const uint8_t kNumVp9Buffers = 8;
const size_t kMaxVp9RefPics = 3;
const size_t kMaxVp9FramesInGof = 0xFF; // 8 bits
const size_t kMaxVp9NumberOfSpatialLayers = 8;
const size_t kMinVp9SpatialLayerWidth = 320;
const size_t kMinVp9SpatialLayerHeight = 180;
enum TemporalStructureMode {
kTemporalStructureMode1, // 1 temporal layer structure - i.e., IPPP...
kTemporalStructureMode2, // 2 temporal layers 01...
kTemporalStructureMode3, // 3 temporal layers 0212...
kTemporalStructureMode4 // 3 temporal layers 02120212...
};
struct GofInfoVP9 {
void SetGofInfoVP9(TemporalStructureMode tm) {
switch (tm) {
case kTemporalStructureMode1:
num_frames_in_gof = 1;
temporal_idx[0] = 0;
temporal_up_switch[0] = false;
num_ref_pics[0] = 1;
pid_diff[0][0] = 1;
break;
case kTemporalStructureMode2:
num_frames_in_gof = 2;
temporal_idx[0] = 0;
temporal_up_switch[0] = false;
num_ref_pics[0] = 1;
pid_diff[0][0] = 2;
temporal_idx[1] = 1;
temporal_up_switch[1] = true;
num_ref_pics[1] = 1;
pid_diff[1][0] = 1;
break;
case kTemporalStructureMode3:
num_frames_in_gof = 4;
temporal_idx[0] = 0;
temporal_up_switch[0] = false;
num_ref_pics[0] = 1;
pid_diff[0][0] = 4;
temporal_idx[1] = 2;
temporal_up_switch[1] = true;
num_ref_pics[1] = 1;
pid_diff[1][0] = 1;
temporal_idx[2] = 1;
temporal_up_switch[2] = true;
num_ref_pics[2] = 1;
pid_diff[2][0] = 2;
temporal_idx[3] = 2;
temporal_up_switch[3] = true;
num_ref_pics[3] = 1;
pid_diff[3][0] = 1;
break;
case kTemporalStructureMode4:
num_frames_in_gof = 8;
temporal_idx[0] = 0;
temporal_up_switch[0] = false;
num_ref_pics[0] = 1;
pid_diff[0][0] = 4;
temporal_idx[1] = 2;
temporal_up_switch[1] = true;
num_ref_pics[1] = 1;
pid_diff[1][0] = 1;
temporal_idx[2] = 1;
temporal_up_switch[2] = true;
num_ref_pics[2] = 1;
pid_diff[2][0] = 2;
temporal_idx[3] = 2;
temporal_up_switch[3] = false;
num_ref_pics[3] = 2;
pid_diff[3][0] = 1;
pid_diff[3][1] = 2;
temporal_idx[4] = 0;
temporal_up_switch[4] = false;
num_ref_pics[4] = 1;
pid_diff[4][0] = 4;
temporal_idx[5] = 2;
temporal_up_switch[5] = false;
num_ref_pics[5] = 2;
pid_diff[5][0] = 1;
pid_diff[5][1] = 2;
temporal_idx[6] = 1;
temporal_up_switch[6] = false;
num_ref_pics[6] = 2;
pid_diff[6][0] = 2;
pid_diff[6][1] = 4;
temporal_idx[7] = 2;
temporal_up_switch[7] = false;
num_ref_pics[7] = 2;
pid_diff[7][0] = 1;
pid_diff[7][1] = 2;
break;
default:
assert(false);
}
}
void CopyGofInfoVP9(const GofInfoVP9& src) {
num_frames_in_gof = src.num_frames_in_gof;
for (size_t i = 0; i < num_frames_in_gof; ++i) {
temporal_idx[i] = src.temporal_idx[i];
temporal_up_switch[i] = src.temporal_up_switch[i];
num_ref_pics[i] = src.num_ref_pics[i];
for (uint8_t r = 0; r < num_ref_pics[i]; ++r) {
pid_diff[i][r] = src.pid_diff[i][r];
}
}
}
size_t num_frames_in_gof;
uint8_t temporal_idx[kMaxVp9FramesInGof];
bool temporal_up_switch[kMaxVp9FramesInGof];
uint8_t num_ref_pics[kMaxVp9FramesInGof];
uint8_t pid_diff[kMaxVp9FramesInGof][kMaxVp9RefPics];
uint16_t pid_start;
};
struct RTPVideoHeaderVP9 {
void InitRTPVideoHeaderVP9() {
inter_pic_predicted = false;
flexible_mode = false;
beginning_of_frame = false;
end_of_frame = false;
ss_data_available = false;
non_ref_for_inter_layer_pred = false;
picture_id = kNoPictureId;
max_picture_id = kMaxTwoBytePictureId;
tl0_pic_idx = kNoTl0PicIdx;
temporal_idx = kNoTemporalIdx;
spatial_idx = kNoSpatialIdx;
temporal_up_switch = false;
inter_layer_predicted = false;
gof_idx = kNoGofIdx;
num_ref_pics = 0;
num_spatial_layers = 1;
first_active_layer = 0;
end_of_picture = true;
}
bool inter_pic_predicted; // This layer frame is dependent on previously
// coded frame(s).
bool flexible_mode; // This frame is in flexible mode.
bool beginning_of_frame; // True if this packet is the first in a VP9 layer
// frame.
bool end_of_frame; // True if this packet is the last in a VP9 layer frame.
bool ss_data_available; // True if SS data is available in this payload
// descriptor.
bool non_ref_for_inter_layer_pred; // True for frame which is not used as
// reference for inter-layer prediction.
int16_t picture_id; // PictureID index, 15 bits;
// kNoPictureId if PictureID does not exist.
int16_t max_picture_id; // Maximum picture ID index; either 0x7F or 0x7FFF;
int16_t tl0_pic_idx; // TL0PIC_IDX, 8 bits;
// kNoTl0PicIdx means no value provided.
uint8_t temporal_idx; // Temporal layer index, or kNoTemporalIdx.
uint8_t spatial_idx; // Spatial layer index, or kNoSpatialIdx.
bool temporal_up_switch; // True if upswitch to higher frame rate is possible
// starting from this frame.
bool inter_layer_predicted; // Frame is dependent on directly lower spatial
// layer frame.
uint8_t gof_idx; // Index to predefined temporal frame info in SS data.
uint8_t num_ref_pics; // Number of reference pictures used by this layer
// frame.
uint8_t pid_diff[kMaxVp9RefPics]; // P_DIFF signaled to derive the PictureID
// of the reference pictures.
int16_t ref_picture_id[kMaxVp9RefPics]; // PictureID of reference pictures.
// SS data.
size_t num_spatial_layers; // Always populated.
size_t first_active_layer; // Not sent on wire, used to adjust ss data.
bool spatial_layer_resolution_present;
uint16_t width[kMaxVp9NumberOfSpatialLayers];
uint16_t height[kMaxVp9NumberOfSpatialLayers];
GofInfoVP9 gof;
bool end_of_picture; // This frame is the last frame in picture.
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_VP9_INCLUDE_VP9_GLOBALS_H_

136
modules/video_coding/codecs/vp9/svc_config.cc Normal file
View File

@ -0,0 +1,136 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/vp9/svc_config.h"
#include <algorithm>
#include <cmath>
#include <vector>
#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
#include "rtc_base/checks.h"
namespace webrtc {
namespace {
const size_t kMinVp9SvcBitrateKbps = 30;
const size_t kMaxNumLayersForScreenSharing = 3;
const float kMaxScreenSharingLayerFramerateFps[] = {5.0, 10.0, 30.0};
const size_t kMinScreenSharingLayerBitrateKbps[] = {30, 200, 500};
const size_t kTargetScreenSharingLayerBitrateKbps[] = {150, 350, 950};
const size_t kMaxScreenSharingLayerBitrateKbps[] = {250, 500, 950};
} // namespace
std::vector<SpatialLayer> ConfigureSvcScreenSharing(size_t input_width,
size_t input_height,
float max_framerate_fps,
size_t num_spatial_layers) {
num_spatial_layers =
std::min(num_spatial_layers, kMaxNumLayersForScreenSharing);
std::vector<SpatialLayer> spatial_layers;
for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
SpatialLayer spatial_layer = {0};
spatial_layer.width = input_width;
spatial_layer.height = input_height;
spatial_layer.maxFramerate =
std::min(kMaxScreenSharingLayerFramerateFps[sl_idx], max_framerate_fps);
spatial_layer.numberOfTemporalLayers = 1;
spatial_layer.minBitrate =
static_cast<int>(kMinScreenSharingLayerBitrateKbps[sl_idx]);
spatial_layer.maxBitrate =
static_cast<int>(kMaxScreenSharingLayerBitrateKbps[sl_idx]);
spatial_layer.targetBitrate =
static_cast<int>(kTargetScreenSharingLayerBitrateKbps[sl_idx]);
spatial_layer.active = true;
spatial_layers.push_back(spatial_layer);
}
return spatial_layers;
}
std::vector<SpatialLayer> ConfigureSvcNormalVideo(size_t input_width,
size_t input_height,
float max_framerate_fps,
size_t first_active_layer,
size_t num_spatial_layers,
size_t num_temporal_layers) {
RTC_DCHECK_LT(first_active_layer, num_spatial_layers);
std::vector<SpatialLayer> spatial_layers;
// Limit number of layers for given resolution.
const size_t num_layers_fit_horz = static_cast<size_t>(std::floor(
1 + std::max(0.0f,
std::log2(1.0f * input_width / kMinVp9SpatialLayerWidth))));
const size_t num_layers_fit_vert = static_cast<size_t>(
std::floor(1 + std::max(0.0f, std::log2(1.0f * input_height /
kMinVp9SpatialLayerHeight))));
num_spatial_layers =
std::min({num_spatial_layers, num_layers_fit_horz, num_layers_fit_vert});
// First active layer must be configured.
num_spatial_layers = std::max(num_spatial_layers, first_active_layer + 1);
for (size_t sl_idx = first_active_layer; sl_idx < num_spatial_layers;
++sl_idx) {
SpatialLayer spatial_layer = {0};
spatial_layer.width = input_width >> (num_spatial_layers - sl_idx - 1);
spatial_layer.height = input_height >> (num_spatial_layers - sl_idx - 1);
spatial_layer.maxFramerate = max_framerate_fps;
spatial_layer.numberOfTemporalLayers = num_temporal_layers;
spatial_layer.active = true;
// minBitrate and maxBitrate formulas were derived from
// subjective-quality data to determing bit rates below which video
// quality is unacceptable and above which additional bits do not provide
// benefit. The formulas express rate in units of kbps.
// TODO(ssilkin): Add to the comment PSNR/SSIM we get at encoding certain
// video to min/max bitrate specified by those formulas.
const size_t num_pixels = spatial_layer.width * spatial_layer.height;
int min_bitrate =
static_cast<int>((600. * std::sqrt(num_pixels) - 95000.) / 1000.);
min_bitrate = std::max(min_bitrate, 0);
spatial_layer.minBitrate =
std::max(static_cast<size_t>(min_bitrate), kMinVp9SvcBitrateKbps);
spatial_layer.maxBitrate =
static_cast<int>((1.6 * num_pixels + 50 * 1000) / 1000);
spatial_layer.targetBitrate =
(spatial_layer.minBitrate + spatial_layer.maxBitrate) / 2;
spatial_layers.push_back(spatial_layer);
}
return spatial_layers;
}
std::vector<SpatialLayer> GetSvcConfig(size_t input_width,
size_t input_height,
float max_framerate_fps,
size_t first_active_layer,
size_t num_spatial_layers,
size_t num_temporal_layers,
bool is_screen_sharing) {
RTC_DCHECK_GT(input_width, 0);
RTC_DCHECK_GT(input_height, 0);
RTC_DCHECK_GT(num_spatial_layers, 0);
RTC_DCHECK_GT(num_temporal_layers, 0);
if (is_screen_sharing) {
return ConfigureSvcScreenSharing(input_width, input_height,
max_framerate_fps, num_spatial_layers);
} else {
return ConfigureSvcNormalVideo(input_width, input_height, max_framerate_fps,
first_active_layer, num_spatial_layers,
num_temporal_layers);
}
}
} // namespace webrtc

31
modules/video_coding/codecs/vp9/svc_config.h Normal file
View File

@ -0,0 +1,31 @@
/* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_VP9_SVC_CONFIG_H_
#define MODULES_VIDEO_CODING_CODECS_VP9_SVC_CONFIG_H_
#include <stddef.h>
#include <vector>
#include "common_types.h" // NOLINT(build/include)
namespace webrtc {
std::vector<SpatialLayer> GetSvcConfig(size_t input_width,
size_t input_height,
float max_framerate_fps,
size_t first_active_layer,
size_t num_spatial_layers,
size_t num_temporal_layers,
bool is_screen_sharing);
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_VP9_SVC_CONFIG_H_

91
modules/video_coding/codecs/vp9/svc_config_unittest.cc Normal file
View File

@ -0,0 +1,91 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/vp9/svc_config.h"
#include <cstddef>
#include <vector>
#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
#include "test/gtest.h"
namespace webrtc {
TEST(SvcConfig, NumSpatialLayers) {
const size_t max_num_spatial_layers = 6;
const size_t first_active_layer = 0;
const size_t num_spatial_layers = 2;
std::vector<SpatialLayer> spatial_layers =
GetSvcConfig(kMinVp9SpatialLayerWidth << (num_spatial_layers - 1),
kMinVp9SpatialLayerHeight << (num_spatial_layers - 1), 30,
first_active_layer, max_num_spatial_layers, 1, false);
EXPECT_EQ(spatial_layers.size(), num_spatial_layers);
}
TEST(SvcConfig, AlwaysSendsAtLeastOneLayer) {
const size_t max_num_spatial_layers = 6;
const size_t first_active_layer = 5;
std::vector<SpatialLayer> spatial_layers =
GetSvcConfig(kMinVp9SpatialLayerWidth, kMinVp9SpatialLayerHeight, 30,
first_active_layer, max_num_spatial_layers, 1, false);
EXPECT_EQ(spatial_layers.size(), 1u);
EXPECT_EQ(spatial_layers.back().width, kMinVp9SpatialLayerWidth);
}
TEST(SvcConfig, SkipsInactiveLayers) {
const size_t num_spatial_layers = 4;
const size_t first_active_layer = 2;
std::vector<SpatialLayer> spatial_layers =
GetSvcConfig(kMinVp9SpatialLayerWidth << (num_spatial_layers - 1),
kMinVp9SpatialLayerHeight << (num_spatial_layers - 1), 30,
first_active_layer, num_spatial_layers, 1, false);
EXPECT_EQ(spatial_layers.size(), 2u);
EXPECT_EQ(spatial_layers.back().width,
kMinVp9SpatialLayerWidth << (num_spatial_layers - 1));
}
TEST(SvcConfig, BitrateThresholds) {
const size_t first_active_layer = 0;
const size_t num_spatial_layers = 3;
std::vector<SpatialLayer> spatial_layers =
GetSvcConfig(kMinVp9SpatialLayerWidth << (num_spatial_layers - 1),
kMinVp9SpatialLayerHeight << (num_spatial_layers - 1), 30,
first_active_layer, num_spatial_layers, 1, false);
EXPECT_EQ(spatial_layers.size(), num_spatial_layers);
for (const SpatialLayer& layer : spatial_layers) {
EXPECT_LE(layer.minBitrate, layer.maxBitrate);
EXPECT_LE(layer.minBitrate, layer.targetBitrate);
EXPECT_LE(layer.targetBitrate, layer.maxBitrate);
}
}
TEST(SvcConfig, ScreenSharing) {
std::vector<SpatialLayer> spatial_layers =
GetSvcConfig(1920, 1080, 30, 1, 3, 3, true);
EXPECT_EQ(spatial_layers.size(), 3UL);
for (size_t i = 0; i < 3; ++i) {
const SpatialLayer& layer = spatial_layers[i];
EXPECT_EQ(layer.width, 1920);
EXPECT_EQ(layer.height, 1080);
EXPECT_EQ(layer.maxFramerate, (i < 1) ? 5 : (i < 2 ? 10 : 30));
EXPECT_EQ(layer.numberOfTemporalLayers, 1);
EXPECT_LE(layer.minBitrate, layer.maxBitrate);
EXPECT_LE(layer.minBitrate, layer.targetBitrate);
EXPECT_LE(layer.targetBitrate, layer.maxBitrate);
}
}
} // namespace webrtc

427
modules/video_coding/codecs/vp9/svc_rate_allocator.cc Normal file
View File

@ -0,0 +1,427 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/vp9/svc_rate_allocator.h"
#include <algorithm>
#include <cmath>
#include <cstddef>
#include <numeric>
#include <vector>
#include "absl/container/inlined_vector.h"
#include "rtc_base/checks.h"
namespace webrtc {
namespace {
const float kSpatialLayeringRateScalingFactor = 0.55f;
const float kTemporalLayeringRateScalingFactor = 0.55f;
// Returns numberOfSpatialLayers if no layers are active.
size_t GetFirstActiveLayer(const VideoCodec& codec) {
RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9);
RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u);
size_t layer = 0;
for (; layer < codec.VP9().numberOfSpatialLayers; ++layer) {
if (codec.spatialLayers[layer].active) {
break;
}
}
return layer;
}
static size_t GetNumActiveSpatialLayers(const VideoCodec& codec) {
RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9);
RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u);
const size_t first_active_layer = GetFirstActiveLayer(codec);
size_t last_active_layer = first_active_layer;
for (; last_active_layer < codec.VP9().numberOfSpatialLayers;
++last_active_layer) {
if (!codec.spatialLayers[last_active_layer].active) {
break;
}
}
return last_active_layer - first_active_layer;
}
std::vector<DataRate> AdjustAndVerify(
const VideoCodec& codec,
size_t first_active_layer,
const std::vector<DataRate>& spatial_layer_rates) {
std::vector<DataRate> adjusted_spatial_layer_rates;
// Keep track of rate that couldn't be applied to the previous layer due to
// max bitrate constraint, try to pass it forward to the next one.
DataRate excess_rate = DataRate::Zero();
for (size_t sl_idx = 0; sl_idx < spatial_layer_rates.size(); ++sl_idx) {
DataRate min_rate = DataRate::KilobitsPerSec(
codec.spatialLayers[first_active_layer + sl_idx].minBitrate);
DataRate max_rate = DataRate::KilobitsPerSec(
codec.spatialLayers[first_active_layer + sl_idx].maxBitrate);
DataRate layer_rate = spatial_layer_rates[sl_idx] + excess_rate;
if (layer_rate < min_rate) {
// Not enough rate to reach min bitrate for desired number of layers,
// abort allocation.
if (spatial_layer_rates.size() == 1) {
return spatial_layer_rates;
}
return adjusted_spatial_layer_rates;
}
if (layer_rate <= max_rate) {
excess_rate = DataRate::Zero();
adjusted_spatial_layer_rates.push_back(layer_rate);
} else {
excess_rate = layer_rate - max_rate;
adjusted_spatial_layer_rates.push_back(max_rate);
}
}
return adjusted_spatial_layer_rates;
}
static std::vector<DataRate> SplitBitrate(size_t num_layers,
DataRate total_bitrate,
float rate_scaling_factor) {
std::vector<DataRate> bitrates;
double denominator = 0.0;
for (size_t layer_idx = 0; layer_idx < num_layers; ++layer_idx) {
denominator += std::pow(rate_scaling_factor, layer_idx);
}
double numerator = std::pow(rate_scaling_factor, num_layers - 1);
for (size_t layer_idx = 0; layer_idx < num_layers; ++layer_idx) {
bitrates.push_back(numerator * total_bitrate / denominator);
numerator /= rate_scaling_factor;
}
const DataRate sum =
std::accumulate(bitrates.begin(), bitrates.end(), DataRate::Zero());
// Keep the sum of split bitrates equal to the total bitrate by adding or
// subtracting bits, which were lost due to rounding, to the latest layer.
if (total_bitrate > sum) {
bitrates.back() += total_bitrate - sum;
} else if (total_bitrate < sum) {
bitrates.back() -= sum - total_bitrate;
}
return bitrates;
}
// Returns the minimum bitrate needed for |num_active_layers| spatial layers to
// become active using the configuration specified by |codec|.
DataRate FindLayerTogglingThreshold(const VideoCodec& codec,
size_t first_active_layer,
size_t num_active_layers) {
if (num_active_layers == 1) {
return DataRate::KilobitsPerSec(codec.spatialLayers[0].minBitrate);
}
if (codec.mode == VideoCodecMode::kRealtimeVideo) {
DataRate lower_bound = DataRate::Zero();
DataRate upper_bound = DataRate::Zero();
if (num_active_layers > 1) {
for (size_t i = 0; i < num_active_layers - 1; ++i) {
lower_bound += DataRate::KilobitsPerSec(
codec.spatialLayers[first_active_layer + i].minBitrate);
upper_bound += DataRate::KilobitsPerSec(
codec.spatialLayers[first_active_layer + i].maxBitrate);
}
}
upper_bound += DataRate::KilobitsPerSec(
codec.spatialLayers[num_active_layers - 1].minBitrate);
// Do a binary search until upper and lower bound is the highest bitrate for
// |num_active_layers| - 1 layers and lowest bitrate for |num_active_layers|
// layers respectively.
while (upper_bound - lower_bound > DataRate::BitsPerSec(1)) {
DataRate try_rate = (lower_bound + upper_bound) / 2;
if (AdjustAndVerify(codec, first_active_layer,
SplitBitrate(num_active_layers, try_rate,
kSpatialLayeringRateScalingFactor))
.size() == num_active_layers) {
upper_bound = try_rate;
} else {
lower_bound = try_rate;
}
}
return upper_bound;
} else {
DataRate toggling_rate = DataRate::Zero();
for (size_t i = 0; i < num_active_layers - 1; ++i) {
toggling_rate += DataRate::KilobitsPerSec(
codec.spatialLayers[first_active_layer + i].targetBitrate);
}
toggling_rate += DataRate::KilobitsPerSec(
codec.spatialLayers[first_active_layer + num_active_layers - 1]
.minBitrate);
return toggling_rate;
}
}
} // namespace
SvcRateAllocator::SvcRateAllocator(const VideoCodec& codec)
: codec_(codec),
experiment_settings_(StableTargetRateExperiment::ParseFromFieldTrials()),
cumulative_layer_start_bitrates_(GetLayerStartBitrates(codec)),
last_active_layer_count_(0) {
RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9);
RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u);
RTC_DCHECK_GT(codec.VP9().numberOfTemporalLayers, 0u);
for (size_t layer_idx = 0; layer_idx < codec.VP9().numberOfSpatialLayers;
++layer_idx) {
// Verify min <= target <= max.
if (codec.spatialLayers[layer_idx].active) {
RTC_DCHECK_GT(codec.spatialLayers[layer_idx].maxBitrate, 0);
RTC_DCHECK_GE(codec.spatialLayers[layer_idx].maxBitrate,
codec.spatialLayers[layer_idx].minBitrate);
RTC_DCHECK_GE(codec.spatialLayers[layer_idx].targetBitrate,
codec.spatialLayers[layer_idx].minBitrate);
RTC_DCHECK_GE(codec.spatialLayers[layer_idx].maxBitrate,
codec.spatialLayers[layer_idx].targetBitrate);
}
}
}
VideoBitrateAllocation SvcRateAllocator::Allocate(
VideoBitrateAllocationParameters parameters) {
DataRate total_bitrate = parameters.total_bitrate;
if (codec_.maxBitrate != 0) {
total_bitrate =
std::min(total_bitrate, DataRate::KilobitsPerSec(codec_.maxBitrate));
}
if (codec_.spatialLayers[0].targetBitrate == 0) {
// Delegate rate distribution to VP9 encoder wrapper if bitrate thresholds
// are not set.
VideoBitrateAllocation bitrate_allocation;
bitrate_allocation.SetBitrate(0, 0, total_bitrate.bps());
return bitrate_allocation;
}
const size_t first_active_layer = GetFirstActiveLayer(codec_);
const size_t num_active_layers = GetNumActiveSpatialLayers(codec_);
size_t num_spatial_layers = num_active_layers;
if (num_spatial_layers == 0) {
return VideoBitrateAllocation(); // All layers are deactivated.
}
// Figure out how many spatial layers should be active.
if (experiment_settings_.IsEnabled() &&
parameters.stable_bitrate > DataRate::Zero()) {
double hysteresis_factor;
if (codec_.mode == VideoCodecMode::kScreensharing) {
hysteresis_factor = experiment_settings_.GetScreenshareHysteresisFactor();
} else {
hysteresis_factor = experiment_settings_.GetVideoHysteresisFactor();
}
DataRate stable_rate =
std::min(parameters.total_bitrate, parameters.stable_bitrate);
// First check if bitrate has grown large enough to enable new layers.
size_t num_enabled_with_hysteresis =
FindNumEnabledLayers(stable_rate / hysteresis_factor);
if (num_enabled_with_hysteresis >= last_active_layer_count_) {
num_spatial_layers = num_enabled_with_hysteresis;
} else {
// We could not enable new layers, check if any should be disabled.
num_spatial_layers =
std::min(last_active_layer_count_, FindNumEnabledLayers(stable_rate));
}
} else {
num_spatial_layers = FindNumEnabledLayers(parameters.total_bitrate);
}
last_active_layer_count_ = num_spatial_layers;
VideoBitrateAllocation allocation;
if (codec_.mode == VideoCodecMode::kRealtimeVideo) {
allocation = GetAllocationNormalVideo(total_bitrate, first_active_layer,
num_spatial_layers);
} else {
allocation = GetAllocationScreenSharing(total_bitrate, first_active_layer,
num_spatial_layers);
}
allocation.set_bw_limited(num_spatial_layers < num_active_layers);
return allocation;
}
VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo(
DataRate total_bitrate,
size_t first_active_layer,
size_t num_spatial_layers) const {
std::vector<DataRate> spatial_layer_rates;
if (num_spatial_layers == 0) {
// Not enough rate for even the base layer. Force allocation at the total
// bitrate anyway.
num_spatial_layers = 1;
spatial_layer_rates.push_back(total_bitrate);
} else {
spatial_layer_rates =
AdjustAndVerify(codec_, first_active_layer,
SplitBitrate(num_spatial_layers, total_bitrate,
kSpatialLayeringRateScalingFactor));
RTC_DCHECK_EQ(spatial_layer_rates.size(), num_spatial_layers);
}
VideoBitrateAllocation bitrate_allocation;
const size_t num_temporal_layers = codec_.VP9().numberOfTemporalLayers;
for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
std::vector<DataRate> temporal_layer_rates =
SplitBitrate(num_temporal_layers, spatial_layer_rates[sl_idx],
kTemporalLayeringRateScalingFactor);
// Distribute rate across temporal layers. Allocate more bits to lower
// layers since they are used for prediction of higher layers and their
// references are far apart.
if (num_temporal_layers == 1) {
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
temporal_layer_rates[0].bps());
} else if (num_temporal_layers == 2) {
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
temporal_layer_rates[1].bps());
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1,
temporal_layer_rates[0].bps());
} else {
RTC_CHECK_EQ(num_temporal_layers, 3);
// In case of three temporal layers the high layer has two frames and the
// middle layer has one frame within GOP (in between two consecutive low
// layer frames). Thus high layer requires more bits (comparing pure
// bitrate of layer, excluding bitrate of base layers) to keep quality on
// par with lower layers.
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
temporal_layer_rates[2].bps());
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1,
temporal_layer_rates[0].bps());
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 2,
temporal_layer_rates[1].bps());
}
}
return bitrate_allocation;
}
// Bit-rate is allocated in such a way, that the highest enabled layer will have
// between min and max bitrate, and all others will have exactly target
// bit-rate allocated.
VideoBitrateAllocation SvcRateAllocator::GetAllocationScreenSharing(
DataRate total_bitrate,
size_t first_active_layer,
size_t num_spatial_layers) const {
VideoBitrateAllocation bitrate_allocation;
if (num_spatial_layers == 0 ||
total_bitrate <
DataRate::KilobitsPerSec(
codec_.spatialLayers[first_active_layer].minBitrate)) {
// Always enable at least one layer.
bitrate_allocation.SetBitrate(first_active_layer, 0, total_bitrate.bps());
return bitrate_allocation;
}
DataRate allocated_rate = DataRate::Zero();
DataRate top_layer_rate = DataRate::Zero();
size_t sl_idx;
for (sl_idx = first_active_layer;
sl_idx < first_active_layer + num_spatial_layers; ++sl_idx) {
const DataRate min_rate =
DataRate::KilobitsPerSec(codec_.spatialLayers[sl_idx].minBitrate);
const DataRate target_rate =
DataRate::KilobitsPerSec(codec_.spatialLayers[sl_idx].targetBitrate);
if (allocated_rate + min_rate > total_bitrate) {
// Use stable rate to determine if layer should be enabled.
break;
}
top_layer_rate = std::min(target_rate, total_bitrate - allocated_rate);
bitrate_allocation.SetBitrate(sl_idx, 0, top_layer_rate.bps());
allocated_rate += top_layer_rate;
}
if (sl_idx > 0 && total_bitrate - allocated_rate > DataRate::Zero()) {
// Add leftover to the last allocated layer.
top_layer_rate = std::min(
top_layer_rate + (total_bitrate - allocated_rate),
DataRate::KilobitsPerSec(codec_.spatialLayers[sl_idx - 1].maxBitrate));
bitrate_allocation.SetBitrate(sl_idx - 1, 0, top_layer_rate.bps());
}
return bitrate_allocation;
}
size_t SvcRateAllocator::FindNumEnabledLayers(DataRate target_rate) const {
if (cumulative_layer_start_bitrates_.empty()) {
return 0;
}
size_t num_enabled_layers = 0;
for (DataRate start_rate : cumulative_layer_start_bitrates_) {
// First layer is always enabled.
if (num_enabled_layers == 0 || start_rate <= target_rate) {
++num_enabled_layers;
} else {
break;
}
}
return num_enabled_layers;
}
DataRate SvcRateAllocator::GetMaxBitrate(const VideoCodec& codec) {
const size_t first_active_layer = GetFirstActiveLayer(codec);
const size_t num_spatial_layers = GetNumActiveSpatialLayers(codec);
DataRate max_bitrate = DataRate::Zero();
for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
max_bitrate += DataRate::KilobitsPerSec(
codec.spatialLayers[first_active_layer + sl_idx].maxBitrate);
}
if (codec.maxBitrate != 0) {
max_bitrate =
std::min(max_bitrate, DataRate::KilobitsPerSec(codec.maxBitrate));
}
return max_bitrate;
}
DataRate SvcRateAllocator::GetPaddingBitrate(const VideoCodec& codec) {
auto start_bitrate = GetLayerStartBitrates(codec);
if (start_bitrate.empty()) {
return DataRate::Zero(); // All layers are deactivated.
}
return start_bitrate.back();
}
absl::InlinedVector<DataRate, kMaxSpatialLayers>
SvcRateAllocator::GetLayerStartBitrates(const VideoCodec& codec) {
absl::InlinedVector<DataRate, kMaxSpatialLayers> start_bitrates;
const size_t first_active_layer = GetFirstActiveLayer(codec);
const size_t num_layers = GetNumActiveSpatialLayers(codec);
DataRate last_rate = DataRate::Zero();
for (size_t i = 1; i <= num_layers; ++i) {
DataRate layer_toggling_rate =
FindLayerTogglingThreshold(codec, first_active_layer, i);
start_bitrates.push_back(layer_toggling_rate);
RTC_DCHECK_LE(last_rate, layer_toggling_rate);
last_rate = layer_toggling_rate;
}
return start_bitrates;
}
} // namespace webrtc

62
modules/video_coding/codecs/vp9/svc_rate_allocator.h Normal file
View File

@ -0,0 +1,62 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_VP9_SVC_RATE_ALLOCATOR_H_
#define MODULES_VIDEO_CODING_CODECS_VP9_SVC_RATE_ALLOCATOR_H_
#include <stddef.h>
#include <stdint.h>
#include "absl/container/inlined_vector.h"
#include "api/video/video_bitrate_allocation.h"
#include "api/video/video_bitrate_allocator.h"
#include "api/video/video_codec_constants.h"
#include "api/video_codecs/video_codec.h"
#include "rtc_base/experiments/stable_target_rate_experiment.h"
namespace webrtc {
class SvcRateAllocator : public VideoBitrateAllocator {
public:
explicit SvcRateAllocator(const VideoCodec& codec);
VideoBitrateAllocation Allocate(
VideoBitrateAllocationParameters parameters) override;
static DataRate GetMaxBitrate(const VideoCodec& codec);
static DataRate GetPaddingBitrate(const VideoCodec& codec);
static absl::InlinedVector<DataRate, kMaxSpatialLayers> GetLayerStartBitrates(
const VideoCodec& codec);
private:
VideoBitrateAllocation GetAllocationNormalVideo(
DataRate total_bitrate,
size_t first_active_layer,
size_t num_spatial_layers) const;
VideoBitrateAllocation GetAllocationScreenSharing(
DataRate total_bitrate,
size_t first_active_layer,
size_t num_spatial_layers) const;
// Returns the number of layers that are active and have enough bitrate to
// actually be enabled.
size_t FindNumEnabledLayers(DataRate target_rate) const;
const VideoCodec codec_;
const StableTargetRateExperiment experiment_settings_;
const absl::InlinedVector<DataRate, kMaxSpatialLayers>
cumulative_layer_start_bitrates_;
size_t last_active_layer_count_;
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_VP9_SVC_RATE_ALLOCATOR_H_

493
modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc Normal file
View File

@ -0,0 +1,493 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/vp9/svc_rate_allocator.h"
#include <algorithm>
#include <vector>
#include "modules/video_coding/codecs/vp9/svc_config.h"
#include "rtc_base/checks.h"
#include "test/field_trial.h"
#include "test/gtest.h"
namespace webrtc {
namespace test {
namespace {
static VideoCodec Configure(size_t width,
size_t height,
size_t num_spatial_layers,
size_t num_temporal_layers,
bool is_screen_sharing) {
VideoCodec codec;
codec.width = width;
codec.height = height;
codec.codecType = kVideoCodecVP9;
codec.mode = is_screen_sharing ? VideoCodecMode::kScreensharing
: VideoCodecMode::kRealtimeVideo;
std::vector<SpatialLayer> spatial_layers =
GetSvcConfig(width, height, 30, /*first_active_layer=*/0,
num_spatial_layers, num_temporal_layers, is_screen_sharing);
RTC_CHECK_LE(spatial_layers.size(), kMaxSpatialLayers);
codec.VP9()->numberOfSpatialLayers =
std::min<unsigned char>(num_spatial_layers, spatial_layers.size());
codec.VP9()->numberOfTemporalLayers = std::min<unsigned char>(
num_temporal_layers, spatial_layers.back().numberOfTemporalLayers);
for (size_t sl_idx = 0; sl_idx < spatial_layers.size(); ++sl_idx) {
codec.spatialLayers[sl_idx] = spatial_layers[sl_idx];
}
return codec;
}
} // namespace
TEST(SvcRateAllocatorTest, SingleLayerFor320x180Input) {
VideoCodec codec = Configure(320, 180, 3, 3, false);
SvcRateAllocator allocator = SvcRateAllocator(codec);
VideoBitrateAllocation allocation =
allocator.Allocate(VideoBitrateAllocationParameters(1000 * 1000, 30));
EXPECT_GT(allocation.GetSpatialLayerSum(0), 0u);
EXPECT_EQ(allocation.GetSpatialLayerSum(1), 0u);
}
TEST(SvcRateAllocatorTest, TwoLayersFor640x360Input) {
VideoCodec codec = Configure(640, 360, 3, 3, false);
SvcRateAllocator allocator = SvcRateAllocator(codec);
VideoBitrateAllocation allocation =
allocator.Allocate(VideoBitrateAllocationParameters(1000 * 1000, 30));
EXPECT_GT(allocation.GetSpatialLayerSum(0), 0u);
EXPECT_GT(allocation.GetSpatialLayerSum(1), 0u);
EXPECT_EQ(allocation.GetSpatialLayerSum(2), 0u);
}
TEST(SvcRateAllocatorTest, ThreeLayersFor1280x720Input) {
VideoCodec codec = Configure(1280, 720, 3, 3, false);
SvcRateAllocator allocator = SvcRateAllocator(codec);
VideoBitrateAllocation allocation =
allocator.Allocate(VideoBitrateAllocationParameters(1000 * 1000, 30));
EXPECT_GT(allocation.GetSpatialLayerSum(0), 0u);
EXPECT_GT(allocation.GetSpatialLayerSum(1), 0u);
EXPECT_GT(allocation.GetSpatialLayerSum(2), 0u);
}
TEST(SvcRateAllocatorTest,
BaseLayerNonZeroBitrateEvenIfTotalIfLessThanMinimum) {
VideoCodec codec = Configure(1280, 720, 3, 3, false);
SvcRateAllocator allocator = SvcRateAllocator(codec);
const SpatialLayer* layers = codec.spatialLayers;
VideoBitrateAllocation allocation = allocator.Allocate(
VideoBitrateAllocationParameters(layers[0].minBitrate * 1000 / 2, 30));
EXPECT_GT(allocation.GetSpatialLayerSum(0), 0u);
EXPECT_LT(allocation.GetSpatialLayerSum(0), layers[0].minBitrate * 1000);
EXPECT_EQ(allocation.GetSpatialLayerSum(1), 0u);
}
TEST(SvcRateAllocatorTest, Disable640x360Layer) {
VideoCodec codec = Configure(1280, 720, 3, 3, false);
SvcRateAllocator allocator = SvcRateAllocator(codec);
const SpatialLayer* layers = codec.spatialLayers;
size_t min_bitrate_for_640x360_layer_kbps =
layers[0].minBitrate + layers[1].minBitrate;
VideoBitrateAllocation allocation =
allocator.Allocate(VideoBitrateAllocationParameters(
min_bitrate_for_640x360_layer_kbps * 1000 - 1, 30));
EXPECT_GT(allocation.GetSpatialLayerSum(0), 0u);
EXPECT_EQ(allocation.GetSpatialLayerSum(1), 0u);
}
TEST(SvcRateAllocatorTest, Disable1280x720Layer) {
VideoCodec codec = Configure(1280, 720, 3, 3, false);
SvcRateAllocator allocator = SvcRateAllocator(codec);
const SpatialLayer* layers = codec.spatialLayers;
size_t min_bitrate_for_1280x720_layer_kbps =
layers[0].minBitrate + layers[1].minBitrate + layers[2].minBitrate;
VideoBitrateAllocation allocation =
allocator.Allocate(VideoBitrateAllocationParameters(
min_bitrate_for_1280x720_layer_kbps * 1000 - 1, 30));
EXPECT_GT(allocation.GetSpatialLayerSum(0), 0u);
EXPECT_GT(allocation.GetSpatialLayerSum(1), 0u);
EXPECT_EQ(allocation.GetSpatialLayerSum(2), 0u);
}
TEST(SvcRateAllocatorTest, BitrateIsCapped) {
VideoCodec codec = Configure(1280, 720, 3, 3, false);
SvcRateAllocator allocator = SvcRateAllocator(codec);
const SpatialLayer* layers = codec.spatialLayers;
const uint32_t link_mbps = 100;
VideoBitrateAllocation allocation = allocator.Allocate(
VideoBitrateAllocationParameters(link_mbps * 1000000, 30));
EXPECT_EQ(allocation.get_sum_kbps(),
layers[0].maxBitrate + layers[1].maxBitrate + layers[2].maxBitrate);
EXPECT_EQ(allocation.GetSpatialLayerSum(0) / 1000, layers[0].maxBitrate);
EXPECT_EQ(allocation.GetSpatialLayerSum(1) / 1000, layers[1].maxBitrate);
EXPECT_EQ(allocation.GetSpatialLayerSum(2) / 1000, layers[2].maxBitrate);
}
TEST(SvcRateAllocatorTest, MinBitrateToGetQualityLayer) {
VideoCodec codec = Configure(1280, 720, 3, 1, true);
SvcRateAllocator allocator = SvcRateAllocator(codec);
const SpatialLayer* layers = codec.spatialLayers;
EXPECT_LE(codec.VP9()->numberOfSpatialLayers, 3U);
VideoBitrateAllocation allocation = allocator.Allocate(
VideoBitrateAllocationParameters(layers[0].minBitrate * 1000, 30));
EXPECT_EQ(allocation.GetSpatialLayerSum(0) / 1000, layers[0].minBitrate);
EXPECT_EQ(allocation.GetSpatialLayerSum(1), 0UL);
allocation = allocator.Allocate(VideoBitrateAllocationParameters(
(layers[0].targetBitrate + layers[1].minBitrate) * 1000, 30));
EXPECT_EQ(allocation.GetSpatialLayerSum(0) / 1000, layers[0].targetBitrate);
EXPECT_EQ(allocation.GetSpatialLayerSum(1) / 1000, layers[1].minBitrate);
}
TEST(SvcRateAllocatorTest, DeactivateHigherLayers) {
for (int deactivated_idx = 2; deactivated_idx >= 0; --deactivated_idx) {
VideoCodec codec = Configure(1280, 720, 3, 1, false);
EXPECT_LE(codec.VP9()->numberOfSpatialLayers, 3U);
for (int i = deactivated_idx; i < 3; ++i)
codec.spatialLayers[i].active = false;
SvcRateAllocator allocator = SvcRateAllocator(codec);
VideoBitrateAllocation allocation = allocator.Allocate(
VideoBitrateAllocationParameters(10 * 1000 * 1000, 30));
// Ensure layers spatial_idx < deactivated_idx are activated.
for (int spatial_idx = 0; spatial_idx < deactivated_idx; ++spatial_idx) {
EXPECT_GT(allocation.GetSpatialLayerSum(spatial_idx), 0UL);
}
// Ensure layers spatial_idx >= deactivated_idx are deactivated.
for (int spatial_idx = deactivated_idx; spatial_idx < 3; ++spatial_idx) {
EXPECT_EQ(allocation.GetSpatialLayerSum(spatial_idx), 0UL);
}
}
}
TEST(SvcRateAllocatorTest, DeactivateLowerLayers) {
for (int deactivated_idx = 0; deactivated_idx < 3; ++deactivated_idx) {
VideoCodec codec = Configure(1280, 720, 3, 1, false);
EXPECT_LE(codec.VP9()->numberOfSpatialLayers, 3U);
for (int i = deactivated_idx; i >= 0; --i)
codec.spatialLayers[i].active = false;
SvcRateAllocator allocator = SvcRateAllocator(codec);
VideoBitrateAllocation allocation = allocator.Allocate(
VideoBitrateAllocationParameters(10 * 1000 * 1000, 30));
// Ensure layers spatial_idx <= deactivated_idx are deactivated.
for (int spatial_idx = 0; spatial_idx <= deactivated_idx; ++spatial_idx) {
EXPECT_EQ(allocation.GetSpatialLayerSum(spatial_idx), 0UL);
}
// Ensure layers spatial_idx > deactivated_idx are activated.
for (int spatial_idx = deactivated_idx + 1; spatial_idx < 3;
++spatial_idx) {
EXPECT_GT(allocation.GetSpatialLayerSum(spatial_idx), 0UL);
}
}
}
TEST(SvcRateAllocatorTest, SignalsBwLimited) {
VideoCodec codec = Configure(1280, 720, 3, 1, false);
SvcRateAllocator allocator = SvcRateAllocator(codec);
// Rough estimate calculated by hand.
uint32_t min_to_enable_all = 900000;
EXPECT_TRUE(
allocator
.Allocate(VideoBitrateAllocationParameters(min_to_enable_all / 2, 30))
.is_bw_limited());
EXPECT_FALSE(
allocator
.Allocate(VideoBitrateAllocationParameters(min_to_enable_all, 30))
.is_bw_limited());
}
TEST(SvcRateAllocatorTest, NoPaddingIfAllLayersAreDeactivated) {
VideoCodec codec = Configure(1280, 720, 3, 1, false);
EXPECT_EQ(codec.VP9()->numberOfSpatialLayers, 3U);
// Deactivation of base layer deactivates all layers.
codec.spatialLayers[0].active = false;
codec.spatialLayers[1].active = false;
codec.spatialLayers[2].active = false;
DataRate padding_rate = SvcRateAllocator::GetPaddingBitrate(codec);
EXPECT_EQ(padding_rate, DataRate::Zero());
}
TEST(SvcRateAllocatorTest, FindLayerTogglingThreshold) {
// Let's unit test a utility method of the unit test...
// Predetermined constants indicating the min bitrate needed for two and three
// layers to be enabled respectively, using the config from Configure() with
// 1280x720 resolution and three spatial layers.
const DataRate kTwoLayerMinRate = DataRate::BitsPerSec(299150);
const DataRate kThreeLayerMinRate = DataRate::BitsPerSec(891052);
VideoCodec codec = Configure(1280, 720, 3, 1, false);
absl::InlinedVector<DataRate, kMaxSpatialLayers> layer_start_bitrates =
SvcRateAllocator::GetLayerStartBitrates(codec);
ASSERT_EQ(layer_start_bitrates.size(), 3u);
EXPECT_EQ(layer_start_bitrates[1], kTwoLayerMinRate);
EXPECT_EQ(layer_start_bitrates[2], kThreeLayerMinRate);
}
class SvcRateAllocatorTestParametrizedContentType
: public ::testing::Test,
public ::testing::WithParamInterface<bool> {
public:
SvcRateAllocatorTestParametrizedContentType()
: is_screen_sharing_(GetParam()) {}
const bool is_screen_sharing_;
};
TEST_P(SvcRateAllocatorTestParametrizedContentType, MaxBitrate) {
VideoCodec codec = Configure(1280, 720, 3, 1, is_screen_sharing_);
EXPECT_EQ(SvcRateAllocator::GetMaxBitrate(codec),
DataRate::KilobitsPerSec(codec.spatialLayers[0].maxBitrate +
codec.spatialLayers[1].maxBitrate +
codec.spatialLayers[2].maxBitrate));
// Deactivate middle layer. This causes deactivation of top layer as well.
codec.spatialLayers[1].active = false;
EXPECT_EQ(SvcRateAllocator::GetMaxBitrate(codec),
DataRate::KilobitsPerSec(codec.spatialLayers[0].maxBitrate));
}
TEST_P(SvcRateAllocatorTestParametrizedContentType, PaddingBitrate) {
VideoCodec codec = Configure(1280, 720, 3, 1, is_screen_sharing_);
SvcRateAllocator allocator = SvcRateAllocator(codec);
DataRate padding_bitrate = SvcRateAllocator::GetPaddingBitrate(codec);
VideoBitrateAllocation allocation =
allocator.Allocate(VideoBitrateAllocationParameters(padding_bitrate, 30));
EXPECT_GT(allocation.GetSpatialLayerSum(0), 0UL);
EXPECT_GT(allocation.GetSpatialLayerSum(1), 0UL);
EXPECT_GT(allocation.GetSpatialLayerSum(2), 0UL);
// Allocate 90% of padding bitrate. Top layer should be disabled.
allocation = allocator.Allocate(
VideoBitrateAllocationParameters(9 * padding_bitrate / 10, 30));
EXPECT_GT(allocation.GetSpatialLayerSum(0), 0UL);
EXPECT_GT(allocation.GetSpatialLayerSum(1), 0UL);
EXPECT_EQ(allocation.GetSpatialLayerSum(2), 0UL);
// Deactivate top layer.
codec.spatialLayers[2].active = false;
padding_bitrate = SvcRateAllocator::GetPaddingBitrate(codec);
allocation =
allocator.Allocate(VideoBitrateAllocationParameters(padding_bitrate, 30));
EXPECT_GT(allocation.GetSpatialLayerSum(0), 0UL);
EXPECT_GT(allocation.GetSpatialLayerSum(1), 0UL);
EXPECT_EQ(allocation.GetSpatialLayerSum(2), 0UL);
allocation = allocator.Allocate(
VideoBitrateAllocationParameters(9 * padding_bitrate / 10, 30));
EXPECT_GT(allocation.GetSpatialLayerSum(0), 0UL);
EXPECT_EQ(allocation.GetSpatialLayerSum(1), 0UL);
EXPECT_EQ(allocation.GetSpatialLayerSum(2), 0UL);
// Deactivate all layers.
codec.spatialLayers[0].active = false;
codec.spatialLayers[1].active = false;
codec.spatialLayers[2].active = false;
padding_bitrate = SvcRateAllocator::GetPaddingBitrate(codec);
// No padding expected.
EXPECT_EQ(DataRate::Zero(), padding_bitrate);
}
TEST_P(SvcRateAllocatorTestParametrizedContentType, StableBitrate) {
ScopedFieldTrials field_trial(
"WebRTC-StableTargetRate/enabled:true,video_hysteresis_factor:1.0,"
"screenshare_hysteresis_factor:1.0/");
const VideoCodec codec = Configure(1280, 720, 3, 1, is_screen_sharing_);
const auto start_rates = SvcRateAllocator::GetLayerStartBitrates(codec);
const DataRate min_rate_two_layers = start_rates[1];
const DataRate min_rate_three_layers = start_rates[2];
const DataRate max_rate_one_layer =
DataRate::KilobitsPerSec(codec.spatialLayers[0].maxBitrate);
const DataRate max_rate_two_layers =
is_screen_sharing_
? DataRate::KilobitsPerSec(codec.spatialLayers[0].targetBitrate +
codec.spatialLayers[1].maxBitrate)
: DataRate::KilobitsPerSec(codec.spatialLayers[0].maxBitrate +
codec.spatialLayers[1].maxBitrate);
SvcRateAllocator allocator = SvcRateAllocator(codec);
// Two layers, stable and target equal.
auto allocation = allocator.Allocate(VideoBitrateAllocationParameters(
/*total_bitrate=*/min_rate_two_layers,
/*stable_bitrate=*/min_rate_two_layers, /*fps=*/30.0));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(1));
EXPECT_EQ(allocation.get_sum_bps(), min_rate_two_layers.bps());
// Two layers, stable bitrate too low for two layers.
allocation = allocator.Allocate(VideoBitrateAllocationParameters(
/*total_bitrate=*/min_rate_two_layers,
/*stable_bitrate=*/min_rate_two_layers - DataRate::BitsPerSec(1),
/*fps=*/30.0));
EXPECT_FALSE(allocation.IsSpatialLayerUsed(1));
EXPECT_EQ(DataRate::BitsPerSec(allocation.get_sum_bps()),
std::min(min_rate_two_layers - DataRate::BitsPerSec(1),
max_rate_one_layer));
// Three layers, stable and target equal.
allocation = allocator.Allocate(VideoBitrateAllocationParameters(
/*total_bitrate=*/min_rate_three_layers,
/*stable_bitrate=*/min_rate_three_layers, /*fps=*/30.0));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(2));
EXPECT_EQ(allocation.get_sum_bps(), min_rate_three_layers.bps());
// Three layers, stable bitrate too low for three layers.
allocation = allocator.Allocate(VideoBitrateAllocationParameters(
/*total_bitrate=*/min_rate_three_layers,
/*stable_bitrate=*/min_rate_three_layers - DataRate::BitsPerSec(1),
/*fps=*/30.0));
EXPECT_FALSE(allocation.IsSpatialLayerUsed(2));
EXPECT_EQ(DataRate::BitsPerSec(allocation.get_sum_bps()),
std::min(min_rate_three_layers - DataRate::BitsPerSec(1),
max_rate_two_layers));
}
TEST_P(SvcRateAllocatorTestParametrizedContentType,
StableBitrateWithHysteresis) {
const VideoCodec codec = Configure(1280, 720, 3, 1, is_screen_sharing_);
const auto start_rates = SvcRateAllocator::GetLayerStartBitrates(codec);
const DataRate min_rate_single_layer = start_rates[0];
const DataRate min_rate_two_layers = start_rates[1];
const DataRate min_rate_three_layers = start_rates[2];
ScopedFieldTrials field_trial(
"WebRTC-StableTargetRate/enabled:true,video_hysteresis_factor:1.1,"
"screenshare_hysteresis_factor:1.1/");
SvcRateAllocator allocator = SvcRateAllocator(codec);
// Always use max bitrate as target, verify only stable is used for layer
// count selection.
const DataRate max_bitrate = allocator.GetMaxBitrate(codec);
// Start with a single layer.
auto allocation = allocator.Allocate(VideoBitrateAllocationParameters(
/*total_bitrate=*/max_bitrate,
/*stable_bitrate=*/min_rate_single_layer, /*fps=*/30.0));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(0));
EXPECT_FALSE(allocation.IsSpatialLayerUsed(1));
EXPECT_FALSE(allocation.IsSpatialLayerUsed(2));
// Min bitrate not enough to enable second layer due to 10% hysteresis.
allocation = allocator.Allocate(VideoBitrateAllocationParameters(
/*total_bitrate=*/max_bitrate,
/*stable_bitrate=*/min_rate_two_layers, /*fps=*/30.0));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(0));
EXPECT_FALSE(allocation.IsSpatialLayerUsed(1));
EXPECT_FALSE(allocation.IsSpatialLayerUsed(2));
// Add hysteresis, second layer should turn on.
allocation = allocator.Allocate(VideoBitrateAllocationParameters(
/*total_bitrate=*/max_bitrate,
/*stable_bitrate=*/min_rate_two_layers * 1.1, /*fps=*/30.0));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(0));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(1));
EXPECT_FALSE(allocation.IsSpatialLayerUsed(2));
// Remove hysteresis, second layer should stay on.
allocation = allocator.Allocate(VideoBitrateAllocationParameters(
/*total_bitrate=*/max_bitrate,
/*stable_bitrate=*/min_rate_two_layers, /*fps=*/30.0));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(0));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(1));
EXPECT_FALSE(allocation.IsSpatialLayerUsed(2));
// Going below min for two layers, second layer should turn off again.
allocation = allocator.Allocate(VideoBitrateAllocationParameters(
/*total_bitrate=*/max_bitrate,
/*stable_bitrate=*/min_rate_two_layers - DataRate::BitsPerSec(1),
/*fps=*/30.0));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(0));
EXPECT_FALSE(allocation.IsSpatialLayerUsed(1));
EXPECT_FALSE(allocation.IsSpatialLayerUsed(2));
// Min bitrate not enough to enable third layer due to 10% hysteresis.
allocation = allocator.Allocate(VideoBitrateAllocationParameters(
/*total_bitrate=*/max_bitrate,
/*stable_bitrate=*/min_rate_three_layers, /*fps=*/30.0));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(0));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(1));
EXPECT_FALSE(allocation.IsSpatialLayerUsed(2));
// Add hysteresis, third layer should turn on.
allocation = allocator.Allocate(VideoBitrateAllocationParameters(
/*total_bitrate=*/max_bitrate,
/*stable_bitrate=*/min_rate_three_layers * 1.1, /*fps=*/30.0));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(0));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(1));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(2));
// Remove hysteresis, third layer should stay on.
allocation = allocator.Allocate(VideoBitrateAllocationParameters(
/*total_bitrate=*/max_bitrate,
/*stable_bitrate=*/min_rate_three_layers, /*fps=*/30.0));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(0));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(1));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(2));
// Going below min for three layers, third layer should turn off again.
allocation = allocator.Allocate(VideoBitrateAllocationParameters(
/*total_bitrate=*/max_bitrate,
/*stable_bitrate=*/min_rate_three_layers - DataRate::BitsPerSec(1),
/*fps=*/30.0));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(0));
EXPECT_TRUE(allocation.IsSpatialLayerUsed(1));
EXPECT_FALSE(allocation.IsSpatialLayerUsed(2));
}
INSTANTIATE_TEST_SUITE_P(_,
SvcRateAllocatorTestParametrizedContentType,
::testing::Bool());
} // namespace test
} // namespace webrtc

1777
modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc Normal file
View File

File diff suppressed because it is too large Load Diff

76
modules/video_coding/codecs/vp9/vp9.cc Normal file
View File

@ -0,0 +1,76 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/vp9/include/vp9.h"
#include <memory>
#include "api/video_codecs/sdp_video_format.h"
#include "modules/video_coding/codecs/vp9/vp9_impl.h"
#include "rtc_base/checks.h"
#include "vpx/vp8cx.h"
#include "vpx/vp8dx.h"
#include "vpx/vpx_codec.h"
namespace webrtc {
std::vector<SdpVideoFormat> SupportedVP9Codecs() {
#ifdef RTC_ENABLE_VP9
// Profile 2 might not be available on some platforms until
// https://bugs.chromium.org/p/webm/issues/detail?id=1544 is solved.
static bool vpx_supports_high_bit_depth =
(vpx_codec_get_caps(vpx_codec_vp9_cx()) & VPX_CODEC_CAP_HIGHBITDEPTH) !=
0 &&
(vpx_codec_get_caps(vpx_codec_vp9_dx()) & VPX_CODEC_CAP_HIGHBITDEPTH) !=
0;
std::vector<SdpVideoFormat> supported_formats{SdpVideoFormat(
cricket::kVp9CodecName,
{{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile0)}})};
if (vpx_supports_high_bit_depth) {
supported_formats.push_back(SdpVideoFormat(
cricket::kVp9CodecName,
{{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile2)}}));
}
return supported_formats;
#else
return std::vector<SdpVideoFormat>();
#endif
}
std::unique_ptr<VP9Encoder> VP9Encoder::Create() {
#ifdef RTC_ENABLE_VP9
return std::make_unique<VP9EncoderImpl>(cricket::VideoCodec());
#else
RTC_NOTREACHED();
return nullptr;
#endif
}
std::unique_ptr<VP9Encoder> VP9Encoder::Create(
const cricket::VideoCodec& codec) {
#ifdef RTC_ENABLE_VP9
return std::make_unique<VP9EncoderImpl>(codec);
#else
RTC_NOTREACHED();
return nullptr;
#endif
}
std::unique_ptr<VP9Decoder> VP9Decoder::Create() {
#ifdef RTC_ENABLE_VP9
return std::make_unique<VP9DecoderImpl>();
#else
RTC_NOTREACHED();
return nullptr;
#endif
}
} // namespace webrtc

183
modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.cc Normal file
View File

@ -0,0 +1,183 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#ifdef RTC_ENABLE_VP9
#include "modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/ref_counted_object.h"
#include "vpx/vpx_codec.h"
#include "vpx/vpx_decoder.h"
#include "vpx/vpx_frame_buffer.h"
namespace webrtc {
uint8_t* Vp9FrameBufferPool::Vp9FrameBuffer::GetData() {
return data_.data<uint8_t>();
}
size_t Vp9FrameBufferPool::Vp9FrameBuffer::GetDataSize() const {
return data_.size();
}
void Vp9FrameBufferPool::Vp9FrameBuffer::SetSize(size_t size) {
data_.SetSize(size);
}
bool Vp9FrameBufferPool::InitializeVpxUsePool(
vpx_codec_ctx* vpx_codec_context) {
RTC_DCHECK(vpx_codec_context);
// Tell libvpx to use this pool.
if (vpx_codec_set_frame_buffer_functions(
// In which context to use these callback functions.
vpx_codec_context,
// Called by libvpx when it needs another frame buffer.
&Vp9FrameBufferPool::VpxGetFrameBuffer,
// Called by libvpx when it no longer uses a frame buffer.
&Vp9FrameBufferPool::VpxReleaseFrameBuffer,
// |this| will be passed as |user_priv| to VpxGetFrameBuffer.
this)) {
// Failed to configure libvpx to use Vp9FrameBufferPool.
return false;
}
return true;
}
rtc::scoped_refptr<Vp9FrameBufferPool::Vp9FrameBuffer>
Vp9FrameBufferPool::GetFrameBuffer(size_t min_size) {
RTC_DCHECK_GT(min_size, 0);
rtc::scoped_refptr<Vp9FrameBuffer> available_buffer = nullptr;
{
rtc::CritScope cs(&buffers_lock_);
// Do we have a buffer we can recycle?
for (const auto& buffer : allocated_buffers_) {
if (buffer->HasOneRef()) {
available_buffer = buffer;
break;
}
}
// Otherwise create one.
if (available_buffer == nullptr) {
available_buffer = new rtc::RefCountedObject<Vp9FrameBuffer>();
allocated_buffers_.push_back(available_buffer);
if (allocated_buffers_.size() > max_num_buffers_) {
RTC_LOG(LS_WARNING)
<< allocated_buffers_.size()
<< " Vp9FrameBuffers have been "
"allocated by a Vp9FrameBufferPool (exceeding what is "
"considered reasonable, "
<< max_num_buffers_ << ").";
// TODO(phoglund): this limit is being hit in tests since Oct 5 2016.
// See https://bugs.chromium.org/p/webrtc/issues/detail?id=6484.
// RTC_NOTREACHED();
}
}
}
available_buffer->SetSize(min_size);
return available_buffer;
}
int Vp9FrameBufferPool::GetNumBuffersInUse() const {
int num_buffers_in_use = 0;
rtc::CritScope cs(&buffers_lock_);
for (const auto& buffer : allocated_buffers_) {
if (!buffer->HasOneRef())
++num_buffers_in_use;
}
return num_buffers_in_use;
}
bool Vp9FrameBufferPool::Resize(size_t max_number_of_buffers) {
rtc::CritScope cs(&buffers_lock_);
size_t used_buffers_count = 0;
for (const auto& buffer : allocated_buffers_) {
// If the buffer is in use, the ref count will be >= 2, one from the list we
// are looping over and one from the application. If the ref count is 1,
// then the list we are looping over holds the only reference and it's safe
// to reuse.
if (!buffer->HasOneRef()) {
used_buffers_count++;
}
}
if (used_buffers_count > max_number_of_buffers) {
return false;
}
max_num_buffers_ = max_number_of_buffers;
size_t buffers_to_purge = allocated_buffers_.size() - max_num_buffers_;
auto iter = allocated_buffers_.begin();
while (iter != allocated_buffers_.end() && buffers_to_purge > 0) {
if ((*iter)->HasOneRef()) {
iter = allocated_buffers_.erase(iter);
buffers_to_purge--;
} else {
++iter;
}
}
return true;
}
void Vp9FrameBufferPool::ClearPool() {
rtc::CritScope cs(&buffers_lock_);
allocated_buffers_.clear();
}
// static
int32_t Vp9FrameBufferPool::VpxGetFrameBuffer(void* user_priv,
size_t min_size,
vpx_codec_frame_buffer* fb) {
RTC_DCHECK(user_priv);
RTC_DCHECK(fb);
#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
// Limit size of 8k YUV highdef frame
size_t size_limit = 7680 * 4320 * 3 / 2 * 2;
if (min_size > size_limit)
return -1;
#endif
Vp9FrameBufferPool* pool = static_cast<Vp9FrameBufferPool*>(user_priv);
rtc::scoped_refptr<Vp9FrameBuffer> buffer = pool->GetFrameBuffer(min_size);
fb->data = buffer->GetData();
fb->size = buffer->GetDataSize();
// Store Vp9FrameBuffer* in |priv| for use in VpxReleaseFrameBuffer.
// This also makes vpx_codec_get_frame return images with their |fb_priv| set
// to |buffer| which is important for external reference counting.
// Release from refptr so that the buffer's |ref_count_| remains 1 when
// |buffer| goes out of scope.
fb->priv = static_cast<void*>(buffer.release());
return 0;
}
// static
int32_t Vp9FrameBufferPool::VpxReleaseFrameBuffer(void* user_priv,
vpx_codec_frame_buffer* fb) {
RTC_DCHECK(user_priv);
RTC_DCHECK(fb);
Vp9FrameBuffer* buffer = static_cast<Vp9FrameBuffer*>(fb->priv);
if (buffer != nullptr) {
buffer->Release();
// When libvpx fails to decode and you continue to try to decode (and fail)
// libvpx can for some reason try to release the same buffer multiple times.
// Setting |priv| to null protects against trying to Release multiple times.
fb->priv = nullptr;
}
return 0;
}
} // namespace webrtc
#endif // RTC_ENABLE_VP9

133
modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h Normal file
View File

@ -0,0 +1,133 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#ifndef MODULES_VIDEO_CODING_CODECS_VP9_VP9_FRAME_BUFFER_POOL_H_
#define MODULES_VIDEO_CODING_CODECS_VP9_VP9_FRAME_BUFFER_POOL_H_
#ifdef RTC_ENABLE_VP9
#include <vector>
#include "api/scoped_refptr.h"
#include "rtc_base/buffer.h"
#include "rtc_base/critical_section.h"
#include "rtc_base/ref_count.h"
struct vpx_codec_ctx;
struct vpx_codec_frame_buffer;
namespace webrtc {
// If more buffers than this are allocated we print warnings and crash if in
// debug mode. VP9 is defined to have 8 reference buffers, of which 3 can be
// referenced by any frame, see
// https://tools.ietf.org/html/draft-grange-vp9-bitstream-00#section-2.2.2.
// Assuming VP9 holds on to at most 8 buffers, any more buffers than that
// would have to be by application code. Decoded frames should not be
// referenced for longer than necessary. If we allow ~60 additional buffers
// then the application has ~1 second to e.g. render each frame of a 60 fps
// video.
constexpr size_t kDefaultMaxNumBuffers = 68;
// This memory pool is used to serve buffers to libvpx for decoding purposes in
// VP9, which is set up in InitializeVPXUsePool. After the initialization any
// time libvpx wants to decode a frame it will use buffers provided and released
// through VpxGetFrameBuffer and VpxReleaseFrameBuffer.
// The benefit of owning the pool that libvpx relies on for decoding is that the
// decoded frames returned by libvpx (from vpx_codec_get_frame) use parts of our
// buffers for the decoded image data. By retaining ownership of this buffer
// using scoped_refptr, the image buffer can be reused by VideoFrames and no
// frame copy has to occur during decoding and frame delivery.
//
// Pseudo example usage case:
// Vp9FrameBufferPool pool;
// pool.InitializeVpxUsePool(decoder_ctx);
// ...
//
// // During decoding, libvpx will get and release buffers from the pool.
// vpx_codec_decode(decoder_ctx, ...);
//
// vpx_image_t* img = vpx_codec_get_frame(decoder_ctx, &iter);
// // Important to use scoped_refptr to protect it against being recycled by
// // the pool.
// scoped_refptr<Vp9FrameBuffer> img_buffer = (Vp9FrameBuffer*)img->fb_priv;
// ...
//
// // Destroying the codec will make libvpx release any buffers it was using.
// vpx_codec_destroy(decoder_ctx);
class Vp9FrameBufferPool {
public:
class Vp9FrameBuffer : public rtc::RefCountInterface {
public:
uint8_t* GetData();
size_t GetDataSize() const;
void SetSize(size_t size);
virtual bool HasOneRef() const = 0;
private:
// Data as an easily resizable buffer.
rtc::Buffer data_;
};
// Configures libvpx to, in the specified context, use this memory pool for
// buffers used to decompress frames. This is only supported for VP9.
bool InitializeVpxUsePool(vpx_codec_ctx* vpx_codec_context);
// Gets a frame buffer of at least |min_size|, recycling an available one or
// creating a new one. When no longer referenced from the outside the buffer
// becomes recyclable.
rtc::scoped_refptr<Vp9FrameBuffer> GetFrameBuffer(size_t min_size);
// Gets the number of buffers currently in use (not ready to be recycled).
int GetNumBuffersInUse() const;
// Changes the max amount of buffers in the pool to the new value.
// Returns true if change was successful and false if the amount of already
// allocated buffers is bigger than new value.
bool Resize(size_t max_number_of_buffers);
// Releases allocated buffers, deleting available buffers. Buffers in use are
// not deleted until they are no longer referenced.
void ClearPool();
// InitializeVpxUsePool configures libvpx to call this function when it needs
// a new frame buffer. Parameters:
// |user_priv| Private data passed to libvpx, InitializeVpxUsePool sets it up
// to be a pointer to the pool.
// |min_size| Minimum size needed by libvpx (to decompress a frame).
// |fb| Pointer to the libvpx frame buffer object, this is updated to
// use the pool's buffer.
// Returns 0 on success. Returns < 0 on failure.
static int32_t VpxGetFrameBuffer(void* user_priv,
size_t min_size,
vpx_codec_frame_buffer* fb);
// InitializeVpxUsePool configures libvpx to call this function when it has
// finished using one of the pool's frame buffer. Parameters:
// |user_priv| Private data passed to libvpx, InitializeVpxUsePool sets it up
// to be a pointer to the pool.
// |fb| Pointer to the libvpx frame buffer object, its |priv| will be
// a pointer to one of the pool's Vp9FrameBuffer.
static int32_t VpxReleaseFrameBuffer(void* user_priv,
vpx_codec_frame_buffer* fb);
private:
// Protects |allocated_buffers_|.
rtc::CriticalSection buffers_lock_;
// All buffers, in use or ready to be recycled.
std::vector<rtc::scoped_refptr<Vp9FrameBuffer>> allocated_buffers_
RTC_GUARDED_BY(buffers_lock_);
size_t max_num_buffers_ = kDefaultMaxNumBuffers;
};
} // namespace webrtc
#endif // RTC_ENABLE_VP9
#endif // MODULES_VIDEO_CODING_CODECS_VP9_VP9_FRAME_BUFFER_POOL_H_

1859
modules/video_coding/codecs/vp9/vp9_impl.cc Normal file
View File

File diff suppressed because it is too large Load Diff

218
modules/video_coding/codecs/vp9/vp9_impl.h Normal file
View File

@ -0,0 +1,218 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#ifndef MODULES_VIDEO_CODING_CODECS_VP9_VP9_IMPL_H_
#define MODULES_VIDEO_CODING_CODECS_VP9_VP9_IMPL_H_
#ifdef RTC_ENABLE_VP9
#include <map>
#include <memory>
#include <string>
#include <vector>
#include "api/fec_controller_override.h"
#include "api/video_codecs/video_encoder.h"
#include "media/base/vp9_profile.h"
#include "modules/video_coding/codecs/vp9/include/vp9.h"
#include "modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h"
#include "modules/video_coding/utility/framerate_controller.h"
#include "vpx/vp8cx.h"
#include "vpx/vpx_decoder.h"
#include "vpx/vpx_encoder.h"
namespace webrtc {
class VP9EncoderImpl : public VP9Encoder {
public:
explicit VP9EncoderImpl(const cricket::VideoCodec& codec);
~VP9EncoderImpl() override;
void SetFecControllerOverride(
FecControllerOverride* fec_controller_override) override;
int Release() override;
int InitEncode(const VideoCodec* codec_settings,
const Settings& settings) override;
int Encode(const VideoFrame& input_image,
const std::vector<VideoFrameType>* frame_types) override;
int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override;
void SetRates(const RateControlParameters& parameters) override;
EncoderInfo GetEncoderInfo() const override;
private:
// Determine number of encoder threads to use.
int NumberOfThreads(int width, int height, int number_of_cores);
// Call encoder initialize function and set control settings.
int InitAndSetControlSettings(const VideoCodec* inst);
void PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
absl::optional<int>* spatial_idx,
const vpx_codec_cx_pkt& pkt,
uint32_t timestamp);
void FillReferenceIndices(const vpx_codec_cx_pkt& pkt,
const size_t pic_num,
const bool inter_layer_predicted,
CodecSpecificInfoVP9* vp9_info);
void UpdateReferenceBuffers(const vpx_codec_cx_pkt& pkt,
const size_t pic_num);
vpx_svc_ref_frame_config_t SetReferences(
bool is_key_pic,
size_t first_active_spatial_layer_id);
bool ExplicitlyConfiguredSpatialLayers() const;
bool SetSvcRates(const VideoBitrateAllocation& bitrate_allocation);
virtual int GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt);
// Callback function for outputting packets per spatial layer.
static void EncoderOutputCodedPacketCallback(vpx_codec_cx_pkt* pkt,
void* user_data);
void DeliverBufferedFrame(bool end_of_picture);
bool DropFrame(uint8_t spatial_idx, uint32_t rtp_timestamp);
// Determine maximum target for Intra frames
//
// Input:
// - optimal_buffer_size : Optimal buffer size
// Return Value : Max target size for Intra frames represented as
// percentage of the per frame bandwidth
uint32_t MaxIntraTarget(uint32_t optimal_buffer_size);
size_t SteadyStateSize(int sid, int tid);
EncodedImage encoded_image_;
CodecSpecificInfo codec_specific_;
EncodedImageCallback* encoded_complete_callback_;
VideoCodec codec_;
const VP9Profile profile_;
bool inited_;
int64_t timestamp_;
int cpu_speed_;
uint32_t rc_max_intra_target_;
vpx_codec_ctx_t* encoder_;
vpx_codec_enc_cfg_t* config_;
vpx_image_t* raw_;
vpx_svc_extra_cfg_t svc_params_;
const VideoFrame* input_image_;
GofInfoVP9 gof_; // Contains each frame's temporal information for
// non-flexible mode.
bool force_key_frame_;
size_t pics_since_key_;
uint8_t num_temporal_layers_;
uint8_t num_spatial_layers_; // Number of configured SLs
uint8_t num_active_spatial_layers_; // Number of actively encoded SLs
uint8_t first_active_layer_;
bool layer_deactivation_requires_key_frame_;
bool is_svc_;
InterLayerPredMode inter_layer_pred_;
bool external_ref_control_;
const bool trusted_rate_controller_;
const bool dynamic_rate_settings_;
bool layer_buffering_;
const bool full_superframe_drop_;
vpx_svc_frame_drop_t svc_drop_frame_;
bool first_frame_in_picture_;
VideoBitrateAllocation current_bitrate_allocation_;
bool ss_info_needed_;
bool force_all_active_layers_;
std::vector<FramerateController> framerate_controller_;
// Used for flexible mode.
bool is_flexible_mode_;
struct RefFrameBuffer {
RefFrameBuffer(size_t pic_num,
size_t spatial_layer_id,
size_t temporal_layer_id)
: pic_num(pic_num),
spatial_layer_id(spatial_layer_id),
temporal_layer_id(temporal_layer_id) {}
RefFrameBuffer() {}
bool operator==(const RefFrameBuffer& o) {
return pic_num == o.pic_num && spatial_layer_id == o.spatial_layer_id &&
temporal_layer_id == o.temporal_layer_id;
}
size_t pic_num = 0;
size_t spatial_layer_id = 0;
size_t temporal_layer_id = 0;
};
std::map<size_t, RefFrameBuffer> ref_buf_;
// Variable frame-rate related fields and methods.
const struct VariableFramerateExperiment {
bool enabled;
// Framerate is limited to this value in steady state.
float framerate_limit;
// This qp or below is considered a steady state.
int steady_state_qp;
// Frames of at least this percentage below ideal for configured bitrate are
// considered in a steady state.
int steady_state_undershoot_percentage;
// Number of consecutive frames with good QP and size required to detect
// the steady state.
int frames_before_steady_state;
} variable_framerate_experiment_;
static VariableFramerateExperiment ParseVariableFramerateConfig(
std::string group_name);
FramerateController variable_framerate_controller_;
int num_steady_state_frames_;
// Only set config when this flag is set.
bool config_changed_;
};
class VP9DecoderImpl : public VP9Decoder {
public:
VP9DecoderImpl();
virtual ~VP9DecoderImpl();
int InitDecode(const VideoCodec* inst, int number_of_cores) override;
int Decode(const EncodedImage& input_image,
bool missing_frames,
int64_t /*render_time_ms*/) override;
int RegisterDecodeCompleteCallback(DecodedImageCallback* callback) override;
int Release() override;
const char* ImplementationName() const override;
private:
int ReturnFrame(const vpx_image_t* img,
uint32_t timestamp,
int qp,
const webrtc::ColorSpace* explicit_color_space);
// Memory pool used to share buffers between libvpx and webrtc.
Vp9FrameBufferPool frame_buffer_pool_;
DecodedImageCallback* decode_complete_callback_;
bool inited_;
vpx_codec_ctx_t* decoder_;
bool key_frame_required_;
};
} // namespace webrtc
#endif // RTC_ENABLE_VP9
#endif // MODULES_VIDEO_CODING_CODECS_VP9_VP9_IMPL_H_

198
modules/video_coding/decoder_database.cc Normal file
View File

@ -0,0 +1,198 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/decoder_database.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
namespace webrtc {
VCMDecoderMapItem::VCMDecoderMapItem(VideoCodec* settings,
int number_of_cores,
bool require_key_frame)
: settings(settings),
number_of_cores(number_of_cores),
require_key_frame(require_key_frame) {
RTC_DCHECK_GE(number_of_cores, 0);
}
VCMExtDecoderMapItem::VCMExtDecoderMapItem(
VideoDecoder* external_decoder_instance,
uint8_t payload_type)
: payload_type(payload_type),
external_decoder_instance(external_decoder_instance) {}
VCMDecoderMapItem::~VCMDecoderMapItem() {}
VCMDecoderDataBase::VCMDecoderDataBase()
: receive_codec_(), dec_map_(), dec_external_map_() {}
VCMDecoderDataBase::~VCMDecoderDataBase() {
ptr_decoder_.reset();
for (auto& kv : dec_map_)
delete kv.second;
for (auto& kv : dec_external_map_)
delete kv.second;
}
bool VCMDecoderDataBase::DeregisterExternalDecoder(uint8_t payload_type) {
ExternalDecoderMap::iterator it = dec_external_map_.find(payload_type);
if (it == dec_external_map_.end()) {
// Not found.
return false;
}
// We can't use payload_type to check if the decoder is currently in use,
// because payload type may be out of date (e.g. before we decode the first
// frame after RegisterReceiveCodec).
if (ptr_decoder_ &&
ptr_decoder_->IsSameDecoder((*it).second->external_decoder_instance)) {
// Release it if it was registered and in use.
ptr_decoder_.reset();
}
DeregisterReceiveCodec(payload_type);
delete it->second;
dec_external_map_.erase(it);
return true;
}
// Add the external decoder object to the list of external decoders.
// Won't be registered as a receive codec until RegisterReceiveCodec is called.
void VCMDecoderDataBase::RegisterExternalDecoder(VideoDecoder* external_decoder,
uint8_t payload_type) {
// If payload value already exists, erase old and insert new.
VCMExtDecoderMapItem* ext_decoder =
new VCMExtDecoderMapItem(external_decoder, payload_type);
DeregisterExternalDecoder(payload_type);
dec_external_map_[payload_type] = ext_decoder;
}
bool VCMDecoderDataBase::RegisterReceiveCodec(const VideoCodec* receive_codec,
int number_of_cores,
bool require_key_frame) {
if (number_of_cores < 0) {
return false;
}
// If payload value already exists, erase old and insert new.
DeregisterReceiveCodec(receive_codec->plType);
VideoCodec* new_receive_codec = new VideoCodec(*receive_codec);
dec_map_[receive_codec->plType] = new VCMDecoderMapItem(
new_receive_codec, number_of_cores, require_key_frame);
return true;
}
bool VCMDecoderDataBase::DeregisterReceiveCodec(uint8_t payload_type) {
DecoderMap::iterator it = dec_map_.find(payload_type);
if (it == dec_map_.end()) {
return false;
}
delete it->second;
dec_map_.erase(it);
if (receive_codec_.plType == payload_type) {
// This codec is currently in use.
memset(&receive_codec_, 0, sizeof(VideoCodec));
}
return true;
}
VCMGenericDecoder* VCMDecoderDataBase::GetDecoder(
const VCMEncodedFrame& frame,
VCMDecodedFrameCallback* decoded_frame_callback) {
RTC_DCHECK(decoded_frame_callback->UserReceiveCallback());
uint8_t payload_type = frame.PayloadType();
if (payload_type == receive_codec_.plType || payload_type == 0) {
return ptr_decoder_.get();
}
// If decoder exists - delete.
if (ptr_decoder_) {
ptr_decoder_.reset();
memset(&receive_codec_, 0, sizeof(VideoCodec));
}
ptr_decoder_ = CreateAndInitDecoder(frame, &receive_codec_);
if (!ptr_decoder_) {
return nullptr;
}
VCMReceiveCallback* callback = decoded_frame_callback->UserReceiveCallback();
callback->OnIncomingPayloadType(receive_codec_.plType);
if (ptr_decoder_->RegisterDecodeCompleteCallback(decoded_frame_callback) <
0) {
ptr_decoder_.reset();
memset(&receive_codec_, 0, sizeof(VideoCodec));
return nullptr;
}
return ptr_decoder_.get();
}
bool VCMDecoderDataBase::PrefersLateDecoding() const {
return ptr_decoder_ ? ptr_decoder_->PrefersLateDecoding() : true;
}
std::unique_ptr<VCMGenericDecoder> VCMDecoderDataBase::CreateAndInitDecoder(
const VCMEncodedFrame& frame,
VideoCodec* new_codec) const {
uint8_t payload_type = frame.PayloadType();
RTC_LOG(LS_INFO) << "Initializing decoder with payload type '"
<< static_cast<int>(payload_type) << "'.";
RTC_DCHECK(new_codec);
const VCMDecoderMapItem* decoder_item = FindDecoderItem(payload_type);
if (!decoder_item) {
RTC_LOG(LS_ERROR) << "Can't find a decoder associated with payload type: "
<< static_cast<int>(payload_type);
return nullptr;
}
std::unique_ptr<VCMGenericDecoder> ptr_decoder;
const VCMExtDecoderMapItem* external_dec_item =
FindExternalDecoderItem(payload_type);
if (external_dec_item) {
// External codec.
ptr_decoder.reset(new VCMGenericDecoder(
external_dec_item->external_decoder_instance, true));
} else {
RTC_LOG(LS_ERROR) << "No decoder of this type exists.";
}
if (!ptr_decoder)
return nullptr;
// Copy over input resolutions to prevent codec reinitialization due to
// the first frame being of a different resolution than the database values.
// This is best effort, since there's no guarantee that width/height have been
// parsed yet (and may be zero).
if (frame.EncodedImage()._encodedWidth > 0 &&
frame.EncodedImage()._encodedHeight > 0) {
decoder_item->settings->width = frame.EncodedImage()._encodedWidth;
decoder_item->settings->height = frame.EncodedImage()._encodedHeight;
}
if (ptr_decoder->InitDecode(decoder_item->settings.get(),
decoder_item->number_of_cores) < 0) {
return nullptr;
}
memcpy(new_codec, decoder_item->settings.get(), sizeof(VideoCodec));
return ptr_decoder;
}
const VCMDecoderMapItem* VCMDecoderDataBase::FindDecoderItem(
uint8_t payload_type) const {
DecoderMap::const_iterator it = dec_map_.find(payload_type);
if (it != dec_map_.end()) {
return (*it).second;
}
return nullptr;
}
const VCMExtDecoderMapItem* VCMDecoderDataBase::FindExternalDecoderItem(
uint8_t payload_type) const {
ExternalDecoderMap::const_iterator it = dec_external_map_.find(payload_type);
if (it != dec_external_map_.end()) {
return (*it).second;
}
return nullptr;
}
} // namespace webrtc

90
modules/video_coding/decoder_database.h Normal file
View File

@ -0,0 +1,90 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_DECODER_DATABASE_H_
#define MODULES_VIDEO_CODING_DECODER_DATABASE_H_
#include <map>
#include <memory>
#include "modules/video_coding/generic_decoder.h"
namespace webrtc {
struct VCMDecoderMapItem {
public:
VCMDecoderMapItem(VideoCodec* settings,
int number_of_cores,
bool require_key_frame);
~VCMDecoderMapItem();
std::unique_ptr<VideoCodec> settings;
int number_of_cores;
bool require_key_frame;
};
struct VCMExtDecoderMapItem {
public:
VCMExtDecoderMapItem(VideoDecoder* external_decoder_instance,
uint8_t payload_type);
uint8_t payload_type;
VideoDecoder* external_decoder_instance;
};
class VCMDecoderDataBase {
public:
VCMDecoderDataBase();
~VCMDecoderDataBase();
bool DeregisterExternalDecoder(uint8_t payload_type);
void RegisterExternalDecoder(VideoDecoder* external_decoder,
uint8_t payload_type);
bool RegisterReceiveCodec(const VideoCodec* receive_codec,
int number_of_cores,
bool require_key_frame);
bool DeregisterReceiveCodec(uint8_t payload_type);
// Returns a decoder specified by frame.PayloadType. The decoded frame
// callback of the decoder is set to |decoded_frame_callback|. If no such
// decoder already exists an instance will be created and initialized.
// nullptr is returned if no decoder with the specified payload type was found
// and the function failed to create one.
VCMGenericDecoder* GetDecoder(
const VCMEncodedFrame& frame,
VCMDecodedFrameCallback* decoded_frame_callback);
// Returns true if the currently active decoder prefer to decode frames late.
// That means that frames must be decoded near the render times stamp.
bool PrefersLateDecoding() const;
private:
typedef std::map<uint8_t, VCMDecoderMapItem*> DecoderMap;
typedef std::map<uint8_t, VCMExtDecoderMapItem*> ExternalDecoderMap;
std::unique_ptr<VCMGenericDecoder> CreateAndInitDecoder(
const VCMEncodedFrame& frame,
VideoCodec* new_codec) const;
const VCMDecoderMapItem* FindDecoderItem(uint8_t payload_type) const;
const VCMExtDecoderMapItem* FindExternalDecoderItem(
uint8_t payload_type) const;
VideoCodec receive_codec_;
std::unique_ptr<VCMGenericDecoder> ptr_decoder_;
DecoderMap dec_map_;
ExternalDecoderMap dec_external_map_;
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_DECODER_DATABASE_H_

Some files were not shown because too many files have changed in this diff Show More