zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove VCMEncodedFrameCallback and VCMGenericEncoder

Browse Source 
This CL takes a few parts of VCMEncodedFrameCallback and
VCMGenericEncoder and folds some aspect directly into
VideoStreamEncoder. Parts related to timing frames are extracted
into a new class FrameEncodeTimer that explicitly handles that.

Bug: webrtc:10164
Change-Id: I9b26f734473b659e4093c84c09fb0ed441290e40
Reviewed-on: https://webrtc-review.googlesource.com/c/124122
Commit-Queue: Erik Språng <sprang@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26862}
This commit is contained in:
Erik Språng
2019-02-26 18:31:00 +01:00
committed by Commit Bot
parent c9d0b08982
commit 6a7baa7d0f
10 changed files with 609 additions and 726 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
modules/video_coding/BUILD.gn
View File

@ -113,8 +113,6 @@ rtc_static_library("video_coding") {
"frame_object.h",
"generic_decoder.cc",
"generic_decoder.h",
"generic_encoder.cc",
"generic_encoder.h",
"h264_sprop_parameter_sets.cc",
"h264_sprop_parameter_sets.h",
"h264_sps_pps_tracker.cc",
@ -844,7 +842,6 @@ if (rtc_include_tests) {
"decoding_state_unittest.cc",
"fec_controller_unittest.cc",
"frame_buffer2_unittest.cc",
"generic_encoder_unittest.cc",
"h264_sprop_parameter_sets_unittest.cc",
"h264_sps_pps_tracker_unittest.cc",
"histogram_unittest.cc",

407
modules/video_coding/generic_encoder.cc
View File

@ -1,407 +0,0 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/generic_encoder.h"
#include <cstddef>
#include <cstdint>
#include <vector>
#include "absl/types/optional.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_content_type.h"
#include "api/video/video_frame_buffer.h"
#include "api/video/video_rotation.h"
#include "api/video/video_timing.h"
#include "modules/include/module_common_types_public.h"
#include "modules/video_coding/include/video_coding_defines.h"
#include "rtc_base/checks.h"
#include "rtc_base/experiments/alr_experiment.h"
#include "rtc_base/logging.h"
#include "rtc_base/time_utils.h"
#include "rtc_base/trace_event.h"
namespace webrtc {
namespace {
const int kMessagesThrottlingThreshold = 2;
const int kThrottleRatio = 100000;
} // namespace
VCMEncodedFrameCallback::TimingFramesLayerInfo::TimingFramesLayerInfo() {}
VCMEncodedFrameCallback::TimingFramesLayerInfo::~TimingFramesLayerInfo() {}
VCMGenericEncoder::VCMGenericEncoder(
VideoEncoder* encoder,
VCMEncodedFrameCallback* encoded_frame_callback,
bool internal_source)
: encoder_(encoder),
vcm_encoded_frame_callback_(encoded_frame_callback),
internal_source_(internal_source),
input_frame_rate_(0),
streams_or_svc_num_(0),
codec_type_(VideoCodecType::kVideoCodecGeneric) {}
VCMGenericEncoder::~VCMGenericEncoder() {}
int32_t VCMGenericEncoder::Release() {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
TRACE_EVENT0("webrtc", "VCMGenericEncoder::Release");
return encoder_->Release();
}
int32_t VCMGenericEncoder::InitEncode(const VideoCodec* settings,
int32_t number_of_cores,
size_t max_payload_size) {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
TRACE_EVENT0("webrtc", "VCMGenericEncoder::InitEncode");
streams_or_svc_num_ = settings->numberOfSimulcastStreams;
codec_type_ = settings->codecType;
if (settings->codecType == kVideoCodecVP9) {
streams_or_svc_num_ = settings->VP9().numberOfSpatialLayers;
}
if (streams_or_svc_num_ == 0)
streams_or_svc_num_ = 1;
vcm_encoded_frame_callback_->SetTimingFramesThresholds(
settings->timing_frame_thresholds);
vcm_encoded_frame_callback_->OnFrameRateChanged(settings->maxFramerate);
if (encoder_->InitEncode(settings, number_of_cores, max_payload_size) != 0) {
RTC_LOG(LS_ERROR) << "Failed to initialize the encoder associated with "
"codec type: "
<< CodecTypeToPayloadString(settings->codecType) << " ("
<< settings->codecType << ")";
return -1;
}
vcm_encoded_frame_callback_->Reset();
encoder_->RegisterEncodeCompleteCallback(vcm_encoded_frame_callback_);
return 0;
}
int32_t VCMGenericEncoder::Encode(const VideoFrame& frame,
const CodecSpecificInfo* codec_specific,
const std::vector<FrameType>& frame_types) {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
TRACE_EVENT1("webrtc", "VCMGenericEncoder::Encode", "timestamp",
frame.timestamp());
for (FrameType frame_type : frame_types)
RTC_DCHECK(frame_type == kVideoFrameKey || frame_type == kVideoFrameDelta);
for (size_t i = 0; i < streams_or_svc_num_; ++i)
vcm_encoded_frame_callback_->OnEncodeStarted(frame.timestamp(),
frame.render_time_ms(), i);
return encoder_->Encode(frame, codec_specific, &frame_types);
}
void VCMGenericEncoder::SetEncoderParameters(
const VideoBitrateAllocation& target_bitrate,
uint32_t input_frame_rate) {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
bool rates_have_changed;
{
rtc::CritScope lock(&params_lock_);
rates_have_changed = target_bitrate != bitrate_allocation_ ||
input_frame_rate != input_frame_rate_;
bitrate_allocation_ = target_bitrate;
input_frame_rate_ = input_frame_rate;
}
if (rates_have_changed) {
int res = encoder_->SetRateAllocation(target_bitrate, input_frame_rate);
if (res != 0) {
RTC_LOG(LS_WARNING) << "Error set encoder rate (total bitrate bps = "
<< target_bitrate.get_sum_bps()
<< ", framerate = " << input_frame_rate
<< "): " << res;
}
vcm_encoded_frame_callback_->OnFrameRateChanged(input_frame_rate);
for (size_t i = 0; i < streams_or_svc_num_; ++i) {
vcm_encoded_frame_callback_->OnTargetBitrateChanged(
target_bitrate.GetSpatialLayerSum(i) / 8, i);
}
}
}
int32_t VCMGenericEncoder::RequestFrame(
const std::vector<FrameType>& frame_types) {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
// TODO(nisse): Used only with internal source. Delete as soon as
// that feature is removed. The only implementation I've been able
// to find ignores what's in the frame. With one exception: It seems
// a few test cases, e.g.,
// VideoSendStreamTest.VideoSendStreamStopSetEncoderRateToZero, set
// internal_source to true and use FakeEncoder. And the latter will
// happily encode this 1x1 frame and pass it on down the pipeline.
return encoder_->Encode(VideoFrame::Builder()
.set_video_frame_buffer(I420Buffer::Create(1, 1))
.set_rotation(kVideoRotation_0)
.set_timestamp_us(0)
.build(),
NULL, &frame_types);
}
bool VCMGenericEncoder::InternalSource() const {
return internal_source_;
}
VideoEncoder::EncoderInfo VCMGenericEncoder::GetEncoderInfo() const {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
return encoder_->GetEncoderInfo();
}
VCMEncodedFrameCallback::VCMEncodedFrameCallback(
EncodedImageCallback* post_encode_callback)
: internal_source_(false),
post_encode_callback_(post_encode_callback),
framerate_(1),
last_timing_frame_time_ms_(-1),
timing_frames_thresholds_({-1, 0}),
incorrect_capture_time_logged_messages_(0),
reordered_frames_logged_messages_(0),
stalled_encoder_logged_messages_(0) {
absl::optional<AlrExperimentSettings> experiment_settings =
AlrExperimentSettings::CreateFromFieldTrial(
AlrExperimentSettings::kStrictPacingAndProbingExperimentName);
if (experiment_settings) {
experiment_groups_[0] = experiment_settings->group_id + 1;
} else {
experiment_groups_[0] = 0;
}
experiment_settings = AlrExperimentSettings::CreateFromFieldTrial(
AlrExperimentSettings::kScreenshareProbingBweExperimentName);
if (experiment_settings) {
experiment_groups_[1] = experiment_settings->group_id + 1;
} else {
experiment_groups_[1] = 0;
}
}
VCMEncodedFrameCallback::~VCMEncodedFrameCallback() {}
void VCMEncodedFrameCallback::OnTargetBitrateChanged(
size_t bitrate_bytes_per_second,
size_t simulcast_svc_idx) {
rtc::CritScope crit(&timing_params_lock_);
if (timing_frames_info_.size() < simulcast_svc_idx + 1)
timing_frames_info_.resize(simulcast_svc_idx + 1);
timing_frames_info_[simulcast_svc_idx].target_bitrate_bytes_per_sec =
bitrate_bytes_per_second;
}
void VCMEncodedFrameCallback::OnFrameRateChanged(size_t framerate) {
rtc::CritScope crit(&timing_params_lock_);
framerate_ = framerate;
}
void VCMEncodedFrameCallback::OnEncodeStarted(uint32_t rtp_timestamp,
int64_t capture_time_ms,
size_t simulcast_svc_idx) {
if (internal_source_) {
return;
}
rtc::CritScope crit(&timing_params_lock_);
if (timing_frames_info_.size() < simulcast_svc_idx + 1)
timing_frames_info_.resize(simulcast_svc_idx + 1);
RTC_DCHECK(
timing_frames_info_[simulcast_svc_idx].encode_start_list.empty() ||
rtc::TimeDiff(capture_time_ms, timing_frames_info_[simulcast_svc_idx]
.encode_start_list.back()
.capture_time_ms) >= 0);
// If stream is disabled due to low bandwidth OnEncodeStarted still will be
// called and have to be ignored.
if (timing_frames_info_[simulcast_svc_idx].target_bitrate_bytes_per_sec == 0)
return;
if (timing_frames_info_[simulcast_svc_idx].encode_start_list.size() ==
kMaxEncodeStartTimeListSize) {
++stalled_encoder_logged_messages_;
if (stalled_encoder_logged_messages_ <= kMessagesThrottlingThreshold ||
stalled_encoder_logged_messages_ % kThrottleRatio == 0) {
RTC_LOG(LS_WARNING) << "Too many frames in the encode_start_list."
" Did encoder stall?";
if (stalled_encoder_logged_messages_ == kMessagesThrottlingThreshold) {
RTC_LOG(LS_WARNING) << "Too many log messages. Further stalled encoder"
"warnings will be throttled.";
}
}
post_encode_callback_->OnDroppedFrame(DropReason::kDroppedByEncoder);
timing_frames_info_[simulcast_svc_idx].encode_start_list.pop_front();
}
timing_frames_info_[simulcast_svc_idx].encode_start_list.emplace_back(
rtp_timestamp, capture_time_ms, rtc::TimeMillis());
}
absl::optional<int64_t> VCMEncodedFrameCallback::ExtractEncodeStartTime(
size_t simulcast_svc_idx,
EncodedImage* encoded_image) {
absl::optional<int64_t> result;
size_t num_simulcast_svc_streams = timing_frames_info_.size();
if (simulcast_svc_idx < num_simulcast_svc_streams) {
auto encode_start_list =
&timing_frames_info_[simulcast_svc_idx].encode_start_list;
// Skip frames for which there was OnEncodeStarted but no OnEncodedImage
// call. These are dropped by encoder internally.
// Because some hardware encoders don't preserve capture timestamp we
// use RTP timestamps here.
while (!encode_start_list->empty() &&
IsNewerTimestamp(encoded_image->Timestamp(),
encode_start_list->front().rtp_timestamp)) {
post_encode_callback_->OnDroppedFrame(DropReason::kDroppedByEncoder);
encode_start_list->pop_front();
}
if (encode_start_list->size() > 0 &&
encode_start_list->front().rtp_timestamp ==
encoded_image->Timestamp()) {
result.emplace(encode_start_list->front().encode_start_time_ms);
if (encoded_image->capture_time_ms_ !=
encode_start_list->front().capture_time_ms) {
// Force correct capture timestamp.
encoded_image->capture_time_ms_ =
encode_start_list->front().capture_time_ms;
++incorrect_capture_time_logged_messages_;
if (incorrect_capture_time_logged_messages_ <=
kMessagesThrottlingThreshold ||
incorrect_capture_time_logged_messages_ % kThrottleRatio == 0) {
RTC_LOG(LS_WARNING)
<< "Encoder is not preserving capture timestamps.";
if (incorrect_capture_time_logged_messages_ ==
kMessagesThrottlingThreshold) {
RTC_LOG(LS_WARNING) << "Too many log messages. Further incorrect "
"timestamps warnings will be throttled.";
}
}
}
encode_start_list->pop_front();
} else {
++reordered_frames_logged_messages_;
if (reordered_frames_logged_messages_ <= kMessagesThrottlingThreshold ||
reordered_frames_logged_messages_ % kThrottleRatio == 0) {
RTC_LOG(LS_WARNING) << "Frame with no encode started time recordings. "
"Encoder may be reordering frames "
"or not preserving RTP timestamps.";
if (reordered_frames_logged_messages_ == kMessagesThrottlingThreshold) {
RTC_LOG(LS_WARNING) << "Too many log messages. Further frames "
"reordering warnings will be throttled.";
}
}
}
}
return result;
}
void VCMEncodedFrameCallback::FillTimingInfo(size_t simulcast_svc_idx,
EncodedImage* encoded_image) {
absl::optional<size_t> outlier_frame_size;
absl::optional<int64_t> encode_start_ms;
uint8_t timing_flags = VideoSendTiming::kNotTriggered;
{
rtc::CritScope crit(&timing_params_lock_);
// Encoders with internal sources do not call OnEncodeStarted
// |timing_frames_info_| may be not filled here.
if (!internal_source_) {
encode_start_ms =
ExtractEncodeStartTime(simulcast_svc_idx, encoded_image);
}
if (timing_frames_info_.size() > simulcast_svc_idx) {
size_t target_bitrate =
timing_frames_info_[simulcast_svc_idx].target_bitrate_bytes_per_sec;
if (framerate_ > 0 && target_bitrate > 0) {
// framerate and target bitrate were reported by encoder.
size_t average_frame_size = target_bitrate / framerate_;
outlier_frame_size.emplace(
average_frame_size *
timing_frames_thresholds_.outlier_ratio_percent / 100);
}
}
// Outliers trigger timing frames, but do not affect scheduled timing
// frames.
if (outlier_frame_size && encoded_image->size() >= *outlier_frame_size) {
timing_flags |= VideoSendTiming::kTriggeredBySize;
}
// Check if it's time to send a timing frame.
int64_t timing_frame_delay_ms =
encoded_image->capture_time_ms_ - last_timing_frame_time_ms_;
// Trigger threshold if it's a first frame, too long passed since the last
// timing frame, or we already sent timing frame on a different simulcast
// stream with the same capture time.
if (last_timing_frame_time_ms_ == -1 ||
timing_frame_delay_ms >= timing_frames_thresholds_.delay_ms ||
timing_frame_delay_ms == 0) {
timing_flags |= VideoSendTiming::kTriggeredByTimer;
last_timing_frame_time_ms_ = encoded_image->capture_time_ms_;
}
} // rtc::CritScope crit(&timing_params_lock_);
int64_t now_ms = rtc::TimeMillis();
// Workaround for chromoting encoder: it passes encode start and finished
// timestamps in |timing_| field, but they (together with capture timestamp)
// are not in the WebRTC clock.
if (internal_source_ && encoded_image->timing_.encode_finish_ms > 0 &&
encoded_image->timing_.encode_start_ms > 0) {
int64_t clock_offset_ms = now_ms - encoded_image->timing_.encode_finish_ms;
// Translate capture timestamp to local WebRTC clock.
encoded_image->capture_time_ms_ += clock_offset_ms;
encoded_image->SetTimestamp(
static_cast<uint32_t>(encoded_image->capture_time_ms_ * 90));
encode_start_ms.emplace(encoded_image->timing_.encode_start_ms +
clock_offset_ms);
}
// If encode start is not available that means that encoder uses internal
// source. In that case capture timestamp may be from a different clock with a
// drift relative to rtc::TimeMillis(). We can't use it for Timing frames,
// because to being sent in the network capture time required to be less than
// all the other timestamps.
if (encode_start_ms) {
encoded_image->SetEncodeTime(*encode_start_ms, now_ms);
encoded_image->timing_.flags = timing_flags;
} else {
encoded_image->timing_.flags = VideoSendTiming::kInvalid;
}
}
EncodedImageCallback::Result VCMEncodedFrameCallback::OnEncodedImage(
const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific,
const RTPFragmentationHeader* fragmentation_header) {
TRACE_EVENT_INSTANT1("webrtc", "VCMEncodedFrameCallback::Encoded",
"timestamp", encoded_image.Timestamp());
const size_t spatial_idx = encoded_image.SpatialIndex().value_or(0);
EncodedImage image_copy(encoded_image);
FillTimingInfo(spatial_idx, &image_copy);
// Piggyback ALR experiment group id and simulcast id into the content type.
uint8_t experiment_id =
experiment_groups_[videocontenttypehelpers::IsScreenshare(
image_copy.content_type_)];
// TODO(ilnik): This will force content type extension to be present even
// for realtime video. At the expense of miniscule overhead we will get
// sliced receive statistics.
RTC_CHECK(videocontenttypehelpers::SetExperimentId(&image_copy.content_type_,
experiment_id));
// We count simulcast streams from 1 on the wire. That's why we set simulcast
// id in content type to +1 of that is actual simulcast index. This is because
// value 0 on the wire is reserved for 'no simulcast stream specified'.
RTC_CHECK(videocontenttypehelpers::SetSimulcastId(
&image_copy.content_type_, static_cast<uint8_t>(spatial_idx + 1)));
return post_encode_callback_->OnEncodedImage(image_copy, codec_specific,
fragmentation_header);
}
} // namespace webrtc

153
modules/video_coding/generic_encoder.h
View File

@ -1,153 +0,0 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_GENERIC_ENCODER_H_
#define MODULES_VIDEO_CODING_GENERIC_ENCODER_H_
#include <stdio.h>
#include <list>
#include <vector>
#include "api/units/data_rate.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/include/video_coding_defines.h"
#include "rtc_base/critical_section.h"
#include "rtc_base/race_checker.h"
namespace webrtc {
class VCMEncodedFrameCallback : public EncodedImageCallback {
public:
explicit VCMEncodedFrameCallback(EncodedImageCallback* post_encode_callback);
~VCMEncodedFrameCallback() override;
// Implements EncodedImageCallback.
EncodedImageCallback::Result OnEncodedImage(
const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific_info,
const RTPFragmentationHeader* fragmentation) override;
void SetInternalSource(bool internal_source) {
internal_source_ = internal_source;
}
// Timing frames configuration methods. These 4 should be called before
// |OnEncodedImage| at least once.
void OnTargetBitrateChanged(size_t bitrate_bytes_per_sec,
size_t simulcast_svc_idx);
void OnFrameRateChanged(size_t framerate);
void OnEncodeStarted(uint32_t rtp_timestamps,
int64_t capture_time_ms,
size_t simulcast_svc_idx);
void SetTimingFramesThresholds(
const VideoCodec::TimingFrameTriggerThresholds& thresholds) {
rtc::CritScope crit(&timing_params_lock_);
timing_frames_thresholds_ = thresholds;
}
// Clears all data stored by OnEncodeStarted().
void Reset() {
rtc::CritScope crit(&timing_params_lock_);
timing_frames_info_.clear();
last_timing_frame_time_ms_ = -1;
reordered_frames_logged_messages_ = 0;
stalled_encoder_logged_messages_ = 0;
}
private:
// For non-internal-source encoders, returns encode started time and fixes
// capture timestamp for the frame, if corrupted by the encoder.
absl::optional<int64_t> ExtractEncodeStartTime(size_t simulcast_svc_idx,
EncodedImage* encoded_image)
RTC_EXCLUSIVE_LOCKS_REQUIRED(timing_params_lock_);
void FillTimingInfo(size_t simulcast_svc_idx, EncodedImage* encoded_image);
rtc::CriticalSection timing_params_lock_;
bool internal_source_;
EncodedImageCallback* const post_encode_callback_;
struct EncodeStartTimeRecord {
EncodeStartTimeRecord(uint32_t timestamp,
int64_t capture_time,
int64_t encode_start_time)
: rtp_timestamp(timestamp),
capture_time_ms(capture_time),
encode_start_time_ms(encode_start_time) {}
uint32_t rtp_timestamp;
int64_t capture_time_ms;
int64_t encode_start_time_ms;
};
struct TimingFramesLayerInfo {
TimingFramesLayerInfo();
~TimingFramesLayerInfo();
size_t target_bitrate_bytes_per_sec = 0;
std::list<EncodeStartTimeRecord> encode_start_list;
};
// Separate instance for each simulcast stream or spatial layer.
std::vector<TimingFramesLayerInfo> timing_frames_info_
RTC_GUARDED_BY(timing_params_lock_);
size_t framerate_ RTC_GUARDED_BY(timing_params_lock_);
int64_t last_timing_frame_time_ms_ RTC_GUARDED_BY(timing_params_lock_);
VideoCodec::TimingFrameTriggerThresholds timing_frames_thresholds_
RTC_GUARDED_BY(timing_params_lock_);
size_t incorrect_capture_time_logged_messages_
RTC_GUARDED_BY(timing_params_lock_);
size_t reordered_frames_logged_messages_ RTC_GUARDED_BY(timing_params_lock_);
size_t stalled_encoder_logged_messages_ RTC_GUARDED_BY(timing_params_lock_);
// Experiment groups parsed from field trials for realtime video ([0]) and
// screenshare ([1]). 0 means no group specified. Positive values are
// experiment group numbers incremented by 1.
uint8_t experiment_groups_[2];
};
class VCMGenericEncoder {
friend class VCMCodecDataBase;
public:
VCMGenericEncoder(VideoEncoder* encoder,
VCMEncodedFrameCallback* encoded_frame_callback,
bool internal_source);
~VCMGenericEncoder();
int32_t Release();
int32_t InitEncode(const VideoCodec* settings,
int32_t number_of_cores,
size_t max_payload_size);
int32_t Encode(const VideoFrame& frame,
const CodecSpecificInfo* codec_specific,
const std::vector<FrameType>& frame_types);
void SetEncoderParameters(const VideoBitrateAllocation& target_bitrate,
uint32_t input_frame_rate);
int32_t RequestFrame(const std::vector<FrameType>& frame_types);
bool InternalSource() const;
VideoEncoder::EncoderInfo GetEncoderInfo() const;
private:
rtc::RaceChecker race_checker_;
VideoEncoder* const encoder_ RTC_GUARDED_BY(race_checker_);
VCMEncodedFrameCallback* const vcm_encoded_frame_callback_;
const bool internal_source_;
rtc::CriticalSection params_lock_;
VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(params_lock_);
uint32_t input_frame_rate_ RTC_GUARDED_BY(params_lock_);
size_t streams_or_svc_num_ RTC_GUARDED_BY(race_checker_);
VideoCodecType codec_type_ RTC_GUARDED_BY(race_checker_);
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_GENERIC_ENCODER_H_

313
modules/video_coding/generic_encoder_unittest.cc
View File

@ -1,313 +0,0 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <cstddef>
#include <vector>
#include "api/video/video_timing.h"
#include "modules/video_coding/generic_encoder.h"
#include "modules/video_coding/include/video_coding_defines.h"
#include "rtc_base/fake_clock.h"
#include "rtc_base/time_utils.h"
#include "test/gtest.h"
namespace webrtc {
namespace test {
namespace {
inline size_t FrameSize(const size_t& min_frame_size,
const size_t& max_frame_size,
const int& s,
const int& i) {
return min_frame_size + (s + 1) * i % (max_frame_size - min_frame_size);
}
class FakeEncodedImageCallback : public EncodedImageCallback {
public:
FakeEncodedImageCallback()
: last_frame_was_timing_(false),
num_frames_dropped_(0),
last_capture_timestamp_(-1) {}
Result OnEncodedImage(const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific_info,
const RTPFragmentationHeader* fragmentation) override {
last_frame_was_timing_ =
encoded_image.timing_.flags != VideoSendTiming::kInvalid &&
encoded_image.timing_.flags != VideoSendTiming::kNotTriggered;
last_capture_timestamp_ = encoded_image.capture_time_ms_;
return Result(Result::OK);
}
void OnDroppedFrame(DropReason reason) override { ++num_frames_dropped_; }
bool WasTimingFrame() { return last_frame_was_timing_; }
size_t GetNumFramesDropped() { return num_frames_dropped_; }
int64_t GetLastCaptureTimestamp() { return last_capture_timestamp_; }
private:
bool last_frame_was_timing_;
size_t num_frames_dropped_;
int64_t last_capture_timestamp_;
};
enum class FrameType {
kNormal,
kTiming,
kDropped,
};
// Emulates |num_frames| on |num_streams| frames with capture timestamps
// increased by 1 from 0. Size of each frame is between
// |min_frame_size| and |max_frame_size|, outliers are counted relatevely to
// |average_frame_sizes[]| for each stream.
std::vector<std::vector<FrameType>> GetTimingFrames(
const int64_t delay_ms,
const size_t min_frame_size,
const size_t max_frame_size,
std::vector<size_t> average_frame_sizes,
const int num_streams,
const int num_frames) {
FakeEncodedImageCallback sink;
VCMEncodedFrameCallback callback(&sink);
const size_t kFramerate = 30;
callback.SetTimingFramesThresholds(
{delay_ms, kDefaultOutlierFrameSizePercent});
callback.OnFrameRateChanged(kFramerate);
int s, i;
std::vector<std::vector<FrameType>> result(num_streams);
for (s = 0; s < num_streams; ++s)
callback.OnTargetBitrateChanged(average_frame_sizes[s] * kFramerate, s);
int64_t current_timestamp = 0;
for (i = 0; i < num_frames; ++i) {
current_timestamp += 1;
for (s = 0; s < num_streams; ++s) {
// every (5+s)-th frame is dropped on s-th stream by design.
bool dropped = i % (5 + s) == 0;
EncodedImage image;
CodecSpecificInfo codec_specific;
image.Allocate(max_frame_size);
image.set_size(FrameSize(min_frame_size, max_frame_size, s, i));
image.capture_time_ms_ = current_timestamp;
image.SetTimestamp(static_cast<uint32_t>(current_timestamp * 90));
image.SetSpatialIndex(s);
codec_specific.codecType = kVideoCodecGeneric;
callback.OnEncodeStarted(static_cast<uint32_t>(current_timestamp * 90),
current_timestamp, s);
if (dropped) {
result[s].push_back(FrameType::kDropped);
continue;
}
callback.OnEncodedImage(image, &codec_specific, nullptr);
if (sink.WasTimingFrame()) {
result[s].push_back(FrameType::kTiming);
} else {
result[s].push_back(FrameType::kNormal);
}
}
}
return result;
}
} // namespace
TEST(TestVCMEncodedFrameCallback, MarksTimingFramesPeriodicallyTogether) {
const int64_t kDelayMs = 29;
const size_t kMinFrameSize = 10;
const size_t kMaxFrameSize = 20;
const int kNumFrames = 1000;
const int kNumStreams = 3;
// No outliers as 1000 is larger than anything from range [10,20].
const std::vector<size_t> kAverageSize = {1000, 1000, 1000};
auto frames = GetTimingFrames(kDelayMs, kMinFrameSize, kMaxFrameSize,
kAverageSize, kNumStreams, kNumFrames);
// Timing frames should be tirggered every delayMs.
// As no outliers are expected, frames on all streams have to be
// marked together.
int last_timing_frame = -1;
for (int i = 0; i < kNumFrames; ++i) {
int num_normal = 0;
int num_timing = 0;
int num_dropped = 0;
for (int s = 0; s < kNumStreams; ++s) {
if (frames[s][i] == FrameType::kTiming) {
++num_timing;
} else if (frames[s][i] == FrameType::kNormal) {
++num_normal;
} else {
++num_dropped;
}
}
// Can't have both normal and timing frames at the same timstamp.
EXPECT_TRUE(num_timing == 0 || num_normal == 0);
if (num_dropped < kNumStreams) {
if (last_timing_frame == -1 || i >= last_timing_frame + kDelayMs) {
// If didn't have timing frames for a period, current sent frame has to
// be one. No normal frames should be sent.
EXPECT_EQ(num_normal, 0);
} else {
// No unneeded timing frames should be sent.
EXPECT_EQ(num_timing, 0);
}
}
if (num_timing > 0)
last_timing_frame = i;
}
}
TEST(TestVCMEncodedFrameCallback, MarksOutliers) {
const int64_t kDelayMs = 29;
const size_t kMinFrameSize = 2495;
const size_t kMaxFrameSize = 2505;
const int kNumFrames = 1000;
const int kNumStreams = 3;
// Possible outliers as 1000 lies in range [995, 1005].
const std::vector<size_t> kAverageSize = {998, 1000, 1004};
auto frames = GetTimingFrames(kDelayMs, kMinFrameSize, kMaxFrameSize,
kAverageSize, kNumStreams, kNumFrames);
// All outliers should be marked.
for (int i = 0; i < kNumFrames; ++i) {
for (int s = 0; s < kNumStreams; ++s) {
if (FrameSize(kMinFrameSize, kMaxFrameSize, s, i) >=
kAverageSize[s] * kDefaultOutlierFrameSizePercent / 100) {
// Too big frame. May be dropped or timing, but not normal.
EXPECT_NE(frames[s][i], FrameType::kNormal);
}
}
}
}
TEST(TestVCMEncodedFrameCallback, NoTimingFrameIfNoEncodeStartTime) {
EncodedImage image;
CodecSpecificInfo codec_specific;
int64_t timestamp = 1;
constexpr size_t kFrameSize = 500;
image.Allocate(kFrameSize);
image.set_size(kFrameSize);
image.capture_time_ms_ = timestamp;
image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
codec_specific.codecType = kVideoCodecGeneric;
FakeEncodedImageCallback sink;
VCMEncodedFrameCallback callback(&sink);
VideoCodec::TimingFrameTriggerThresholds thresholds;
thresholds.delay_ms = 1; // Make all frames timing frames.
callback.SetTimingFramesThresholds(thresholds);
callback.OnTargetBitrateChanged(500, 0);
// Verify a single frame works with encode start time set.
callback.OnEncodeStarted(static_cast<uint32_t>(timestamp * 90), timestamp, 0);
callback.OnEncodedImage(image, &codec_specific, nullptr);
EXPECT_TRUE(sink.WasTimingFrame());
// New frame, now skip OnEncodeStarted. Should not result in timing frame.
image.capture_time_ms_ = ++timestamp;
image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
callback.OnEncodedImage(image, &codec_specific, nullptr);
EXPECT_FALSE(sink.WasTimingFrame());
}
TEST(TestVCMEncodedFrameCallback, AdjustsCaptureTimeForInternalSourceEncoder) {
rtc::ScopedFakeClock clock;
clock.SetTimeMicros(1234567);
EncodedImage image;
CodecSpecificInfo codec_specific;
const int64_t kEncodeStartDelayMs = 2;
const int64_t kEncodeFinishDelayMs = 10;
int64_t timestamp = 1;
constexpr size_t kFrameSize = 500;
image.Allocate(kFrameSize);
image.set_size(kFrameSize);
image.capture_time_ms_ = timestamp;
image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
codec_specific.codecType = kVideoCodecGeneric;
FakeEncodedImageCallback sink;
VCMEncodedFrameCallback callback(&sink);
callback.SetInternalSource(true);
VideoCodec::TimingFrameTriggerThresholds thresholds;
thresholds.delay_ms = 1; // Make all frames timing frames.
callback.SetTimingFramesThresholds(thresholds);
callback.OnTargetBitrateChanged(500, 0);
// Verify a single frame without encode timestamps isn't a timing frame.
callback.OnEncodedImage(image, &codec_specific, nullptr);
EXPECT_FALSE(sink.WasTimingFrame());
// New frame, but this time with encode timestamps set in timing_.
// This should be a timing frame.
image.capture_time_ms_ = ++timestamp;
image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
image.timing_.encode_start_ms = timestamp + kEncodeStartDelayMs;
image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
callback.OnEncodedImage(image, &codec_specific, nullptr);
EXPECT_TRUE(sink.WasTimingFrame());
// Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
// capture timestamp should be kEncodeFinishDelayMs in the past.
EXPECT_EQ(
sink.GetLastCaptureTimestamp(),
clock.TimeNanos() / rtc::kNumNanosecsPerMillisec - kEncodeFinishDelayMs);
}
TEST(TestVCMEncodedFrameCallback, NotifiesAboutDroppedFrames) {
EncodedImage image;
CodecSpecificInfo codec_specific;
const int64_t kTimestampMs1 = 47721840;
const int64_t kTimestampMs2 = 47721850;
const int64_t kTimestampMs3 = 47721860;
const int64_t kTimestampMs4 = 47721870;
codec_specific.codecType = kVideoCodecGeneric;
FakeEncodedImageCallback sink;
VCMEncodedFrameCallback callback(&sink);
// Any non-zero bitrate needed to be set before the first frame.
callback.OnTargetBitrateChanged(500, 0);
image.capture_time_ms_ = kTimestampMs1;
image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
callback.OnEncodeStarted(image.Timestamp(), image.capture_time_ms_, 0);
EXPECT_EQ(0u, sink.GetNumFramesDropped());
callback.OnEncodedImage(image, &codec_specific, nullptr);
image.capture_time_ms_ = kTimestampMs2;
image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
callback.OnEncodeStarted(image.Timestamp(), image.capture_time_ms_, 0);
// No OnEncodedImageCall for timestamp2. Yet, at this moment it's not known
// that frame with timestamp2 was dropped.
EXPECT_EQ(0u, sink.GetNumFramesDropped());
image.capture_time_ms_ = kTimestampMs3;
image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
callback.OnEncodeStarted(image.Timestamp(), image.capture_time_ms_, 0);
callback.OnEncodedImage(image, &codec_specific, nullptr);
EXPECT_EQ(1u, sink.GetNumFramesDropped());
image.capture_time_ms_ = kTimestampMs4;
image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
callback.OnEncodeStarted(image.Timestamp(), image.capture_time_ms_, 0);
callback.OnEncodedImage(image, &codec_specific, nullptr);
EXPECT_EQ(1u, sink.GetNumFramesDropped());
}
TEST(TestVCMEncodedFrameCallback, RestoresCaptureTimestamps) {
EncodedImage image;
CodecSpecificInfo codec_specific;
const int64_t kTimestampMs = 123456;
codec_specific.codecType = kVideoCodecGeneric;
FakeEncodedImageCallback sink;
VCMEncodedFrameCallback callback(&sink);
// Any non-zero bitrate needed to be set before the first frame.
callback.OnTargetBitrateChanged(500, 0);
image.capture_time_ms_ = kTimestampMs; // Incorrect timesetamp.
image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
callback.OnEncodeStarted(image.Timestamp(), image.capture_time_ms_, 0);
image.capture_time_ms_ = 0; // Incorrect timesetamp.
callback.OnEncodedImage(image, &codec_specific, nullptr);
EXPECT_EQ(kTimestampMs, sink.GetLastCaptureTimestamp());
}
} // namespace test
} // namespace webrtc

1
modules/video_coding/video_coding_impl.h
View File

@ -21,7 +21,6 @@
#include "modules/video_coding/decoder_database.h"
#include "modules/video_coding/frame_buffer.h"
#include "modules/video_coding/generic_decoder.h"
#include "modules/video_coding/generic_encoder.h"
#include "modules/video_coding/jitter_buffer.h"
#include "modules/video_coding/receiver.h"
#include "modules/video_coding/timing.h"