zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extract simulcast rate allocation outside of video encoder.

Browse Source 
This is a first step to refactor this code.
I'm deprecating https://codereview.webrtc.org/1913073002 and
implementing this in smaller more isolated steps.

BUG=webrtc:5206
R=asapersson@webrtc.org, kjellander@webrtc.org, noahric@chromium.org

Review URL: https://codereview.webrtc.org/2288223002 .

Cr-Commit-Position: refs/heads/master@{#14186}
This commit is contained in:
Erik Språng
2016-09-12 16:04:43 +02:00
parent 7b11c65a17
commit 78ce619a0c
11 changed files with 374 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc
View File

@ -17,6 +17,7 @@
#include "webrtc/base/checks.h"
#include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h"
#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h"
#include "webrtc/system_wrappers/include/clock.h"
namespace {
@ -26,14 +27,6 @@ const unsigned int kDefaultMaxQp = 56;
// Max qp for lowest spatial resolution when doing simulcast.
const unsigned int kLowestResMaxQp = 45;
uint32_t SumStreamTargetBitrate(int streams, const webrtc::VideoCodec& codec) {
uint32_t bitrate_sum = 0;
for (int i = 0; i < streams; ++i) {
bitrate_sum += codec.simulcastStream[i].targetBitrate;
}
return bitrate_sum;
}
uint32_t SumStreamMaxBitrate(int streams, const webrtc::VideoCodec& codec) {
uint32_t bitrate_sum = 0;
for (int i = 0; i < streams; ++i) {
@ -92,13 +85,8 @@ int VerifyCodec(const webrtc::VideoCodec* inst) {
return WEBRTC_VIDEO_CODEC_OK;
}
// TL1 FrameDropper's max time to drop frames.
const float kTl1MaxTimeToDropFrames = 20.0f;
struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory {
ScreenshareTemporalLayersFactory()
: tl1_frame_dropper_(kTl1MaxTimeToDropFrames) {}
ScreenshareTemporalLayersFactory() {}
virtual ~ScreenshareTemporalLayersFactory() {}
virtual webrtc::TemporalLayers* Create(int num_temporal_layers,
@ -106,9 +94,6 @@ struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory {
return new webrtc::ScreenshareLayers(num_temporal_layers, rand(),
webrtc::Clock::GetRealTimeClock());
}
mutable webrtc::FrameDropper tl0_frame_dropper_;
mutable webrtc::FrameDropper tl1_frame_dropper_;
};
// An EncodedImageCallback implementation that forwards on calls to a
@ -139,9 +124,10 @@ namespace webrtc {
SimulcastEncoderAdapter::SimulcastEncoderAdapter(VideoEncoderFactory* factory)
: factory_(factory),
encoded_complete_callback_(NULL),
encoded_complete_callback_(nullptr),
implementation_name_("SimulcastEncoderAdapter") {
memset(&codec_, 0, sizeof(webrtc::VideoCodec));
rate_allocator_.reset(new SimulcastRateAllocator(codec_));
}
SimulcastEncoderAdapter::~SimulcastEncoderAdapter() {
@ -189,6 +175,9 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst,
}
codec_ = *inst;
rate_allocator_.reset(new SimulcastRateAllocator(codec_));
std::vector<uint32_t> start_bitrates =
rate_allocator_->GetAllocation(codec_.startBitrate);
// Special mode when screensharing on a single stream.
if (number_of_streams == 1 && inst->mode == kScreensharing) {
@ -200,15 +189,18 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst,
// Create |number_of_streams| of encoder instances and init them.
for (int i = 0; i < number_of_streams; ++i) {
VideoCodec stream_codec;
bool send_stream = true;
uint32_t start_bitrate_kbps = start_bitrates[i];
if (!doing_simulcast) {
stream_codec = codec_;
stream_codec.numberOfSimulcastStreams = 1;
} else {
// Cap start bitrate to the min bitrate in order to avoid strange codec
// behavior. Since sending sending will be false, this should not matter.
start_bitrate_kbps =
std::max(codec_.simulcastStream[i].minBitrate, start_bitrate_kbps);
bool highest_resolution_stream = (i == (number_of_streams - 1));
PopulateStreamCodec(&codec_, i, number_of_streams,
highest_resolution_stream, &stream_codec,
&send_stream);
PopulateStreamCodec(&codec_, i, start_bitrate_kbps,
highest_resolution_stream, &stream_codec);
}
// TODO(ronghuawu): Remove once this is handled in VP8EncoderImpl.
@ -225,7 +217,8 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst,
EncodedImageCallback* callback = new AdapterEncodedImageCallback(this, i);
encoder->RegisterEncodeCompleteCallback(callback);
streaminfos_.push_back(StreamInfo(encoder, callback, stream_codec.width,
stream_codec.height, send_stream));
stream_codec.height,
start_bitrate_kbps > 0));
if (i != 0)
implementation_name += ", ";
implementation_name += streaminfos_[i].encoder->ImplementationName();
@ -363,6 +356,8 @@ int SimulcastEncoderAdapter::SetRates(uint32_t new_bitrate_kbit,
if (codec_.maxBitrate > 0 && new_bitrate_kbit > codec_.maxBitrate) {
new_bitrate_kbit = codec_.maxBitrate;
}
std::vector<uint32_t> stream_bitrates;
if (new_bitrate_kbit > 0) {
// Make sure the bitrate fits the configured min bitrates. 0 is a special
// value that means paused, though, so leave it alone.
@ -373,19 +368,20 @@ int SimulcastEncoderAdapter::SetRates(uint32_t new_bitrate_kbit,
new_bitrate_kbit < codec_.simulcastStream[0].minBitrate) {
new_bitrate_kbit = codec_.simulcastStream[0].minBitrate;
}
stream_bitrates = rate_allocator_->GetAllocation(new_bitrate_kbit);
}
codec_.maxFramerate = new_framerate;
bool send_stream = true;
uint32_t stream_bitrate = 0;
// Disable any stream not in the current allocation.
stream_bitrates.resize(streaminfos_.size(), 0U);
for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) {
stream_bitrate = GetStreamBitrate(stream_idx, streaminfos_.size(),
new_bitrate_kbit, &send_stream);
uint32_t stream_bitrate_kbps = stream_bitrates[stream_idx];
// Need a key frame if we have not sent this stream before.
if (send_stream && !streaminfos_[stream_idx].send_stream) {
if (stream_bitrate_kbps > 0 && !streaminfos_[stream_idx].send_stream) {
streaminfos_[stream_idx].key_frame_request = true;
}
streaminfos_[stream_idx].send_stream = send_stream;
streaminfos_[stream_idx].send_stream = stream_bitrate_kbps > 0;
// TODO(holmer): This is a temporary hack for screensharing, where we
// interpret the startBitrate as the encoder target bitrate. This is
@ -395,14 +391,15 @@ int SimulcastEncoderAdapter::SetRates(uint32_t new_bitrate_kbit,
if (codec_.targetBitrate > 0 &&
(codec_.codecSpecific.VP8.numberOfTemporalLayers == 2 ||
codec_.simulcastStream[0].numberOfTemporalLayers == 2)) {
stream_bitrate = std::min(codec_.maxBitrate, stream_bitrate);
stream_bitrate_kbps = std::min(codec_.maxBitrate, stream_bitrate_kbps);
// TODO(ronghuawu): Can't change max bitrate via the VideoEncoder
// interface. And VP8EncoderImpl doesn't take negative framerate.
// max_bitrate = std::min(codec_.maxBitrate, stream_bitrate);
// max_bitrate = std::min(codec_.maxBitrate, stream_bitrate_kbps);
// new_framerate = -1;
}
streaminfos_[stream_idx].encoder->SetRates(stream_bitrate, new_framerate);
streaminfos_[stream_idx].encoder->SetRates(stream_bitrate_kbps,
new_framerate);
}
return WEBRTC_VIDEO_CODEC_OK;
@ -422,61 +419,12 @@ EncodedImageCallback::Result SimulcastEncoderAdapter::OnEncodedImage(
encodedImage, &stream_codec_specific, fragmentation);
}
uint32_t SimulcastEncoderAdapter::GetStreamBitrate(
int stream_idx,
size_t total_number_of_streams,
uint32_t new_bitrate_kbit,
bool* send_stream) const {
if (total_number_of_streams == 1) {
*send_stream = true;
return new_bitrate_kbit;
}
// The bitrate needed to start sending this stream is given by the
// minimum bitrate allowed for encoding this stream, plus the sum target
// rates of all lower streams.
uint32_t sum_target_lower_streams =
SumStreamTargetBitrate(stream_idx, codec_);
uint32_t bitrate_to_send_this_layer =
codec_.simulcastStream[stream_idx].minBitrate + sum_target_lower_streams;
if (new_bitrate_kbit >= bitrate_to_send_this_layer) {
// We have enough bandwidth to send this stream.
*send_stream = true;
// Bitrate for this stream is the new bitrate (|new_bitrate_kbit|) minus the
// sum target rates of the lower streams, and capped to a maximum bitrate.
// The maximum cap depends on whether we send the next higher stream.
// If we will be sending the next higher stream, |max_rate| is given by
// current stream's |targetBitrate|, otherwise it's capped by |maxBitrate|.
if (stream_idx < codec_.numberOfSimulcastStreams - 1) {
unsigned int max_rate = codec_.simulcastStream[stream_idx].maxBitrate;
if (new_bitrate_kbit >=
SumStreamTargetBitrate(stream_idx + 1, codec_) +
codec_.simulcastStream[stream_idx + 1].minBitrate) {
max_rate = codec_.simulcastStream[stream_idx].targetBitrate;
}
return std::min(new_bitrate_kbit - sum_target_lower_streams, max_rate);
} else {
// For the highest stream (highest resolution), the |targetBitRate| and
// |maxBitrate| are not used. Any excess bitrate (above the targets of
// all lower streams) is given to this (highest resolution) stream.
return new_bitrate_kbit - sum_target_lower_streams;
}
} else {
// Not enough bitrate for this stream.
// Return our max bitrate of |stream_idx| - 1, but we don't send it. We need
// to keep this resolution coding in order for the multi-encoder to work.
*send_stream = false;
return codec_.simulcastStream[stream_idx - 1].maxBitrate;
}
}
void SimulcastEncoderAdapter::PopulateStreamCodec(
const webrtc::VideoCodec* inst,
int stream_index,
size_t total_number_of_streams,
uint32_t start_bitrate_kbps,
bool highest_resolution_stream,
webrtc::VideoCodec* stream_codec,
bool* send_stream) {
webrtc::VideoCodec* stream_codec) {
*stream_codec = *inst;
// Stream specific settings.
@ -505,9 +453,7 @@ void SimulcastEncoderAdapter::PopulateStreamCodec(
}
// TODO(ronghuawu): what to do with targetBitrate.
int stream_bitrate = GetStreamBitrate(stream_index, total_number_of_streams,
inst->startBitrate, send_stream);
stream_codec->startBitrate = stream_bitrate;
stream_codec->startBitrate = start_bitrate_kbps;
}
bool SimulcastEncoderAdapter::Initialized() const {

17
webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h
View File

@ -20,6 +20,8 @@
namespace webrtc {
class SimulcastRateAllocator;
class VideoEncoderFactory {
public:
virtual VideoEncoder* Create() = 0;
@ -91,22 +93,12 @@ class SimulcastEncoderAdapter : public VP8Encoder {
bool send_stream;
};
// Get the stream bitrate, for the stream |stream_idx|, given the bitrate
// |new_bitrate_kbit| and the actual configured stream count in
// |total_number_of_streams|. The function also returns whether there's enough
// bandwidth to send this stream via |send_stream|.
uint32_t GetStreamBitrate(int stream_idx,
size_t total_number_of_streams,
uint32_t new_bitrate_kbit,
bool* send_stream) const;
// Populate the codec settings for each stream.
void PopulateStreamCodec(const webrtc::VideoCodec* inst,
int stream_index,
size_t total_number_of_streams,
uint32_t start_bitrate_kbps,
bool highest_resolution_stream,
webrtc::VideoCodec* stream_codec,
bool* send_stream);
webrtc::VideoCodec* stream_codec);
bool Initialized() const;
@ -116,6 +108,7 @@ class SimulcastEncoderAdapter : public VP8Encoder {
std::vector<StreamInfo> streaminfos_;
EncodedImageCallback* encoded_complete_callback_;
std::string implementation_name_;
std::unique_ptr<SimulcastRateAllocator> rate_allocator_;
};
} // namespace webrtc

57
webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
View File

@ -29,6 +29,7 @@
#include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h"
#include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h"
#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h"
#include "webrtc/system_wrappers/include/clock.h"
namespace webrtc {
@ -59,46 +60,6 @@ int GCD(int a, int b) {
return b;
}
std::vector<int> GetStreamBitratesKbps(const VideoCodec& codec,
int bitrate_to_allocate_kbps) {
if (codec.numberOfSimulcastStreams <= 1) {
return std::vector<int>(1, bitrate_to_allocate_kbps);
}
std::vector<int> bitrates_kbps(codec.numberOfSimulcastStreams);
// Allocate min -> target bitrates as long as we have bitrate to spend.
size_t last_active_stream = 0;
for (size_t i = 0; i < static_cast<size_t>(codec.numberOfSimulcastStreams) &&
bitrate_to_allocate_kbps >=
static_cast<int>(codec.simulcastStream[i].minBitrate);
++i) {
last_active_stream = i;
int allocated_bitrate_kbps =
std::min(static_cast<int>(codec.simulcastStream[i].targetBitrate),
bitrate_to_allocate_kbps);
bitrates_kbps[i] = allocated_bitrate_kbps;
bitrate_to_allocate_kbps -= allocated_bitrate_kbps;
}
// Spend additional bits on the highest-quality active layer, up to max
// bitrate.
// TODO(pbos): Consider spending additional bits on last_active_stream-1 down
// to 0 and not just the top layer when we have additional bitrate to spend.
int allocated_bitrate_kbps = std::min(
static_cast<int>(codec.simulcastStream[last_active_stream].maxBitrate -
bitrates_kbps[last_active_stream]),
bitrate_to_allocate_kbps);
bitrates_kbps[last_active_stream] += allocated_bitrate_kbps;
bitrate_to_allocate_kbps -= allocated_bitrate_kbps;
// Make sure we can always send something. Suspending below min bitrate is
// controlled outside the codec implementation and is not overriden by this.
if (bitrates_kbps[0] < static_cast<int>(codec.simulcastStream[0].minBitrate))
bitrates_kbps[0] = static_cast<int>(codec.simulcastStream[0].minBitrate);
return bitrates_kbps;
}
uint32_t SumStreamMaxBitrate(int streams, const VideoCodec& codec) {
uint32_t bitrate_sum = 0;
for (int i = 0; i < streams; ++i) {
@ -149,10 +110,9 @@ VP8Decoder* VP8Decoder::Create() {
return new VP8DecoderImpl();
}
const float kTl1MaxTimeToDropFrames = 20.0f;
VP8EncoderImpl::VP8EncoderImpl()
: encoded_complete_callback_(NULL),
: encoded_complete_callback_(nullptr),
rate_allocator_(new SimulcastRateAllocator(codec_)),
inited_(false),
timestamp_(0),
feedback_mode_(false),
@ -162,8 +122,6 @@ VP8EncoderImpl::VP8EncoderImpl()
token_partitions_(VP8_ONE_TOKENPARTITION),
down_scale_requested_(false),
down_scale_bitrate_(0),
tl0_frame_dropper_(),
tl1_frame_dropper_(kTl1MaxTimeToDropFrames),
key_frame_request_(kMaxSimulcastStreams, false),
quality_scaler_enabled_(false) {
uint32_t seed = rtc::Time32();
@ -273,8 +231,8 @@ int VP8EncoderImpl::SetRates(uint32_t new_bitrate_kbit,
}
}
std::vector<int> stream_bitrates =
GetStreamBitratesKbps(codec_, new_bitrate_kbit);
std::vector<uint32_t> stream_bitrates =
rate_allocator_->GetAllocation(new_bitrate_kbit);
size_t stream_idx = encoders_.size() - 1;
for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) {
if (encoders_.size() > 1)
@ -403,6 +361,7 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst,
timestamp_ = 0;
codec_ = *inst;
rate_allocator_.reset(new SimulcastRateAllocator(codec_));
// Code expects simulcastStream resolutions to be correct, make sure they are
// filled even when there are no simulcast layers.
@ -570,8 +529,8 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst,
// Note the order we use is different from webm, we have lowest resolution
// at position 0 and they have highest resolution at position 0.
int stream_idx = encoders_.size() - 1;
std::vector<int> stream_bitrates =
GetStreamBitratesKbps(codec_, inst->startBitrate);
std::vector<uint32_t> stream_bitrates =
rate_allocator_->GetAllocation(inst->startBitrate);
SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx);
configurations_[0].rc_target_bitrate = stream_bitrates[stream_idx];
temporal_layers_[stream_idx]->ConfigureBitrates(

6
webrtc/modules/video_coding/codecs/vp8/vp8_impl.h
View File

@ -13,6 +13,7 @@
#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_VP8_IMPL_H_
#define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_VP8_IMPL_H_
#include <memory>
#include <vector>
// NOTE: This include order must remain to avoid compile errors, even though
@ -26,12 +27,12 @@
#include "webrtc/modules/video_coding/include/video_codec_interface.h"
#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
#include "webrtc/modules/video_coding/codecs/vp8/reference_picture_selection.h"
#include "webrtc/modules/video_coding/utility/frame_dropper.h"
#include "webrtc/modules/video_coding/utility/quality_scaler.h"
#include "webrtc/video_frame.h"
namespace webrtc {
class SimulcastRateAllocator;
class TemporalLayers;
class VP8EncoderImpl : public VP8Encoder {
@ -93,6 +94,7 @@ class VP8EncoderImpl : public VP8Encoder {
EncodedImageCallback* encoded_complete_callback_;
VideoCodec codec_;
std::unique_ptr<SimulcastRateAllocator> rate_allocator_;
bool inited_;
int64_t timestamp_;
bool feedback_mode_;
@ -104,8 +106,6 @@ class VP8EncoderImpl : public VP8Encoder {
std::vector<TemporalLayers*> temporal_layers_;
bool down_scale_requested_;
uint32_t down_scale_bitrate_;
FrameDropper tl0_frame_dropper_;
FrameDropper tl1_frame_dropper_;
std::vector<uint16_t> picture_id_;
std::vector<int> last_key_frame_picture_id_;
std::vector<bool> key_frame_request_;