zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for optional fields in FixedLengthDeltaEncoder

Browse Source 
Optional fields are those which only occur sometimes. For example,
the sequence number field in an RTP packet always occurs, but
fields in optional RTP extensions only occur sometimes.

Bug: webrtc:8111
Change-Id: Iff2c35b73530c0a1db68e547b4caf34434aa4ace
Reviewed-on: https://webrtc-review.googlesource.com/c/103362
Commit-Queue: Elad Alon <eladalon@webrtc.org>
Reviewed-by: Björn Terelius <terelius@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#25336}
This commit is contained in:
Elad Alon
2018-10-24 14:06:34 +02:00
committed by Commit Bot
parent c6ec4b1fa9
commit 257ed437f0
3 changed files with 355 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

233
logging/rtc_event_log/encoder/delta_encoding.cc
View File

@ -188,7 +188,6 @@ class FixedLengthEncodingParameters final {
// Whether the values of the sequence are optional. That is, it may be
// that some of them do not have a value (not even a sentinel value indicating
// invalidity).
// TODO(eladalon): Add support for optional elements.
bool values_optional() const { return values_optional_; }
// Number of bits necessary to hold the largest value in the sequence.
@ -226,18 +225,20 @@ class FixedLengthDeltaEncoder final {
// determine whether it was produced by FixedLengthDeltaEncoder, and can
// therefore be decoded by FixedLengthDeltaDecoder, or whether it was produced
// by a different encoder.
static std::string EncodeDeltas(uint64_t base,
const std::vector<uint64_t>& values);
static std::string EncodeDeltas(
absl::optional<uint64_t> base,
const std::vector<absl::optional<uint64_t>>& values);
private:
// Calculate min/max values of unsigned/signed deltas, given the bit width
// of all the values in the series.
static void CalculateMinAndMaxDeltas(uint64_t base,
const std::vector<uint64_t>& values,
uint64_t bit_width,
uint64_t* max_unsigned_delta,
uint64_t* max_pos_signed_delta,
uint64_t* min_neg_signed_delta);
static void CalculateMinAndMaxDeltas(
uint64_t base,
const std::vector<absl::optional<uint64_t>>& values,
uint64_t bit_width,
uint64_t* max_unsigned_delta,
uint64_t* max_pos_signed_delta,
uint64_t* min_neg_signed_delta);
// No effect outside of unit tests.
// In unit tests, may lead to forcing signed/unsigned deltas, etc.
@ -251,17 +252,18 @@ class FixedLengthDeltaEncoder final {
// Therefore, it was deemed acceptable to let them have a reference to
// |values|, whose lifetime must exceed the lifetime of |this|.
FixedLengthDeltaEncoder(const FixedLengthEncodingParameters& params,
uint64_t base,
const std::vector<uint64_t>& values);
absl::optional<uint64_t> base,
const std::vector<absl::optional<uint64_t>>& values,
size_t existent_values_count);
// Perform delta-encoding using the parameters given to the ctor on the
// sequence of values given to the ctor.
std::string Encode();
// Exact lengths.
size_t OutputLengthBytes() const;
size_t OutputLengthBytes(size_t existent_values_count) const;
size_t HeaderLengthBits() const;
size_t EncodedDeltasLengthBits() const;
size_t EncodedDeltasLengthBits(size_t existent_values_count) const;
// Encode the compression parameters into the stream.
void EncodeHeader();
@ -278,11 +280,11 @@ class FixedLengthDeltaEncoder final {
// The encoding scheme assumes that at least one value is transmitted OOB,
// so that the first value can be encoded as a delta from that OOB value,
// which is |base_|.
const uint64_t base_;
const absl::optional<uint64_t> base_;
// The values to be encoded.
// Note: This is a non-owning reference. See comment above ctor for details.
const std::vector<uint64_t>& values_;
const std::vector<absl::optional<uint64_t>>& values_;
// Buffer into which encoded values will be written.
// This is created dynmically as a way to enforce that the rest of the
@ -295,15 +297,34 @@ class FixedLengthDeltaEncoder final {
// TODO(eladalon): Reduce the number of passes.
std::string FixedLengthDeltaEncoder::EncodeDeltas(
uint64_t base,
const std::vector<uint64_t>& values) {
absl::optional<uint64_t> base,
const std::vector<absl::optional<uint64_t>>& values) {
RTC_DCHECK(!values.empty());
bool non_decreasing = base <= values[0];
uint64_t max_value_including_base = std::max(base, values[0]);
for (size_t i = 1; i < values.size(); ++i) {
non_decreasing &= (values[i - 1] <= values[i]);
max_value_including_base = std::max(max_value_including_base, values[i]);
// As a special case, if all of the elements are identical to the base,
// (including, for optional fields, about their existence/non-existence),
// the empty string is used to signal that.
if (std::all_of(
values.cbegin(), values.cend(),
[base](absl::optional<uint64_t> val) { return val == base; })) {
return std::string();
}
const uint64_t base_val = base.value_or(0u);
bool non_decreasing = true;
uint64_t max_value_including_base = base_val;
uint64_t previous = base_val;
size_t existent_values_count = 0;
for (size_t i = 0; i < values.size(); ++i) {
if (!values[i].has_value()) {
continue;
}
++existent_values_count;
non_decreasing &= (previous <= values[i].value());
max_value_including_base =
std::max(max_value_including_base, values[i].value());
previous = values[i].value();
}
// If the sequence is non-decreasing, it may be assumed to have width = 64;
@ -314,16 +335,9 @@ std::string FixedLengthDeltaEncoder::EncodeDeltas(
uint64_t max_unsigned_delta;
uint64_t max_pos_signed_delta;
uint64_t min_neg_signed_delta;
CalculateMinAndMaxDeltas(base, values, value_width_bits, &max_unsigned_delta,
&max_pos_signed_delta, &min_neg_signed_delta);
// We indicate the special case of all values being equal to the base with
// the empty string.
if (max_unsigned_delta == 0) {
RTC_DCHECK(std::all_of(values.cbegin(), values.cend(),
[base](uint64_t val) { return val == base; }));
return std::string();
}
CalculateMinAndMaxDeltas(base_val, values, value_width_bits,
&max_unsigned_delta, &max_pos_signed_delta,
&min_neg_signed_delta);
const uint64_t delta_width_bits_unsigned =
UnsignedBitWidth(max_unsigned_delta);
@ -336,7 +350,7 @@ std::string FixedLengthDeltaEncoder::EncodeDeltas(
const uint64_t delta_width_bits =
signed_deltas ? delta_width_bits_signed : delta_width_bits_unsigned;
const bool values_optional = false;
const bool values_optional = (existent_values_count < values.size());
FixedLengthEncodingParameters params(delta_width_bits, signed_deltas,
values_optional, value_width_bits);
@ -345,13 +359,13 @@ std::string FixedLengthDeltaEncoder::EncodeDeltas(
ConsiderTestOverrides(&params, delta_width_bits_signed,
delta_width_bits_unsigned);
FixedLengthDeltaEncoder encoder(params, base, values);
FixedLengthDeltaEncoder encoder(params, base, values, existent_values_count);
return encoder.Encode();
}
void FixedLengthDeltaEncoder::CalculateMinAndMaxDeltas(
uint64_t base,
const std::vector<uint64_t>& values,
const std::vector<absl::optional<uint64_t>>& values,
uint64_t bit_width,
uint64_t* max_unsigned_delta_out,
uint64_t* max_pos_signed_delta_out,
@ -369,8 +383,14 @@ void FixedLengthDeltaEncoder::CalculateMinAndMaxDeltas(
uint64_t prev = base;
for (size_t i = 0; i < values.size(); ++i) {
const uint64_t forward_delta = UnsignedDelta(prev, values[i], bit_mask);
const uint64_t backward_delta = UnsignedDelta(values[i], prev, bit_mask);
if (!values[i].has_value()) {
continue;
}
const uint64_t current = values[i].value();
const uint64_t forward_delta = UnsignedDelta(prev, current, bit_mask);
const uint64_t backward_delta = UnsignedDelta(current, prev, bit_mask);
max_unsigned_delta = std::max(max_unsigned_delta, forward_delta);
@ -380,7 +400,7 @@ void FixedLengthDeltaEncoder::CalculateMinAndMaxDeltas(
min_neg_signed_delta = std::max(min_neg_signed_delta, backward_delta);
}
prev = values[i];
prev = current;
}
*max_unsigned_delta_out = max_unsigned_delta;
@ -405,28 +425,42 @@ void FixedLengthDeltaEncoder::ConsiderTestOverrides(
FixedLengthDeltaEncoder::FixedLengthDeltaEncoder(
const FixedLengthEncodingParameters& params,
uint64_t base,
const std::vector<uint64_t>& values)
absl::optional<uint64_t> base,
const std::vector<absl::optional<uint64_t>>& values,
size_t existent_values_count)
: params_(params), base_(base), values_(values) {
RTC_DCHECK(!values_.empty());
writer_ = absl::make_unique<BitWriter>(OutputLengthBytes());
writer_ =
absl::make_unique<BitWriter>(OutputLengthBytes(existent_values_count));
}
std::string FixedLengthDeltaEncoder::Encode() {
EncodeHeader();
uint64_t previous = base_;
for (uint64_t value : values_) {
EncodeDelta(previous, value);
previous = value;
if (params_.values_optional()) {
// Encode which values exist and which don't.
for (absl::optional<uint64_t> value : values_) {
writer_->WriteBits(value.has_value() ? 1u : 0u, 1);
}
}
uint64_t previous = base_.has_value() ? base_.value() : 0u;
for (absl::optional<uint64_t> value : values_) {
if (!value.has_value()) {
RTC_DCHECK(params_.values_optional());
continue;
}
EncodeDelta(previous, value.value());
previous = value.value();
}
return writer_->GetString();
}
size_t FixedLengthDeltaEncoder::OutputLengthBytes() const {
const size_t length_bits = HeaderLengthBits() + EncodedDeltasLengthBits();
return BitsToBytes(length_bits);
size_t FixedLengthDeltaEncoder::OutputLengthBytes(
size_t existent_values_count) const {
return BitsToBytes(HeaderLengthBits() +
EncodedDeltasLengthBits(existent_values_count));
}
size_t FixedLengthDeltaEncoder::HeaderLengthBits() const {
@ -441,11 +475,21 @@ size_t FixedLengthDeltaEncoder::HeaderLengthBits() const {
}
}
size_t FixedLengthDeltaEncoder::EncodedDeltasLengthBits() const {
// TODO(eladalon): When optional values are supported, iterate over the
// deltas to check the exact cost of each.
RTC_DCHECK(!params_.values_optional());
return values_.size() * params_.delta_width_bits();
size_t FixedLengthDeltaEncoder::EncodedDeltasLengthBits(
size_t existent_values_count) const {
if (params_.values_optional()) {
RTC_DCHECK_EQ(std::count_if(values_.begin(), values_.end(),
[](absl::optional<uint64_t> val) {
return val.has_value();
}),
existent_values_count);
// One bit for each delta, to indicate if the value exists, and delta_width
// for each existent value, to indicate the delta itself.
return (1 * values_.size()) +
(params_.delta_width_bits() * existent_values_count);
} else {
return values_.size() * params_.delta_width_bits();
}
}
void FixedLengthDeltaEncoder::EncodeHeader() {
@ -534,9 +578,10 @@ class FixedLengthDeltaDecoder final {
// original values, this will return the sequence of original values.
// If an error occurs (can happen if |input| is corrupt), an empty
// vector will be returned.
static std::vector<uint64_t> DecodeDeltas(const std::string& input,
uint64_t base,
size_t num_of_deltas);
static std::vector<absl::optional<uint64_t>> DecodeDeltas(
const std::string& input,
absl::optional<uint64_t> base,
size_t num_of_deltas);
private:
// Reads the encoding header in |input| and returns a FixedLengthDeltaDecoder
@ -547,8 +592,10 @@ class FixedLengthDeltaDecoder final {
// When a valid FixedLengthDeltaDecoder is returned, this does not mean that
// the entire stream is free of error. Rather, only the encoding header is
// examined and guaranteed.
static std::unique_ptr<FixedLengthDeltaDecoder>
Create(const std::string& input, uint64_t base, size_t num_of_deltas);
static std::unique_ptr<FixedLengthDeltaDecoder> Create(
const std::string& input,
absl::optional<uint64_t> base,
size_t num_of_deltas);
// FixedLengthDeltaDecoder objects are to be created by DecodeDeltas() and
// released by it before it returns. They're mostly a convenient way to
@ -558,11 +605,11 @@ class FixedLengthDeltaDecoder final {
// of |reader|'s underlying buffer.
FixedLengthDeltaDecoder(std::unique_ptr<rtc::BitBuffer> reader,
const FixedLengthEncodingParameters& params,
uint64_t base,
absl::optional<uint64_t> base,
size_t num_of_deltas);
// Perform the decoding using the parameters given to the ctor.
std::vector<uint64_t> Decode();
std::vector<absl::optional<uint64_t>> Decode();
// Attempt to parse a delta from the input reader.
// Returns true/false for success/failure.
@ -590,7 +637,7 @@ class FixedLengthDeltaDecoder final {
// The encoding scheme assumes that at least one value is transmitted OOB,
// so that the first value can be encoded as a delta from that OOB value,
// which is |base_|.
const uint64_t base_;
const absl::optional<uint64_t> base_;
// The number of values to be known to be decoded.
const size_t num_of_deltas_;
@ -618,13 +665,13 @@ bool FixedLengthDeltaDecoder::IsSuitableDecoderFor(const std::string& input) {
EncodingType::kFixedSizeSignedDeltasEarlyWrapAndOptSupported;
}
std::vector<uint64_t> FixedLengthDeltaDecoder::DecodeDeltas(
std::vector<absl::optional<uint64_t>> FixedLengthDeltaDecoder::DecodeDeltas(
const std::string& input,
uint64_t base,
absl::optional<uint64_t> base,
size_t num_of_deltas) {
auto decoder = FixedLengthDeltaDecoder::Create(input, base, num_of_deltas);
if (!decoder) {
return std::vector<uint64_t>();
return std::vector<absl::optional<uint64_t>>();
}
return decoder->Decode();
@ -632,7 +679,7 @@ std::vector<uint64_t> FixedLengthDeltaDecoder::DecodeDeltas(
std::unique_ptr<FixedLengthDeltaDecoder> FixedLengthDeltaDecoder::Create(
const std::string& input,
uint64_t base,
absl::optional<uint64_t> base,
size_t num_of_deltas) {
if (input.length() < kBitsInHeaderForEncodingType) {
return nullptr;
@ -685,10 +732,6 @@ std::unique_ptr<FixedLengthDeltaDecoder> FixedLengthDeltaDecoder::Create(
}
RTC_DCHECK_LE(read_buffer, 1);
values_optional = rtc::dchecked_cast<bool>(read_buffer);
if (values_optional) {
RTC_LOG(LS_WARNING) << "Not implemented.";
return nullptr;
}
// value_width_bits
if (!reader->ReadBits(&read_buffer, kBitsInHeaderForValueWidthBits)) {
@ -716,28 +759,48 @@ std::unique_ptr<FixedLengthDeltaDecoder> FixedLengthDeltaDecoder::Create(
FixedLengthDeltaDecoder::FixedLengthDeltaDecoder(
std::unique_ptr<rtc::BitBuffer> reader,
const FixedLengthEncodingParameters& params,
uint64_t base,
absl::optional<uint64_t> base,
size_t num_of_deltas)
: reader_(std::move(reader)),
params_(params),
base_(base),
num_of_deltas_(num_of_deltas) {
RTC_DCHECK(reader_);
// TODO(eladalon): Support optional values.
RTC_DCHECK(!params.values_optional()) << "Not implemented.";
}
std::vector<uint64_t> FixedLengthDeltaDecoder::Decode() {
std::vector<uint64_t> values(num_of_deltas_);
std::vector<absl::optional<uint64_t>> FixedLengthDeltaDecoder::Decode() {
RTC_DCHECK(reader_);
uint64_t previous = base_;
std::vector<bool> existing_values(num_of_deltas_);
if (params_.values_optional()) {
for (size_t i = 0; i < num_of_deltas_; ++i) {
uint32_t exists;
if (!reader_->ReadBits(&exists, 1u)) {
RTC_LOG(LS_WARNING) << "Failed to read existence-indicating bit.";
return std::vector<absl::optional<uint64_t>>();
}
RTC_DCHECK_LE(exists, 1u);
existing_values[i] = (exists == 1);
}
} else {
std::fill(existing_values.begin(), existing_values.end(), true);
}
std::vector<absl::optional<uint64_t>> values(num_of_deltas_);
uint64_t previous = base_.has_value() ? base_.value() : 0u;
for (size_t i = 0; i < num_of_deltas_; ++i) {
if (!existing_values[i]) {
RTC_DCHECK(params_.values_optional());
continue;
}
uint64_t delta;
if (!ParseDelta(&delta)) {
return std::vector<uint64_t>();
return std::vector<absl::optional<uint64_t>>();
}
values[i] = ApplyDelta(previous, delta);
previous = values[i];
previous = values[i].value();
}
return values;
@ -745,7 +808,6 @@ std::vector<uint64_t> FixedLengthDeltaDecoder::Decode() {
bool FixedLengthDeltaDecoder::ParseDelta(uint64_t* delta) {
RTC_DCHECK(reader_);
RTC_DCHECK(!params_.values_optional()) << "Not implemented."; // Reminder.
// BitBuffer and BitBufferWriter read/write higher bits before lower bits.
@ -781,7 +843,6 @@ bool FixedLengthDeltaDecoder::ParseDelta(uint64_t* delta) {
uint64_t FixedLengthDeltaDecoder::ApplyDelta(uint64_t base,
uint64_t delta) const {
RTC_DCHECK(!params_.values_optional()) << "Not implemented."; // Reminder.
RTC_DCHECK_LE(base, MaxUnsignedValueOfBitWidth(params_.value_width_bits()));
RTC_DCHECK_LE(delta, MaxUnsignedValueOfBitWidth(params_.delta_width_bits()));
return params_.signed_deltas() ? ApplySignedDelta(base, delta)
@ -816,20 +877,22 @@ uint64_t FixedLengthDeltaDecoder::ApplySignedDelta(uint64_t base,
} // namespace
std::string EncodeDeltas(uint64_t base, const std::vector<uint64_t>& values) {
std::string EncodeDeltas(absl::optional<uint64_t> base,
const std::vector<absl::optional<uint64_t>>& values) {
// TODO(eladalon): Support additional encodings.
return FixedLengthDeltaEncoder::EncodeDeltas(base, values);
}
std::vector<uint64_t> DecodeDeltas(const std::string& input,
uint64_t base,
size_t num_of_deltas) {
std::vector<absl::optional<uint64_t>> DecodeDeltas(
const std::string& input,
absl::optional<uint64_t> base,
size_t num_of_deltas) {
RTC_DCHECK_GT(num_of_deltas, 0); // Allows empty vector to indicate error.
// The empty string is a special case indicating that all values were equal
// to the base.
if (input.empty()) {
std::vector<uint64_t> result(num_of_deltas);
std::vector<absl::optional<uint64_t>> result(num_of_deltas);
std::fill(result.begin(), result.end(), base);
return result;
}
@ -839,7 +902,7 @@ std::vector<uint64_t> DecodeDeltas(const std::string& input,
}
RTC_LOG(LS_WARNING) << "Could not decode delta-encoded stream.";
return std::vector<uint64_t>();
return std::vector<absl::optional<uint64_t>>();
}
void SetFixedLengthEncoderDeltaSignednessForTesting(bool signedness) {