zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add spatial layers support to video analyze pipeline.

Browse Source 
To support analyze of spatial layers we will continue sending them
into the network on encoder side, but will mark which should be then
discarded and which should be processed. On decoder side we will drop
layers, if they should be discarded and decode only parts, that
should be processed.

Bug: webrtc:10138
Change-Id: Ic8b8fe7787674c0ec49b879fcc29e54e8e3d787f
Reviewed-on: https://webrtc-review.googlesource.com/c/123185
Reviewed-by: Peter Slatala <psla@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26784}
This commit is contained in:
Artem Titov
2019-02-20 21:13:14 +01:00
committed by Commit Bot
parent 8e689203e0
commit 32232e92f3
27 changed files with 851 additions and 486 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
test/pc/e2e/BUILD.gn
View File

@ -12,13 +12,13 @@ group("e2e") {
testonly = true
deps = [
":default_encoded_image_id_injector",
":encoded_image_id_injector_api",
":default_encoded_image_data_injector",
":encoded_image_data_injector_api",
":example_video_quality_analyzer",
":id_generator",
":quality_analyzing_video_decoder",
":quality_analyzing_video_encoder",
":single_process_encoded_image_id_injector",
":single_process_encoded_image_data_injector",
]
if (rtc_include_tests) {
deps += [
@ -34,17 +34,17 @@ if (rtc_include_tests) {
testonly = true
deps = [
":default_encoded_image_id_injector_unittest",
":default_encoded_image_data_injector_unittest",
":peer_connection_e2e_smoke_test",
":single_process_encoded_image_id_injector_unittest",
":single_process_encoded_image_data_injector_unittest",
]
}
}
rtc_source_set("encoded_image_id_injector_api") {
rtc_source_set("encoded_image_data_injector_api") {
visibility = [ "*" ]
sources = [
"analyzer/video/encoded_image_id_injector.h",
"analyzer/video/encoded_image_data_injector.h",
]
deps = [
@ -52,15 +52,15 @@ rtc_source_set("encoded_image_id_injector_api") {
]
}
rtc_source_set("default_encoded_image_id_injector") {
rtc_source_set("default_encoded_image_data_injector") {
visibility = [ "*" ]
sources = [
"analyzer/video/default_encoded_image_id_injector.cc",
"analyzer/video/default_encoded_image_id_injector.h",
"analyzer/video/default_encoded_image_data_injector.cc",
"analyzer/video/default_encoded_image_data_injector.h",
]
deps = [
":encoded_image_id_injector_api",
":encoded_image_data_injector_api",
"../../../api/video:encoded_image",
"../../../rtc_base:checks",
"../../../rtc_base:criticalsection",
@ -68,15 +68,15 @@ rtc_source_set("default_encoded_image_id_injector") {
]
}
rtc_source_set("single_process_encoded_image_id_injector") {
rtc_source_set("single_process_encoded_image_data_injector") {
visibility = [ "*" ]
sources = [
"analyzer/video/single_process_encoded_image_id_injector.cc",
"analyzer/video/single_process_encoded_image_id_injector.h",
"analyzer/video/single_process_encoded_image_data_injector.cc",
"analyzer/video/single_process_encoded_image_data_injector.h",
]
deps = [
":encoded_image_id_injector_api",
":encoded_image_data_injector_api",
"../../../api/video:encoded_image",
"../../../rtc_base:checks",
"../../../rtc_base:criticalsection",
@ -100,7 +100,7 @@ rtc_source_set("quality_analyzing_video_decoder") {
"analyzer/video/quality_analyzing_video_decoder.h",
]
deps = [
":encoded_image_id_injector_api",
":encoded_image_data_injector_api",
":id_generator",
"../../../api/video:encoded_image",
"../../../api/video:video_frame",
@ -121,7 +121,7 @@ rtc_source_set("quality_analyzing_video_encoder") {
"analyzer/video/quality_analyzing_video_encoder.h",
]
deps = [
":encoded_image_id_injector_api",
":encoded_image_data_injector_api",
":id_generator",
"../../../api/video:encoded_image",
"../../../api/video:video_frame",
@ -143,7 +143,7 @@ if (rtc_include_tests) {
"analyzer/video/video_quality_analyzer_injection_helper.h",
]
deps = [
":encoded_image_id_injector_api",
":encoded_image_data_injector_api",
":id_generator",
":quality_analyzing_video_decoder",
":quality_analyzing_video_encoder",
@ -164,8 +164,8 @@ if (rtc_include_tests) {
"test_peer.h",
]
deps = [
":default_encoded_image_id_injector",
":encoded_image_id_injector_api",
":default_encoded_image_data_injector",
":encoded_image_data_injector_api",
":example_video_quality_analyzer",
":video_quality_analyzer_injection_helper",
"../../../api:array_view",
@ -206,7 +206,7 @@ if (rtc_include_tests) {
]
deps = [
":example_video_quality_analyzer",
":single_process_encoded_image_id_injector",
":single_process_encoded_image_data_injector",
":test_peer",
":video_quality_analyzer_injection_helper",
"../../../api:libjingle_peerconnection_api",
@ -231,26 +231,26 @@ if (rtc_include_tests) {
}
}
rtc_source_set("single_process_encoded_image_id_injector_unittest") {
rtc_source_set("single_process_encoded_image_data_injector_unittest") {
testonly = true
sources = [
"analyzer/video/single_process_encoded_image_id_injector_unittest.cc",
"analyzer/video/single_process_encoded_image_data_injector_unittest.cc",
]
deps = [
":single_process_encoded_image_id_injector",
":single_process_encoded_image_data_injector",
"../../../api/video:encoded_image",
"../../../rtc_base:rtc_base_approved",
"../../../test:test_support",
]
}
rtc_source_set("default_encoded_image_id_injector_unittest") {
rtc_source_set("default_encoded_image_data_injector_unittest") {
testonly = true
sources = [
"analyzer/video/default_encoded_image_id_injector_unittest.cc",
"analyzer/video/default_encoded_image_data_injector_unittest.cc",
]
deps = [
":default_encoded_image_id_injector",
":default_encoded_image_data_injector",
"../../../api/video:encoded_image",
"../../../rtc_base:rtc_base_approved",
"../../../test:test_support",

45
test/pc/e2e/analyzer/video/default_encoded_image_id_injector.cc → test/pc/e2e/analyzer/video/default_encoded_image_data_injector.cc
View File

@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h"
#include "test/pc/e2e/analyzer/video/default_encoded_image_data_injector.h"
#include <cstddef>
@ -32,16 +32,18 @@ constexpr size_t kBuffersPoolPerCodingEntity = 256;
} // namespace
DefaultEncodedImageIdInjector::DefaultEncodedImageIdInjector() {
DefaultEncodedImageDataInjector::DefaultEncodedImageDataInjector() {
for (size_t i = 0;
i < kPreInitCodingEntitiesCount * kBuffersPoolPerCodingEntity; ++i) {
bufs_pool_.push_back(
absl::make_unique<std::vector<uint8_t>>(kInitialBufferSize));
}
}
DefaultEncodedImageIdInjector::~DefaultEncodedImageIdInjector() = default;
DefaultEncodedImageDataInjector::~DefaultEncodedImageDataInjector() = default;
EncodedImage DefaultEncodedImageIdInjector::InjectId(uint16_t id,
EncodedImage DefaultEncodedImageDataInjector::InjectData(
uint16_t id,
bool discard,
const EncodedImage& source,
int coding_entity_id) {
ExtendIfRequired(coding_entity_id);
@ -57,14 +59,18 @@ EncodedImage DefaultEncodedImageIdInjector::InjectId(uint16_t id,
source.size());
out.data()[0] = id & 0x00ff;
out.data()[1] = (id & 0xff00) >> 8;
out.data()[2] = source.size() & 0x00ff;
out.data()[3] = (source.size() & 0xff00) >> 8;
out.data()[4] = (source.size() & 0xff00) >> 16;
out.data()[5] = (source.size() & 0xff00) >> 24;
out.data()[2] = source.size() & 0x000000ff;
out.data()[3] = (source.size() & 0x0000ff00) >> 8;
out.data()[4] = (source.size() & 0x00ff0000) >> 16;
out.data()[5] = (source.size() & 0xff000000) >> 24;
// We will store discard flag in the high bit of high byte of the size.
RTC_CHECK_LT(source.size(), 1U << 31) << "High bit is already in use";
out.data()[5] = out.data()[5] | ((discard ? 1 : 0) << 7);
return out;
}
EncodedImageWithId DefaultEncodedImageIdInjector::ExtractId(
EncodedImageExtractionResult DefaultEncodedImageDataInjector::ExtractData(
const EncodedImage& source,
int coding_entity_id) {
ExtendIfRequired(coding_entity_id);
@ -79,6 +85,7 @@ EncodedImageWithId DefaultEncodedImageIdInjector::ExtractId(
size_t source_pos = 0;
size_t out_pos = 0;
absl::optional<uint16_t> id = absl::nullopt;
bool discard = true;
while (source_pos < source.size()) {
RTC_CHECK_LE(source_pos + kEncodedImageBufferExpansion, source.size());
uint16_t next_id =
@ -89,21 +96,29 @@ EncodedImageWithId DefaultEncodedImageIdInjector::ExtractId(
id = next_id;
uint32_t length = source.data()[source_pos + 2] +
(source.data()[source_pos + 3] << 8) +
(source.data()[source_pos + 3] << 16) +
(source.data()[source_pos + 3] << 24);
(source.data()[source_pos + 4] << 16) +
((source.data()[source_pos + 5] << 24) & 0b01111111);
bool current_discard = (source.data()[source_pos + 5] & 0b10000000) != 0;
RTC_CHECK_LE(source_pos + kEncodedImageBufferExpansion + length,
source.size());
if (!current_discard) {
// Copy next encoded image payload from concatenated buffer only if it is
// not discarded.
memcpy(&out.data()[out_pos],
&source.data()[source_pos + kEncodedImageBufferExpansion], length);
source_pos += length + kEncodedImageBufferExpansion;
out_pos += length;
}
source_pos += length + kEncodedImageBufferExpansion;
// Extraction result is discarded only if all encoded partitions are
// discarded.
discard = discard && current_discard;
}
out.set_size(out_pos);
return EncodedImageWithId{id.value(), out};
return EncodedImageExtractionResult{id.value(), out, discard};
}
void DefaultEncodedImageIdInjector::ExtendIfRequired(int coding_entity_id) {
void DefaultEncodedImageDataInjector::ExtendIfRequired(int coding_entity_id) {
rtc::CritScope crit(&lock_);
if (coding_entities_.find(coding_entity_id) != coding_entities_.end()) {
// This entity is already known for this injector, so buffers are allocated.
@ -124,7 +139,7 @@ void DefaultEncodedImageIdInjector::ExtendIfRequired(int coding_entity_id) {
}
}
std::vector<uint8_t>* DefaultEncodedImageIdInjector::NextBuffer() {
std::vector<uint8_t>* DefaultEncodedImageDataInjector::NextBuffer() {
rtc::CritScope crit(&lock_);
// Get buffer from the front of the queue, return it to the caller and
// put in the back

45
test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h → test/pc/e2e/analyzer/video/default_encoded_image_data_injector.h
View File

@ -8,8 +8,8 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_ENCODED_IMAGE_ID_INJECTOR_H_
#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_ENCODED_IMAGE_ID_INJECTOR_H_
#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_ENCODED_IMAGE_DATA_INJECTOR_H_
#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_ENCODED_IMAGE_DATA_INJECTOR_H_
#include <cstdint>
#include <deque>
@ -20,20 +20,21 @@
#include "api/video/encoded_image.h"
#include "rtc_base/critical_section.h"
#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h"
#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h"
namespace webrtc {
namespace test {
// Injects frame id into EncodedImage payload buffer. The payload buffer will
// be prepended in the injector with 2 bytes frame id and 4 bytes original
// buffer length. It is assumed, that frame's data can't be more then 2^32
// bytes. In the decoder, frame id will be extracted and the length will be used
// to restore original buffer.
// Injects frame id and discard flag into EncodedImage payload buffer. The
// payload buffer will be prepended in the injector with 2 bytes frame id and 4
// bytes original buffer length. Discarded flag will be put into the highest bit
// of the length. It is assumed, that frame's data can't be more then 2^31
// bytes. In the decoder, frame id and discard flag will be extracted and the
// length will be used to restore original buffer.
//
// The data in the EncodedImage on encoder side after injection will look like
// this:
// 4 bytes frame length
// 4 bytes frame length + discard flag
// _ _ _↓_ _ _ _________________
// | | | original buffer |
// ¯↑¯ ¯ ¯ ¯ ¯ ¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯
@ -48,10 +49,12 @@ namespace test {
// 1. pos = 0
// 2. Extract id from buf[pos] and buf[pos + 1]
// 3. Extract length from buf[pos + 2]..buf[pos + 5]
// 4. Copy |length| bytes starting from buf[pos + 6] to output buffer.
// 5. pos = pos + length + 6
// 6. If pos < buf.length - go to the step 2.
// Also it will be checked, that all extracted ids are equals.
// 4. Extract discard flag from length highest bit.
// 5. If discard flag is False, copy |length| bytes starting from buf[pos + 6]
// to output buffer.
// 6. pos = pos + length + 6
// 7. If pos < buf.length - go to the step 2.
// Also it will check, that all extracted ids are equals.
//
// Because EncodedImage doesn't take ownership of its buffer, injector will keep
// ownership of the buffers that will be used for EncodedImages with injected
@ -67,16 +70,18 @@ namespace test {
// preallocate buffers for 2 coding entities, so 512 buffers with initial size
// 2KB. If in some point of time bigger buffer will be required, it will be also
// extended.
class DefaultEncodedImageIdInjector : public EncodedImageIdInjector,
public EncodedImageIdExtractor {
class DefaultEncodedImageDataInjector : public EncodedImageDataInjector,
public EncodedImageDataExtractor {
public:
DefaultEncodedImageIdInjector();
~DefaultEncodedImageIdInjector() override;
DefaultEncodedImageDataInjector();
~DefaultEncodedImageDataInjector() override;
EncodedImage InjectId(uint16_t id,
// TODO(titovartem) add support for discard injection and update the doc.
EncodedImage InjectData(uint16_t id,
bool discard,
const EncodedImage& source,
int coding_entity_id) override;
EncodedImageWithId ExtractId(const EncodedImage& source,
EncodedImageExtractionResult ExtractData(const EncodedImage& source,
int coding_entity_id) override;
private:
@ -99,4 +104,4 @@ class DefaultEncodedImageIdInjector : public EncodedImageIdInjector,
} // namespace test
} // namespace webrtc
#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_ENCODED_IMAGE_ID_INJECTOR_H_
#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_ENCODED_IMAGE_DATA_INJECTOR_H_

191
test/pc/e2e/analyzer/video/default_encoded_image_data_injector_unittest.cc Normal file
View File

@ -0,0 +1,191 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "test/pc/e2e/analyzer/video/default_encoded_image_data_injector.h"
#include <utility>
#include "api/video/encoded_image.h"
#include "rtc_base/buffer.h"
#include "test/gtest.h"
namespace webrtc {
namespace test {
namespace {
rtc::Buffer CreateBufferOfSizeNFilledWithValuesFromX(size_t n, uint8_t x) {
rtc::Buffer buffer(n);
for (size_t i = 0; i < n; ++i) {
buffer[i] = static_cast<uint8_t>(x + i);
}
return buffer;
}
} // namespace
TEST(DefaultEncodedImageDataInjector, InjectExtractDiscardFalse) {
DefaultEncodedImageDataInjector injector;
rtc::Buffer buffer = CreateBufferOfSizeNFilledWithValuesFromX(10, 1);
EncodedImage source(buffer.data(), 10, 10);
source.SetTimestamp(123456789);
EncodedImageExtractionResult out =
injector.ExtractData(injector.InjectData(512, false, source, 1), 2);
EXPECT_EQ(out.id, 512);
EXPECT_FALSE(out.discard);
EXPECT_EQ(out.image.size(), 10ul);
for (int i = 0; i < 10; ++i) {
EXPECT_EQ(out.image.data()[i], i + 1);
}
}
TEST(DefaultEncodedImageDataInjector, InjectExtractDiscardTrue) {
DefaultEncodedImageDataInjector injector;
rtc::Buffer buffer = CreateBufferOfSizeNFilledWithValuesFromX(10, 1);
EncodedImage source(buffer.data(), 10, 10);
source.SetTimestamp(123456789);
EncodedImageExtractionResult out =
injector.ExtractData(injector.InjectData(512, true, source, 1), 2);
EXPECT_EQ(out.id, 512);
EXPECT_TRUE(out.discard);
EXPECT_EQ(out.image.size(), 0ul);
}
TEST(DefaultEncodedImageDataInjector, Inject3Extract3) {
DefaultEncodedImageDataInjector injector;
rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1);
rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11);
rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21);
// 1st frame
EncodedImage source1(buffer1.data(), 10, 10);
source1.SetTimestamp(123456710);
// 2nd frame 1st spatial layer
EncodedImage source2(buffer2.data(), 10, 10);
source2.SetTimestamp(123456720);
// 2nd frame 2nd spatial layer
EncodedImage source3(buffer3.data(), 10, 10);
source3.SetTimestamp(123456720);
EncodedImage intermediate1 = injector.InjectData(510, false, source1, 1);
EncodedImage intermediate2 = injector.InjectData(520, true, source2, 1);
EncodedImage intermediate3 = injector.InjectData(520, false, source3, 1);
// Extract ids in different order.
EncodedImageExtractionResult out3 = injector.ExtractData(intermediate3, 2);
EncodedImageExtractionResult out1 = injector.ExtractData(intermediate1, 2);
EncodedImageExtractionResult out2 = injector.ExtractData(intermediate2, 2);
EXPECT_EQ(out1.id, 510);
EXPECT_FALSE(out1.discard);
EXPECT_EQ(out1.image.size(), 10ul);
for (int i = 0; i < 10; ++i) {
EXPECT_EQ(out1.image.data()[i], i + 1);
}
EXPECT_EQ(out2.id, 520);
EXPECT_TRUE(out2.discard);
EXPECT_EQ(out2.image.size(), 0ul);
EXPECT_EQ(out3.id, 520);
EXPECT_FALSE(out3.discard);
EXPECT_EQ(out3.image.size(), 10ul);
for (int i = 0; i < 10; ++i) {
EXPECT_EQ(out3.image.data()[i], i + 21);
}
}
TEST(DefaultEncodedImageDataInjector, InjectExtractFromConcatenated) {
DefaultEncodedImageDataInjector injector;
rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1);
rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11);
rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21);
EncodedImage source1(buffer1.data(), 10, 10);
source1.SetTimestamp(123456710);
EncodedImage source2(buffer2.data(), 10, 10);
source2.SetTimestamp(123456710);
EncodedImage source3(buffer3.data(), 10, 10);
source3.SetTimestamp(123456710);
// Inject id into 3 images with same frame id.
EncodedImage intermediate1 = injector.InjectData(512, false, source1, 1);
EncodedImage intermediate2 = injector.InjectData(512, true, source2, 1);
EncodedImage intermediate3 = injector.InjectData(512, false, source3, 1);
// Concatenate them into single encoded image, like it can be done in jitter
// buffer.
size_t concatenated_length =
intermediate1.size() + intermediate2.size() + intermediate3.size();
rtc::Buffer concatenated_buffer;
concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size());
concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size());
concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size());
EncodedImage concatenated(concatenated_buffer.data(), concatenated_length,
concatenated_length);
// Extract frame id from concatenated image
EncodedImageExtractionResult out = injector.ExtractData(concatenated, 2);
EXPECT_EQ(out.id, 512);
EXPECT_FALSE(out.discard);
EXPECT_EQ(out.image.size(), 2 * 10ul);
for (int i = 0; i < 10; ++i) {
EXPECT_EQ(out.image.data()[i], i + 1);
EXPECT_EQ(out.image.data()[i + 10], i + 21);
}
}
TEST(DefaultEncodedImageDataInjector,
InjectExtractFromConcatenatedAllDiscarded) {
DefaultEncodedImageDataInjector injector;
rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1);
rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11);
rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21);
EncodedImage source1(buffer1.data(), 10, 10);
source1.SetTimestamp(123456710);
EncodedImage source2(buffer2.data(), 10, 10);
source2.SetTimestamp(123456710);
EncodedImage source3(buffer3.data(), 10, 10);
source3.SetTimestamp(123456710);
// Inject id into 3 images with same frame id.
EncodedImage intermediate1 = injector.InjectData(512, true, source1, 1);
EncodedImage intermediate2 = injector.InjectData(512, true, source2, 1);
EncodedImage intermediate3 = injector.InjectData(512, true, source3, 1);
// Concatenate them into single encoded image, like it can be done in jitter
// buffer.
size_t concatenated_length =
intermediate1.size() + intermediate2.size() + intermediate3.size();
rtc::Buffer concatenated_buffer;
concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size());
concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size());
concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size());
EncodedImage concatenated(concatenated_buffer.data(), concatenated_length,
concatenated_length);
// Extract frame id from concatenated image
EncodedImageExtractionResult out = injector.ExtractData(concatenated, 2);
EXPECT_EQ(out.id, 512);
EXPECT_TRUE(out.discard);
EXPECT_EQ(out.image.size(), 0ul);
}
} // namespace test
} // namespace webrtc

136
test/pc/e2e/analyzer/video/default_encoded_image_id_injector_unittest.cc
View File

@ -1,136 +0,0 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h"
#include <utility>
#include "api/video/encoded_image.h"
#include "rtc_base/buffer.h"
#include "test/gtest.h"
namespace webrtc {
namespace test {
namespace {
rtc::Buffer CreateBufferOfSizeNFilledWithValuesFromX(size_t n, uint8_t x) {
rtc::Buffer buffer(n);
for (size_t i = 0; i < n; ++i) {
buffer[i] = static_cast<uint8_t>(x + i);
}
return buffer;
}
} // namespace
TEST(DefaultEncodedImageIdInjector, InjectExtract) {
DefaultEncodedImageIdInjector injector;
rtc::Buffer buffer = CreateBufferOfSizeNFilledWithValuesFromX(10, 1);
EncodedImage source(buffer.data(), 10, 10);
source.SetTimestamp(123456789);
EncodedImageWithId out =
injector.ExtractId(injector.InjectId(512, source, 1), 2);
ASSERT_EQ(out.id, 512);
ASSERT_EQ(out.image.size(), 10ul);
for (int i = 0; i < 10; ++i) {
ASSERT_EQ(out.image.data()[i], i + 1);
}
}
TEST(DefaultEncodedImageIdInjector, Inject3Extract3) {
DefaultEncodedImageIdInjector injector;
rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1);
rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11);
rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21);
// 1st frame
EncodedImage source1(buffer1.data(), 10, 10);
source1.SetTimestamp(123456710);
// 2nd frame 1st spatial layer
EncodedImage source2(buffer2.data(), 10, 10);
source2.SetTimestamp(123456720);
// 2nd frame 2nd spatial layer
EncodedImage source3(buffer3.data(), 10, 10);
source3.SetTimestamp(123456720);
EncodedImage intermediate1 = injector.InjectId(510, source1, 1);
EncodedImage intermediate2 = injector.InjectId(520, source2, 1);
EncodedImage intermediate3 = injector.InjectId(520, source3, 1);
// Extract ids in different order.
EncodedImageWithId out3 = injector.ExtractId(intermediate3, 2);
EncodedImageWithId out1 = injector.ExtractId(intermediate1, 2);
EncodedImageWithId out2 = injector.ExtractId(intermediate2, 2);
ASSERT_EQ(out1.id, 510);
ASSERT_EQ(out1.image.size(), 10ul);
for (int i = 0; i < 10; ++i) {
ASSERT_EQ(out1.image.data()[i], i + 1);
}
ASSERT_EQ(out2.id, 520);
ASSERT_EQ(out2.image.size(), 10ul);
for (int i = 0; i < 10; ++i) {
ASSERT_EQ(out2.image.data()[i], i + 11);
}
ASSERT_EQ(out3.id, 520);
ASSERT_EQ(out3.image.size(), 10ul);
for (int i = 0; i < 10; ++i) {
ASSERT_EQ(out3.image.data()[i], i + 21);
}
}
TEST(DefaultEncodedImageIdInjector, InjectExtractFromConcatenated) {
DefaultEncodedImageIdInjector injector;
rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1);
rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11);
rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21);
EncodedImage source1(buffer1.data(), 10, 10);
source1.SetTimestamp(123456710);
EncodedImage source2(buffer2.data(), 10, 10);
source2.SetTimestamp(123456710);
EncodedImage source3(buffer3.data(), 10, 10);
source3.SetTimestamp(123456710);
// Inject id into 3 images with same frame id.
EncodedImage intermediate1 = injector.InjectId(512, source1, 1);
EncodedImage intermediate2 = injector.InjectId(512, source2, 1);
EncodedImage intermediate3 = injector.InjectId(512, source3, 1);
// Concatenate them into single encoded image, like it can be done in jitter
// buffer.
size_t concatenated_length =
intermediate1.size() + intermediate2.size() + intermediate3.size();
rtc::Buffer concatenated_buffer;
concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size());
concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size());
concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size());
EncodedImage concatenated(concatenated_buffer.data(), concatenated_length,
concatenated_length);
// Extract frame id from concatenated image
EncodedImageWithId out = injector.ExtractId(concatenated, 2);
ASSERT_EQ(out.id, 512);
ASSERT_EQ(out.image.size(), 3 * 10ul);
for (int i = 0; i < 10; ++i) {
ASSERT_EQ(out.image.data()[i], i + 1);
ASSERT_EQ(out.image.data()[i + 10], i + 11);
ASSERT_EQ(out.image.data()[i + 20], i + 21);
}
}
} // namespace test
} // namespace webrtc

7
test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
View File

@ -299,6 +299,13 @@ void DefaultVideoQualityAnalyzer::Stop() {
ReportResults();
}
std::string DefaultVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) {
rtc::CritScope crit1(&lock_);
auto it = frame_stats_.find(frame_id);
RTC_DCHECK(it != frame_stats_.end()) << "Unknown frame_id=" << frame_id;
return it->second.stream_label;
}
std::set<std::string> DefaultVideoQualityAnalyzer::GetKnownVideoStreams()
const {
rtc::CritScope crit2(&comparison_lock_);

1
test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
View File

@ -136,6 +136,7 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
void OnEncoderError(const VideoFrame& frame, int32_t error_code) override;
void OnDecoderError(uint16_t frame_id, int32_t error_code) override;
void Stop() override;
std::string GetStreamLabel(uint16_t frame_id) override;
// Returns set of stream labels, that were met during test call.
std::set<std::string> GetKnownVideoStreams() const;

60
test/pc/e2e/analyzer/video/encoded_image_data_injector.h Normal file
View File

@ -0,0 +1,60 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_
#define TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_
#include <cstdint>
#include <utility>
#include "api/video/encoded_image.h"
namespace webrtc {
namespace test {
// Injects frame id into EncodedImage on encoder side
class EncodedImageDataInjector {
public:
virtual ~EncodedImageDataInjector() = default;
// Return encoded image with specified |id| and |discard| flag injected into
// its payload. |discard| flag mean does analyzing decoder should discard this
// encoded image because it belongs to unnecessary simulcast stream or spatial
// layer. |coding_entity_id| is unique id of decoder or encoder.
virtual EncodedImage InjectData(uint16_t id,
bool discard,
const EncodedImage& source,
int coding_entity_id) = 0;
};
struct EncodedImageExtractionResult {
uint16_t id;
EncodedImage image;
// Is true if encoded image should be discarded. It is used to filter out
// unnecessary spatial layers and simulcast streams.
bool discard;
};
// Extracts frame id from EncodedImage on decoder side.
class EncodedImageDataExtractor {
public:
virtual ~EncodedImageDataExtractor() = default;
// Returns encoded image id, extracted from payload and also encoded image
// with its original payload. For concatenated spatial layers it should be the
// same id. |coding_entity_id| is unique id of decoder or encoder.
virtual EncodedImageExtractionResult ExtractData(const EncodedImage& source,
int coding_entity_id) = 0;
};
} // namespace test
} // namespace webrtc
#endif // TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_

54
test/pc/e2e/analyzer/video/encoded_image_id_injector.h
View File

@ -1,54 +0,0 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_ID_INJECTOR_H_
#define TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_ID_INJECTOR_H_
#include <cstdint>
#include <utility>
#include "api/video/encoded_image.h"
namespace webrtc {
namespace test {
// Injects frame id into EncodedImage on encoder side
class EncodedImageIdInjector {
public:
virtual ~EncodedImageIdInjector() = default;
// Return encoded image with specified |id| injected into its payload.
// |coding_entity_id| is unique id of decoder or encoder.
virtual EncodedImage InjectId(uint16_t id,
const EncodedImage& source,
int coding_entity_id) = 0;
};
struct EncodedImageWithId {
uint16_t id;
EncodedImage image;
};
// Extracts frame id from EncodedImage on decoder side.
class EncodedImageIdExtractor {
public:
virtual ~EncodedImageIdExtractor() = default;
// Returns encoded image id, extracted from payload and also encoded image
// with its original payload. For concatenated spatial layers it should be the
// same id. |coding_entity_id| is unique id of decoder or encoder.
virtual EncodedImageWithId ExtractId(const EncodedImage& source,
int coding_entity_id) = 0;
};
} // namespace test
} // namespace webrtc
#endif // TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_ID_INJECTOR_H_

12
test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc
View File

@ -28,12 +28,16 @@ uint16_t ExampleVideoQualityAnalyzer::OnFrameCaptured(
auto it = frames_in_flight_.find(frame_id);
if (it == frames_in_flight_.end()) {
frames_in_flight_.insert(frame_id);
frames_to_stream_label_.insert({frame_id, stream_label});
} else {
RTC_LOG(WARNING) << "Meet new frame with the same id: " << frame_id
<< ". Assumes old one as dropped";
// We needn't insert frame to frames_in_flight_, because it is already
// there.
++frames_dropped_;
auto stream_it = frames_to_stream_label_.find(frame_id);
RTC_CHECK(stream_it != frames_to_stream_label_.end());
stream_it->second = stream_label;
}
++frames_captured_;
return frame_id;
@ -95,6 +99,14 @@ void ExampleVideoQualityAnalyzer::Stop() {
frames_dropped_ += frames_in_flight_.size();
}
std::string ExampleVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) {
rtc::CritScope crit(&lock_);
auto it = frames_to_stream_label_.find(frame_id);
RTC_DCHECK(it != frames_to_stream_label_.end())
<< "Unknown frame_id=" << frame_id;
return it->second;
}
uint64_t ExampleVideoQualityAnalyzer::frames_captured() const {
rtc::CritScope crit(&lock_);
return frames_captured_;

3
test/pc/e2e/analyzer/video/example_video_quality_analyzer.h
View File

@ -12,6 +12,7 @@
#define TEST_PC_E2E_ANALYZER_VIDEO_EXAMPLE_VIDEO_QUALITY_ANALYZER_H_
#include <atomic>
#include <map>
#include <set>
#include <string>
@ -48,6 +49,7 @@ class ExampleVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
void OnEncoderError(const VideoFrame& frame, int32_t error_code) override;
void OnDecoderError(uint16_t frame_id, int32_t error_code) override;
void Stop() override;
std::string GetStreamLabel(uint16_t frame_id) override;
uint64_t frames_captured() const;
uint64_t frames_sent() const;
@ -66,6 +68,7 @@ class ExampleVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
// need to keep them to correctly determine dropped frames and also correctly
// process frame id overlap.
std::set<uint16_t> frames_in_flight_ RTC_GUARDED_BY(lock_);
std::map<uint16_t, std::string> frames_to_stream_label_ RTC_GUARDED_BY(lock_);
uint16_t next_frame_id_ RTC_GUARDED_BY(lock_) = 0;
uint64_t frames_captured_ RTC_GUARDED_BY(lock_) = 0;
uint64_t frames_sent_ RTC_GUARDED_BY(lock_) = 0;

9
test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc
View File

@ -25,7 +25,7 @@ namespace test {
QualityAnalyzingVideoDecoder::QualityAnalyzingVideoDecoder(
int id,
std::unique_ptr<VideoDecoder> delegate,
EncodedImageIdExtractor* extractor,
EncodedImageDataExtractor* extractor,
VideoQualityAnalyzerInterface* analyzer)
: id_(id),
implementation_name_("AnalyzingDecoder-" +
@ -53,7 +53,10 @@ int32_t QualityAnalyzingVideoDecoder::Decode(
// owner of original buffer will be responsible for deleting it, or extractor
// can create a new buffer. In such case extractor will be responsible for
// deleting it.
EncodedImageWithId out = extractor_->ExtractId(input_image, id_);
EncodedImageExtractionResult out = extractor_->ExtractData(input_image, id_);
// TODO(titovartem) add support for simulcast.
RTC_CHECK(!out.discard) << "Simulcast is not supported yet";
EncodedImage* origin_image;
{
@ -195,7 +198,7 @@ void QualityAnalyzingVideoDecoder::OnFrameDecoded(
QualityAnalyzingVideoDecoderFactory::QualityAnalyzingVideoDecoderFactory(
std::unique_ptr<VideoDecoderFactory> delegate,
IdGenerator<int>* id_generator,
EncodedImageIdExtractor* extractor,
EncodedImageDataExtractor* extractor,
VideoQualityAnalyzerInterface* analyzer)
: delegate_(std::move(delegate)),
id_generator_(id_generator),

14
test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h
View File

@ -22,7 +22,7 @@
#include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/video_decoder_factory.h"
#include "rtc_base/critical_section.h"
#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h"
#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h"
#include "test/pc/e2e/analyzer/video/id_generator.h"
#include "test/pc/e2e/api/video_quality_analyzer_interface.h"
@ -51,10 +51,10 @@ class QualityAnalyzingVideoDecoder : public VideoDecoder {
public:
// Creates analyzing decoder. |id| is unique coding entity id, that will
// be used to distinguish all encoders and decoders inside
// EncodedImageIdInjector and EncodedImageIdExtracor.
// EncodedImageDataInjector and EncodedImageIdExtracor.
QualityAnalyzingVideoDecoder(int id,
std::unique_ptr<VideoDecoder> delegate,
EncodedImageIdExtractor* extractor,
EncodedImageDataExtractor* extractor,
VideoQualityAnalyzerInterface* analyzer);
~QualityAnalyzingVideoDecoder() override;
@ -101,7 +101,7 @@ class QualityAnalyzingVideoDecoder : public VideoDecoder {
const int id_;
const std::string implementation_name_;
std::unique_ptr<VideoDecoder> delegate_;
EncodedImageIdExtractor* const extractor_;
EncodedImageDataExtractor* const extractor_;
VideoQualityAnalyzerInterface* const analyzer_;
std::unique_ptr<DecoderCallback> analyzing_callback_;
@ -112,7 +112,7 @@ class QualityAnalyzingVideoDecoder : public VideoDecoder {
std::map<uint32_t, uint16_t> timestamp_to_frame_id_ RTC_GUARDED_BY(lock_);
// Stores currently being decoded images by frame id. Because
// EncodedImageIdExtractor can create new copy on EncodedImage we need to
// EncodedImageDataExtractor can create new copy on EncodedImage we need to
// ensure, that this image won't be deleted during async decoding. To do it
// all images are putted into this map and removed from here inside callback.
std::map<uint16_t, EncodedImage> decoding_images_ RTC_GUARDED_BY(lock_);
@ -126,7 +126,7 @@ class QualityAnalyzingVideoDecoderFactory : public VideoDecoderFactory {
QualityAnalyzingVideoDecoderFactory(
std::unique_ptr<VideoDecoderFactory> delegate,
IdGenerator<int>* id_generator,
EncodedImageIdExtractor* extractor,
EncodedImageDataExtractor* extractor,
VideoQualityAnalyzerInterface* analyzer);
~QualityAnalyzingVideoDecoderFactory() override;
@ -141,7 +141,7 @@ class QualityAnalyzingVideoDecoderFactory : public VideoDecoderFactory {
private:
std::unique_ptr<VideoDecoderFactory> delegate_;
IdGenerator<int>* const id_generator_;
EncodedImageIdExtractor* const extractor_;
EncodedImageDataExtractor* const extractor_;
VideoQualityAnalyzerInterface* const analyzer_;
};

98
test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc
View File

@ -13,7 +13,9 @@
#include <utility>
#include "absl/memory/memory.h"
#include "api/video/video_codec_type.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/critical_section.h"
#include "rtc_base/logging.h"
namespace webrtc {
@ -27,10 +29,12 @@ constexpr size_t kMaxFrameInPipelineCount = 1000;
QualityAnalyzingVideoEncoder::QualityAnalyzingVideoEncoder(
int id,
std::unique_ptr<VideoEncoder> delegate,
EncodedImageIdInjector* injector,
std::map<std::string, absl::optional<int>> stream_required_spatial_index,
EncodedImageDataInjector* injector,
VideoQualityAnalyzerInterface* analyzer)
: id_(id),
delegate_(std::move(delegate)),
stream_required_spatial_index_(std::move(stream_required_spatial_index)),
injector_(injector),
analyzer_(analyzer) {}
QualityAnalyzingVideoEncoder::~QualityAnalyzingVideoEncoder() = default;
@ -39,6 +43,29 @@ int32_t QualityAnalyzingVideoEncoder::InitEncode(
const VideoCodec* codec_settings,
int32_t number_of_cores,
size_t max_payload_size) {
rtc::CritScope crit(&lock_);
mode_ = SimulcastMode::kNormal;
if (codec_settings->codecType == kVideoCodecVP9) {
if (codec_settings->VP9().numberOfSpatialLayers > 1) {
switch (codec_settings->VP9().interLayerPred) {
case InterLayerPredMode::kOn:
mode_ = SimulcastMode::kSVC;
break;
case InterLayerPredMode::kOnKeyPic:
mode_ = SimulcastMode::kKSVC;
break;
case InterLayerPredMode::kOff:
mode_ = SimulcastMode::kSimulcast;
break;
default:
RTC_NOTREACHED() << "Unknown codec_settings->VP9().interLayerPred";
break;
}
}
}
if (codec_settings->numberOfSimulcastStreams > 1) {
mode_ = SimulcastMode::kSimulcast;
}
return delegate_->InitEncode(codec_settings, number_of_cores,
max_payload_size);
}
@ -109,7 +136,7 @@ VideoEncoder::EncoderInfo QualityAnalyzingVideoEncoder::GetEncoderInfo() const {
}
// It is assumed, that encoded callback will be always invoked with encoded
// images that correspond to the frames if the same sequence, that frames
// images that correspond to the frames in the same sequence, that frames
// arrived. In other words, assume we have frames F1, F2 and F3 and they have
// corresponding encoded images I1, I2 and I3. In such case if we will call
// encode first with F1, then with F2 and then with F3, then encoder callback
@ -126,6 +153,7 @@ EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage(
const CodecSpecificInfo* codec_specific_info,
const RTPFragmentationHeader* fragmentation) {
uint16_t frame_id;
bool discard = false;
{
rtc::CritScope crit(&lock_);
std::pair<uint32_t, uint16_t> timestamp_frame_id;
@ -155,15 +183,22 @@ EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage(
EncodedImageCallback::Result::Error::OK);
}
frame_id = timestamp_frame_id.second;
discard = ShouldDiscard(frame_id, encoded_image);
}
if (!discard) {
// Analyzer should see only encoded images, that weren't discarded.
analyzer_->OnFrameEncoded(frame_id, encoded_image);
}
// Image id injector injects frame id into provided EncodedImage and returns
// the image with a) modified original buffer (in such case the current owner
// of the buffer will be responsible for deleting it) or b) a new buffer (in
// such case injector will be responsible for deleting it).
const EncodedImage& image = injector_->InjectId(frame_id, encoded_image, id_);
// Image data injector injects frame id and discard flag into provided
// EncodedImage and returns the image with a) modified original buffer (in
// such case the current owner of the buffer will be responsible for deleting
// it) or b) a new buffer (in such case injector will be responsible for
// deleting it).
const EncodedImage& image =
injector_->InjectData(frame_id, discard, encoded_image, id_);
{
rtc::CritScope crit(&lock_);
RTC_DCHECK(delegate_callback_);
@ -180,12 +215,57 @@ void QualityAnalyzingVideoEncoder::OnDroppedFrame(
delegate_callback_->OnDroppedFrame(reason);
}
bool QualityAnalyzingVideoEncoder::ShouldDiscard(
uint16_t frame_id,
const EncodedImage& encoded_image) {
std::string stream_label = analyzer_->GetStreamLabel(frame_id);
absl::optional<int> required_spatial_index =
stream_required_spatial_index_[stream_label];
if (required_spatial_index) {
RTC_CHECK(encoded_image.SpatialIndex())
<< "Specific spatial layer/simulcast stream requested for track, but "
"now spatial layers/simulcast streams produced by encoder. "
"stream_label="
<< stream_label
<< "; required_spatial_index=" << *required_spatial_index;
RTC_CHECK(mode_ != SimulcastMode::kNormal)
<< "Analyzing encoder is in kNormal "
"mode, but spatial layer/simulcast "
"stream met.";
if (mode_ == SimulcastMode::kSimulcast) {
// In simulcast mode only encoded images with required spatial index are
// interested, so all others have to be discarded.
return *encoded_image.SpatialIndex() != *required_spatial_index;
} else if (mode_ == SimulcastMode::kSVC) {
// In SVC mode encoded images with spatial indexes that are equal or
// less than required one are interesting, so all above have to be
// discarded.
return *encoded_image.SpatialIndex() > *required_spatial_index;
} else if (mode_ == SimulcastMode::kKSVC) {
// In KSVC mode for key frame encoded images with spatial indexes that
// are equal or less than required one are interesting, so all above
// have to be discarded. For other frames only required spatial index
// is interesting, so all others have to be discarded.
if (encoded_image._frameType == FrameType::kVideoFrameKey) {
return *encoded_image.SpatialIndex() > *required_spatial_index;
} else {
return *encoded_image.SpatialIndex() != *required_spatial_index;
}
} else {
RTC_NOTREACHED() << "Unsupported encoder mode";
}
}
return false;
}
QualityAnalyzingVideoEncoderFactory::QualityAnalyzingVideoEncoderFactory(
std::unique_ptr<VideoEncoderFactory> delegate,
std::map<std::string, absl::optional<int>> stream_required_spatial_index,
IdGenerator<int>* id_generator,
EncodedImageIdInjector* injector,
EncodedImageDataInjector* injector,
VideoQualityAnalyzerInterface* analyzer)
: delegate_(std::move(delegate)),
stream_required_spatial_index_(std::move(stream_required_spatial_index)),
id_generator_(id_generator),
injector_(injector),
analyzer_(analyzer) {}
@ -208,7 +288,7 @@ QualityAnalyzingVideoEncoderFactory::CreateVideoEncoder(
const SdpVideoFormat& format) {
return absl::make_unique<QualityAnalyzingVideoEncoder>(
id_generator_->GetNextId(), delegate_->CreateVideoEncoder(format),
injector_, analyzer_);
stream_required_spatial_index_, injector_, analyzer_);
}
} // namespace test

73
test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h
View File

@ -22,7 +22,7 @@
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "rtc_base/critical_section.h"
#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h"
#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h"
#include "test/pc/e2e/analyzer/video/id_generator.h"
#include "test/pc/e2e/api/video_quality_analyzer_interface.h"
@ -41,7 +41,7 @@ namespace test {
//
// When origin encoder encodes the image it will call quality encoder's special
// callback, where video analyzer will be called again and then frame id will be
// injected into EncodedImage with passed EncodedImageIdInjector. Then new
// injected into EncodedImage with passed EncodedImageDataInjector. Then new
// EncodedImage will be passed to origin callback, provided by user.
//
// Quality encoder registers its own callback in origin encoder at the same
@ -51,10 +51,12 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder,
public:
// Creates analyzing encoder. |id| is unique coding entity id, that will
// be used to distinguish all encoders and decoders inside
// EncodedImageIdInjector and EncodedImageIdExtracor.
QualityAnalyzingVideoEncoder(int id,
// EncodedImageDataInjector and EncodedImageIdExtracor.
QualityAnalyzingVideoEncoder(
int id,
std::unique_ptr<VideoEncoder> delegate,
EncodedImageIdInjector* injector,
std::map<std::string, absl::optional<int>> stream_required_spatial_index,
EncodedImageDataInjector* injector,
VideoQualityAnalyzerInterface* analyzer);
~QualityAnalyzingVideoEncoder() override;
@ -81,9 +83,61 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder,
void OnDroppedFrame(DropReason reason) override;
private:
enum SimulcastMode {
// In this mode encoder assumes not more than 1 encoded image per video
// frame
kNormal,
// Next modes are to test video conference behavior. For conference sender
// will send multiple spatial layers/simulcast streams for single video
// track and there is some Selective Forwarding Unit (SFU), that forwards
// only best one, that will pass through downlink to the receiver.
//
// Here this behavior will be partly emulated. Sender will send all spatial
// layers/simulcast streams and then some of them will be filtered out on
// the receiver side. During test setup user can specify which spatial
// layer/simulcast stream is required, what will simulated which spatial
// layer/simulcast stream will be chosen by SFU in the real world. Then
// sender will mark encoded images for all spatial layers above required or
// all simulcast streams except required as to be discarded and on receiver
// side they will be discarded in quality analyzing decoder and won't be
// passed into delegate decoder.
//
// If the sender for some reasons won't send specified spatial layer, then
// receiver still will fall back on lower spatial layers. But for simulcast
// streams if required one won't be sent, receiver will assume all frames
// in that period as dropped and will experience video freeze.
//
// Test based on this simulation will be used to evaluate video quality
// of concrete spatial layers/simulcast streams and also check distribution
// of bandwidth between spatial layers/simulcast streams by BWE.
// In this mode encoder assumes that for each frame simulcast encoded
// images will be produced. So all simulcast streams except required will
// be marked as to be discarded in decoder and won't reach video quality
// analyzer.
kSimulcast,
// In this mode encoder assumes that for each frame encoded images for
// different spatial layers will be produced. So all spatial layers above
// required will be marked to be discarded in decoder and won't reach
// video quality analyzer.
kSVC,
// In this mode encoder assumes that for each frame encoded images for
// different spatial layers will be produced. Compared to kSVC mode
// spatial layers that are above required will be marked to be discarded
// only for key frames and for regular frames all except required spatial
// layer will be marked as to be discarded in decoder and won't reach video
// quality analyzer.
kKSVC
};
bool ShouldDiscard(uint16_t frame_id, const EncodedImage& encoded_image)
RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_);
const int id_;
std::unique_ptr<VideoEncoder> delegate_;
EncodedImageIdInjector* const injector_;
std::map<std::string, absl::optional<int>> stream_required_spatial_index_;
EncodedImageDataInjector* const injector_;
VideoQualityAnalyzerInterface* const analyzer_;
// VideoEncoder interface assumes async delivery of encoded images.
@ -91,6 +145,7 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder,
// from received VideoFrame to resulted EncodedImage.
rtc::CriticalSection lock_;
SimulcastMode mode_ RTC_GUARDED_BY(lock_);
EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(lock_);
std::list<std::pair<uint32_t, uint16_t>> timestamp_to_frame_id_list_
RTC_GUARDED_BY(lock_);
@ -103,8 +158,9 @@ class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory {
public:
QualityAnalyzingVideoEncoderFactory(
std::unique_ptr<VideoEncoderFactory> delegate,
std::map<std::string, absl::optional<int>> stream_required_spatial_index,
IdGenerator<int>* id_generator,
EncodedImageIdInjector* injector,
EncodedImageDataInjector* injector,
VideoQualityAnalyzerInterface* analyzer);
~QualityAnalyzingVideoEncoderFactory() override;
@ -117,8 +173,9 @@ class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory {
private:
std::unique_ptr<VideoEncoderFactory> delegate_;
std::map<std::string, absl::optional<int>> stream_required_spatial_index_;
IdGenerator<int>* const id_generator_;
EncodedImageIdInjector* const injector_;
EncodedImageDataInjector* const injector_;
VideoQualityAnalyzerInterface* const analyzer_;
};

45
test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.cc → test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc
View File

@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h"
#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h"
#include <cstddef>
@ -26,19 +26,21 @@ constexpr size_t kUsedBufferSize = 3;
} // namespace
SingleProcessEncodedImageIdInjector::SingleProcessEncodedImageIdInjector() =
default;
SingleProcessEncodedImageIdInjector::~SingleProcessEncodedImageIdInjector() =
SingleProcessEncodedImageDataInjector::SingleProcessEncodedImageDataInjector() =
default;
SingleProcessEncodedImageDataInjector::
~SingleProcessEncodedImageDataInjector() = default;
EncodedImage SingleProcessEncodedImageIdInjector::InjectId(
EncodedImage SingleProcessEncodedImageDataInjector::InjectData(
uint16_t id,
bool discard,
const EncodedImage& source,
int coding_entity_id) {
RTC_CHECK(source.size() >= kUsedBufferSize);
ExtractionInfo info;
info.length = source.size();
info.discard = discard;
memcpy(info.origin_data, source.data(), kUsedBufferSize);
{
rtc::CritScope crit(&lock_);
@ -55,18 +57,24 @@ EncodedImage SingleProcessEncodedImageIdInjector::InjectId(
return out;
}
EncodedImageWithId SingleProcessEncodedImageIdInjector::ExtractId(
EncodedImageExtractionResult SingleProcessEncodedImageDataInjector::ExtractData(
const EncodedImage& source,
int coding_entity_id) {
EncodedImage out = source;
// Both |source| and |out| image will share the same buffer for payload or
// out will have a copy for it, so we can operate on the |out| buffer only.
uint8_t* buffer = out.data();
size_t size = out.size();
size_t pos = 0;
absl::optional<uint16_t> id = absl::nullopt;
while (pos < source.size()) {
bool discard = true;
while (pos < size) {
// Extract frame id from first 2 bytes of the payload.
uint16_t next_id = source.data()[pos] + (source.data()[pos + 1] << 8);
uint16_t next_id = buffer[pos] + (buffer[pos + 1] << 8);
// Extract frame sub id from second 2 byte of the payload.
uint16_t sub_id = source.data()[pos + 2];
uint16_t sub_id = buffer[pos + 2];
RTC_CHECK(!id || id.value() == next_id)
<< "Different frames encoded into single encoded image: " << id.value()
@ -87,17 +95,28 @@ EncodedImageWithId SingleProcessEncodedImageIdInjector::ExtractId(
ext_vector_it->second.infos.erase(info_it);
}
memcpy(&out.data()[pos], info.origin_data, kUsedBufferSize);
if (info.discard) {
// If this encoded image is marked to be discarded - erase it's payload
// from the buffer.
memmove(&buffer[pos], &buffer[pos + info.length],
size - pos - info.length);
size -= info.length;
} else {
memmove(&buffer[pos], info.origin_data, kUsedBufferSize);
pos += info.length;
}
// We need to discard encoded image only if all concatenated encoded images
// have to be discarded.
discard = discard & info.discard;
}
out.set_size(pos);
return EncodedImageWithId{id.value(), out};
return EncodedImageExtractionResult{id.value(), out, discard};
}
SingleProcessEncodedImageIdInjector::ExtractionInfoVector::
SingleProcessEncodedImageDataInjector::ExtractionInfoVector::
ExtractionInfoVector() = default;
SingleProcessEncodedImageIdInjector::ExtractionInfoVector::
SingleProcessEncodedImageDataInjector::ExtractionInfoVector::
~ExtractionInfoVector() = default;
} // namespace test

42
test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h → test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h
View File

@ -8,8 +8,8 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_ID_INJECTOR_H_
#define TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_ID_INJECTOR_H_
#ifndef TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_
#define TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_
#include <cstdint>
#include <map>
@ -19,35 +19,38 @@
#include "api/video/encoded_image.h"
#include "rtc_base/critical_section.h"
#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h"
#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h"
namespace webrtc {
namespace test {
// Based on assumption that all call participants are in the same OS process
// and uses same QualityAnalyzingVideoContext to obtain EncodedImageIdInjector.
// and uses same QualityAnalyzingVideoContext to obtain
// EncodedImageDataInjector.
//
// To inject frame id into EncodedImage injector uses first 2 bytes of
// EncodedImage payload. Then it uses 3rd byte for frame sub id, that is
// required to distinguish different spatial layers. The origin data from these
// 3 bytes will be stored inside injector's internal storage and then will be
// restored during extraction phase.
// To inject frame id and discard flag into EncodedImage injector uses first 2
// bytes of EncodedImage payload. Then it uses 3rd byte for frame sub id, that
// is required to distinguish different spatial layers. The origin data from
// these 3 bytes will be stored inside injector's internal storage and then will
// be restored during extraction phase.
//
// This injector won't add any extra overhead into EncodedImage payload and
// support frames with any size of payload. Also assumes that every EncodedImage
// payload size is greater or equals to 3 bytes
class SingleProcessEncodedImageIdInjector : public EncodedImageIdInjector,
public EncodedImageIdExtractor {
class SingleProcessEncodedImageDataInjector : public EncodedImageDataInjector,
public EncodedImageDataExtractor {
public:
SingleProcessEncodedImageIdInjector();
~SingleProcessEncodedImageIdInjector() override;
SingleProcessEncodedImageDataInjector();
~SingleProcessEncodedImageDataInjector() override;
// Id will be injected into EncodedImage buffer directly. This buffer won't
// be fully copied, so |source| image buffer will be also changed.
EncodedImage InjectId(uint16_t id,
// Id and discard flag will be injected into EncodedImage buffer directly.
// This buffer won't be fully copied, so |source| image buffer will be also
// changed.
EncodedImage InjectData(uint16_t id,
bool discard,
const EncodedImage& source,
int coding_entity_id) override;
EncodedImageWithId ExtractId(const EncodedImage& source,
EncodedImageExtractionResult ExtractData(const EncodedImage& source,
int coding_entity_id) override;
private:
@ -58,6 +61,9 @@ class SingleProcessEncodedImageIdInjector : public EncodedImageIdInjector,
uint8_t sub_id;
// Length of the origin buffer encoded image.
size_t length;
// Flag to show is this encoded images should be discarded by analyzing
// decoder because of not required spatial layer/simulcast stream.
bool discard;
// Data from first 3 bytes of origin encoded image's payload.
uint8_t origin_data[3];
};
@ -84,4 +90,4 @@ class SingleProcessEncodedImageIdInjector : public EncodedImageIdInjector,
} // namespace test
} // namespace webrtc
#endif // TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_ID_INJECTOR_H_
#endif // TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_

198
test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc Normal file
View File

@ -0,0 +1,198 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h"
#include <utility>
#include "api/video/encoded_image.h"
#include "rtc_base/buffer.h"
#include "test/gtest.h"
namespace webrtc {
namespace test {
namespace {
rtc::Buffer CreateBufferOfSizeNFilledWithValuesFromX(size_t n, uint8_t x) {
rtc::Buffer buffer(n);
for (size_t i = 0; i < n; ++i) {
buffer[i] = static_cast<uint8_t>(x + i);
}
return buffer;
}
} // namespace
TEST(SingleProcessEncodedImageDataInjector, InjectExtractDiscardFalse) {
SingleProcessEncodedImageDataInjector injector;
rtc::Buffer buffer = CreateBufferOfSizeNFilledWithValuesFromX(10, 1);
EncodedImage source(buffer.data(), 10, 10);
source.SetTimestamp(123456789);
EncodedImageExtractionResult out =
injector.ExtractData(injector.InjectData(512, false, source, 1), 2);
EXPECT_EQ(out.id, 512);
EXPECT_FALSE(out.discard);
EXPECT_EQ(out.image.size(), 10ul);
EXPECT_EQ(out.image.capacity(), 10ul);
for (int i = 0; i < 10; ++i) {
EXPECT_EQ(out.image.data()[i], i + 1);
}
}
TEST(SingleProcessEncodedImageDataInjector, InjectExtractDiscardTrue) {
SingleProcessEncodedImageDataInjector injector;
rtc::Buffer buffer = CreateBufferOfSizeNFilledWithValuesFromX(10, 1);
EncodedImage source(buffer.data(), 10, 10);
source.SetTimestamp(123456789);
EncodedImageExtractionResult out =
injector.ExtractData(injector.InjectData(512, true, source, 1), 2);
EXPECT_EQ(out.id, 512);
EXPECT_TRUE(out.discard);
EXPECT_EQ(out.image.size(), 0ul);
EXPECT_EQ(out.image.capacity(), 10ul);
}
TEST(SingleProcessEncodedImageDataInjector, Inject3Extract3) {
SingleProcessEncodedImageDataInjector injector;
rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1);
rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11);
rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21);
// 1st frame
EncodedImage source1(buffer1.data(), 10, 10);
source1.SetTimestamp(123456710);
// 2nd frame 1st spatial layer
EncodedImage source2(buffer2.data(), 10, 10);
source2.SetTimestamp(123456720);
// 2nd frame 2nd spatial layer
EncodedImage source3(buffer3.data(), 10, 10);
source3.SetTimestamp(123456720);
EncodedImage intermediate1 = injector.InjectData(510, false, source1, 1);
EncodedImage intermediate2 = injector.InjectData(520, true, source2, 1);
EncodedImage intermediate3 = injector.InjectData(520, false, source3, 1);
// Extract ids in different order.
EncodedImageExtractionResult out3 = injector.ExtractData(intermediate3, 2);
EncodedImageExtractionResult out1 = injector.ExtractData(intermediate1, 2);
EncodedImageExtractionResult out2 = injector.ExtractData(intermediate2, 2);
EXPECT_EQ(out1.id, 510);
EXPECT_FALSE(out1.discard);
EXPECT_EQ(out1.image.size(), 10ul);
EXPECT_EQ(out1.image.capacity(), 10ul);
for (int i = 0; i < 10; ++i) {
EXPECT_EQ(out1.image.data()[i], i + 1);
}
EXPECT_EQ(out2.id, 520);
EXPECT_TRUE(out2.discard);
EXPECT_EQ(out2.image.size(), 0ul);
EXPECT_EQ(out2.image.capacity(), 10ul);
EXPECT_EQ(out3.id, 520);
EXPECT_FALSE(out3.discard);
EXPECT_EQ(out3.image.size(), 10ul);
EXPECT_EQ(out3.image.capacity(), 10ul);
for (int i = 0; i < 10; ++i) {
EXPECT_EQ(out3.image.data()[i], i + 21);
}
}
TEST(SingleProcessEncodedImageDataInjector, InjectExtractFromConcatenated) {
SingleProcessEncodedImageDataInjector injector;
rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1);
rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11);
rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21);
EncodedImage source1(buffer1.data(), 10, 10);
source1.SetTimestamp(123456710);
EncodedImage source2(buffer2.data(), 10, 10);
source2.SetTimestamp(123456710);
EncodedImage source3(buffer3.data(), 10, 10);
source3.SetTimestamp(123456710);
// Inject id into 3 images with same frame id.
EncodedImage intermediate1 = injector.InjectData(512, false, source1, 1);
EncodedImage intermediate2 = injector.InjectData(512, true, source2, 1);
EncodedImage intermediate3 = injector.InjectData(512, false, source3, 1);
// Concatenate them into single encoded image, like it can be done in jitter
// buffer.
size_t concatenated_length =
intermediate1.size() + intermediate2.size() + intermediate3.size();
rtc::Buffer concatenated_buffer;
concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size());
concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size());
concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size());
EncodedImage concatenated(concatenated_buffer.data(), concatenated_length,
concatenated_length);
// Extract frame id from concatenated image
EncodedImageExtractionResult out = injector.ExtractData(concatenated, 2);
EXPECT_EQ(out.id, 512);
EXPECT_FALSE(out.discard);
EXPECT_EQ(out.image.size(), 2 * 10ul);
EXPECT_EQ(out.image.capacity(), 3 * 10ul);
for (int i = 0; i < 10; ++i) {
EXPECT_EQ(out.image.data()[i], i + 1);
EXPECT_EQ(out.image.data()[i + 10], i + 21);
}
}
TEST(SingleProcessEncodedImageDataInjector,
InjectExtractFromConcatenatedAllDiscarded) {
SingleProcessEncodedImageDataInjector injector;
rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1);
rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11);
rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21);
EncodedImage source1(buffer1.data(), 10, 10);
source1.SetTimestamp(123456710);
EncodedImage source2(buffer2.data(), 10, 10);
source2.SetTimestamp(123456710);
EncodedImage source3(buffer3.data(), 10, 10);
source3.SetTimestamp(123456710);
// Inject id into 3 images with same frame id.
EncodedImage intermediate1 = injector.InjectData(512, true, source1, 1);
EncodedImage intermediate2 = injector.InjectData(512, true, source2, 1);
EncodedImage intermediate3 = injector.InjectData(512, true, source3, 1);
// Concatenate them into single encoded image, like it can be done in jitter
// buffer.
size_t concatenated_length =
intermediate1.size() + intermediate2.size() + intermediate3.size();
rtc::Buffer concatenated_buffer;
concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size());
concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size());
concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size());
EncodedImage concatenated(concatenated_buffer.data(), concatenated_length,
concatenated_length);
// Extract frame id from concatenated image
EncodedImageExtractionResult out = injector.ExtractData(concatenated, 2);
EXPECT_EQ(out.id, 512);
EXPECT_TRUE(out.discard);
EXPECT_EQ(out.image.size(), 0ul);
EXPECT_EQ(out.image.capacity(), 3 * 10ul);
}
} // namespace test
} // namespace webrtc

141
test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector_unittest.cc
View File

@ -1,141 +0,0 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h"
#include <utility>
#include "api/video/encoded_image.h"
#include "rtc_base/buffer.h"
#include "test/gtest.h"
namespace webrtc {
namespace test {
namespace {
rtc::Buffer CreateBufferOfSizeNFilledWithValuesFromX(size_t n, uint8_t x) {
rtc::Buffer buffer(n);
for (size_t i = 0; i < n; ++i) {
buffer[i] = static_cast<uint8_t>(x + i);
}
return buffer;
}
} // namespace
TEST(SingleProcessEncodedImageIdInjector, InjectExtract) {
SingleProcessEncodedImageIdInjector injector;
rtc::Buffer buffer = CreateBufferOfSizeNFilledWithValuesFromX(10, 1);
EncodedImage source(buffer.data(), 10, 10);
source.SetTimestamp(123456789);
EncodedImageWithId out =
injector.ExtractId(injector.InjectId(512, source, 1), 2);
ASSERT_EQ(out.id, 512);
ASSERT_EQ(out.image.size(), 10ul);
ASSERT_EQ(out.image.capacity(), 10ul);
for (int i = 0; i < 10; ++i) {
ASSERT_EQ(out.image.data()[i], i + 1);
}
}
TEST(SingleProcessEncodedImageIdInjector, Inject3Extract3) {
SingleProcessEncodedImageIdInjector injector;
rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1);
rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11);
rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21);
// 1st frame
EncodedImage source1(buffer1.data(), 10, 10);
source1.SetTimestamp(123456710);
// 2nd frame 1st spatial layer
EncodedImage source2(buffer2.data(), 10, 10);
source2.SetTimestamp(123456720);
// 2nd frame 2nd spatial layer
EncodedImage source3(buffer3.data(), 10, 10);
source3.SetTimestamp(123456720);
EncodedImage intermediate1 = injector.InjectId(510, source1, 1);
EncodedImage intermediate2 = injector.InjectId(520, source2, 1);
EncodedImage intermediate3 = injector.InjectId(520, source3, 1);
// Extract ids in different order.
EncodedImageWithId out3 = injector.ExtractId(intermediate3, 2);
EncodedImageWithId out1 = injector.ExtractId(intermediate1, 2);
EncodedImageWithId out2 = injector.ExtractId(intermediate2, 2);
ASSERT_EQ(out1.id, 510);
ASSERT_EQ(out1.image.size(), 10ul);
ASSERT_EQ(out1.image.capacity(), 10ul);
for (int i = 0; i < 10; ++i) {
ASSERT_EQ(out1.image.data()[i], i + 1);
}
ASSERT_EQ(out2.id, 520);
ASSERT_EQ(out2.image.size(), 10ul);
ASSERT_EQ(out2.image.capacity(), 10ul);
for (int i = 0; i < 10; ++i) {
ASSERT_EQ(out2.image.data()[i], i + 11);
}
ASSERT_EQ(out3.id, 520);
ASSERT_EQ(out3.image.size(), 10ul);
ASSERT_EQ(out3.image.capacity(), 10ul);
for (int i = 0; i < 10; ++i) {
ASSERT_EQ(out3.image.data()[i], i + 21);
}
}
TEST(SingleProcessEncodedImageIdInjector, InjectExtractFromConcatenated) {
SingleProcessEncodedImageIdInjector injector;
rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1);
rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11);
rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21);
EncodedImage source1(buffer1.data(), 10, 10);
source1.SetTimestamp(123456710);
EncodedImage source2(buffer2.data(), 10, 10);
source2.SetTimestamp(123456710);
EncodedImage source3(buffer3.data(), 10, 10);
source3.SetTimestamp(123456710);
// Inject id into 3 images with same frame id.
EncodedImage intermediate1 = injector.InjectId(512, source1, 1);
EncodedImage intermediate2 = injector.InjectId(512, source2, 1);
EncodedImage intermediate3 = injector.InjectId(512, source3, 1);
// Concatenate them into single encoded image, like it can be done in jitter
// buffer.
size_t concatenated_length =
intermediate1.size() + intermediate2.size() + intermediate3.size();
rtc::Buffer concatenated_buffer;
concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size());
concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size());
concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size());
EncodedImage concatenated(concatenated_buffer.data(), concatenated_length,
concatenated_length);
// Extract frame id from concatenated image
EncodedImageWithId out = injector.ExtractId(concatenated, 2);
ASSERT_EQ(out.id, 512);
ASSERT_EQ(out.image.size(), 3 * 10ul);
ASSERT_EQ(out.image.capacity(), 3 * 10ul);
for (int i = 0; i < 10; ++i) {
ASSERT_EQ(out.image.data()[i], i + 1);
ASSERT_EQ(out.image.data()[i + 10], i + 11);
ASSERT_EQ(out.image.data()[i + 20], i + 21);
}
}
} // namespace test
} // namespace webrtc

12
test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc
View File

@ -88,8 +88,8 @@ class AnalyzingVideoSink : public rtc::VideoSinkInterface<VideoFrame> {
VideoQualityAnalyzerInjectionHelper::VideoQualityAnalyzerInjectionHelper(
std::unique_ptr<VideoQualityAnalyzerInterface> analyzer,
EncodedImageIdInjector* injector,
EncodedImageIdExtractor* extractor)
EncodedImageDataInjector* injector,
EncodedImageDataExtractor* extractor)
: analyzer_(std::move(analyzer)),
injector_(injector),
extractor_(extractor),
@ -102,10 +102,12 @@ VideoQualityAnalyzerInjectionHelper::~VideoQualityAnalyzerInjectionHelper() =
std::unique_ptr<VideoEncoderFactory>
VideoQualityAnalyzerInjectionHelper::WrapVideoEncoderFactory(
std::unique_ptr<VideoEncoderFactory> delegate) const {
std::unique_ptr<VideoEncoderFactory> delegate,
std::map<std::string, absl::optional<int>> stream_required_spatial_index)
const {
return absl::make_unique<QualityAnalyzingVideoEncoderFactory>(
std::move(delegate), encoding_entities_id_generator_.get(), injector_,
analyzer_.get());
std::move(delegate), std::move(stream_required_spatial_index),
encoding_entities_id_generator_.get(), injector_, analyzer_.get());
}
std::unique_ptr<VideoDecoderFactory>

15
test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h
View File

@ -11,6 +11,7 @@
#ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_ANALYZER_INJECTION_HELPER_H_
#define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_ANALYZER_INJECTION_HELPER_H_
#include <map>
#include <memory>
#include <string>
@ -19,7 +20,7 @@
#include "api/video_codecs/video_decoder_factory.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "test/frame_generator.h"
#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h"
#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h"
#include "test/pc/e2e/analyzer/video/id_generator.h"
#include "test/pc/e2e/api/video_quality_analyzer_interface.h"
#include "test/testsupport/video_frame_writer.h"
@ -33,14 +34,16 @@ class VideoQualityAnalyzerInjectionHelper {
public:
VideoQualityAnalyzerInjectionHelper(
std::unique_ptr<VideoQualityAnalyzerInterface> analyzer,
EncodedImageIdInjector* injector,
EncodedImageIdExtractor* extractor);
EncodedImageDataInjector* injector,
EncodedImageDataExtractor* extractor);
~VideoQualityAnalyzerInjectionHelper();
// Wraps video encoder factory to give video quality analyzer access to frames
// before encoding and encoded images after.
std::unique_ptr<VideoEncoderFactory> WrapVideoEncoderFactory(
std::unique_ptr<VideoEncoderFactory> delegate) const;
std::unique_ptr<VideoEncoderFactory> delegate,
std::map<std::string, absl::optional<int>> stream_required_spatial_index)
const;
// Wraps video decoder factory to give video quality analyzer access to
// received encoded images and frames, that were decoded from them.
std::unique_ptr<VideoDecoderFactory> WrapVideoDecoderFactory(
@ -66,8 +69,8 @@ class VideoQualityAnalyzerInjectionHelper {
private:
std::unique_ptr<VideoQualityAnalyzerInterface> analyzer_;
EncodedImageIdInjector* injector_;
EncodedImageIdExtractor* extractor_;
EncodedImageDataInjector* injector_;
EncodedImageDataExtractor* extractor_;
std::unique_ptr<IdGenerator<int>> encoding_entities_id_generator_;
};

13
test/pc/e2e/api/peerconnection_quality_test_fixture.h
View File

@ -125,6 +125,19 @@ class PeerConnectionE2EQualityTestFixture {
absl::optional<std::string> input_file_name;
// If specified screen share video stream will be created as input.
absl::optional<ScreenShareConfig> screen_share_config;
// Specifies spatial index of the video stream to analyze.
// There are 3 cases:
// 1. |target_spatial_index| omitted: in such case it will be assumed that
// video stream has not spatial layers and simulcast streams.
// 2. |target_spatial_index| presented and simulcast encoder is used:
// in such case |target_spatial_index| will specify the index of
// simulcast strem, that should be analyzed. Other streams will be
// dropped.
// 3. |target_spatial_index| presented and SVP encoder is used:
// in such case |target_spatial_index| will specify the top interesting
// spatial layer and all layers bellow, including target one will be
// processed. All layers above target one will be dropped.
absl::optional<int> target_spatial_index;
// If specified the input stream will be also copied to specified file.
// It is actually one of the test's output file, which contains copy of what
// was captured during the test for this video stream on sender side.

2
test/pc/e2e/api/video_quality_analyzer_interface.h
View File

@ -93,6 +93,8 @@ class VideoQualityAnalyzerInterface {
// Tells analyzer that analysis complete and it should calculate final
// statistics.
virtual void Stop() {}
virtual std::string GetStreamLabel(uint16_t frame_id) = 0;
};
} // namespace webrtc

2
test/pc/e2e/peer_connection_quality_test.cc
View File

@ -101,7 +101,7 @@ PeerConnectionE2EQualityTest::PeerConnectionE2EQualityTest(
video_quality_analyzer = absl::make_unique<ExampleVideoQualityAnalyzer>();
}
encoded_image_id_controller_ =
absl::make_unique<SingleProcessEncodedImageIdInjector>();
absl::make_unique<SingleProcessEncodedImageDataInjector>();
video_quality_analyzer_injection_helper_ =
absl::make_unique<VideoQualityAnalyzerInjectionHelper>(
std::move(video_quality_analyzer), encoded_image_id_controller_.get(),

4
test/pc/e2e/peer_connection_quality_test.h
View File

@ -18,7 +18,7 @@
#include "rtc_base/task_queue.h"
#include "rtc_base/thread.h"
#include "system_wrappers/include/clock.h"
#include "test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h"
#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h"
#include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h"
#include "test/pc/e2e/api/peerconnection_quality_test_fixture.h"
#include "test/pc/e2e/test_peer.h"
@ -81,7 +81,7 @@ class PeerConnectionE2EQualityTest
Clock* const clock_;
std::unique_ptr<VideoQualityAnalyzerInjectionHelper>
video_quality_analyzer_injection_helper_;
std::unique_ptr<SingleProcessEncodedImageIdInjector>
std::unique_ptr<SingleProcessEncodedImageDataInjector>
encoded_image_id_controller_;
std::unique_ptr<TestPeer> alice_;

31
test/pc/e2e/test_peer.cc
View File

@ -29,7 +29,7 @@
#include "rtc_base/location.h"
#include "rtc_base/network.h"
#include "test/frame_generator_capturer.h"
#include "test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h"
#include "test/pc/e2e/analyzer/video/default_encoded_image_data_injector.h"
#include "test/pc/e2e/analyzer/video/example_video_quality_analyzer.h"
#include "test/testsupport/copy_to_file_audio_capturer.h"
@ -122,7 +122,8 @@ rtc::scoped_refptr<AudioDeviceModule> CreateAudioDeviceModule(
std::unique_ptr<VideoEncoderFactory> CreateVideoEncoderFactory(
PeerConnectionFactoryComponents* pcf_dependencies,
VideoQualityAnalyzerInjectionHelper* video_analyzer_helper) {
VideoQualityAnalyzerInjectionHelper* video_analyzer_helper,
std::map<std::string, absl::optional<int>> stream_required_spatial_index) {
std::unique_ptr<VideoEncoderFactory> video_encoder_factory;
if (pcf_dependencies->video_encoder_factory != nullptr) {
video_encoder_factory = std::move(pcf_dependencies->video_encoder_factory);
@ -130,7 +131,8 @@ std::unique_ptr<VideoEncoderFactory> CreateVideoEncoderFactory(
video_encoder_factory = CreateBuiltinVideoEncoderFactory();
}
return video_analyzer_helper->WrapVideoEncoderFactory(
std::move(video_encoder_factory));
std::move(video_encoder_factory),
std::move(stream_required_spatial_index));
}
std::unique_ptr<VideoDecoderFactory> CreateVideoDecoderFactory(
@ -149,13 +151,15 @@ std::unique_ptr<VideoDecoderFactory> CreateVideoDecoderFactory(
std::unique_ptr<cricket::MediaEngineInterface> CreateMediaEngine(
PeerConnectionFactoryComponents* pcf_dependencies,
absl::optional<AudioConfig> audio_config,
std::map<std::string, absl::optional<int>> stream_required_spatial_index,
VideoQualityAnalyzerInjectionHelper* video_analyzer_helper,
absl::optional<std::string> audio_output_file_name) {
rtc::scoped_refptr<AudioDeviceModule> adm = CreateAudioDeviceModule(
std::move(audio_config), std::move(audio_output_file_name));
std::unique_ptr<VideoEncoderFactory> video_encoder_factory =
CreateVideoEncoderFactory(pcf_dependencies, video_analyzer_helper);
CreateVideoEncoderFactory(pcf_dependencies, video_analyzer_helper,
std::move(stream_required_spatial_index));
std::unique_ptr<VideoDecoderFactory> video_decoder_factory =
CreateVideoDecoderFactory(pcf_dependencies, video_analyzer_helper);
@ -173,6 +177,7 @@ std::unique_ptr<cricket::MediaEngineInterface> CreateMediaEngine(
PeerConnectionFactoryDependencies CreatePCFDependencies(
std::unique_ptr<PeerConnectionFactoryComponents> pcf_dependencies,
absl::optional<AudioConfig> audio_config,
std::map<std::string, absl::optional<int>> stream_required_spatial_index,
VideoQualityAnalyzerInjectionHelper* video_analyzer_helper,
rtc::Thread* network_thread,
rtc::Thread* signaling_thread,
@ -181,7 +186,8 @@ PeerConnectionFactoryDependencies CreatePCFDependencies(
pcf_deps.network_thread = network_thread;
pcf_deps.signaling_thread = signaling_thread;
pcf_deps.media_engine = CreateMediaEngine(
pcf_dependencies.get(), std::move(audio_config), video_analyzer_helper,
pcf_dependencies.get(), std::move(audio_config),
std::move(stream_required_spatial_index), video_analyzer_helper,
std::move(audio_output_file_name));
pcf_deps.call_factory = std::move(pcf_dependencies->call_factory);
@ -254,10 +260,23 @@ std::unique_ptr<TestPeer> TestPeer::CreateTestPeer(
SetMandatoryEntities(components.get());
params->rtc_configuration.sdp_semantics = SdpSemantics::kUnifiedPlan;
std::map<std::string, absl::optional<int>> stream_required_spatial_index;
for (auto& video_config : params->video_configs) {
// Stream label should be set by fixture implementation here.
RTC_DCHECK(video_config.stream_label);
bool res = stream_required_spatial_index
.insert({*video_config.stream_label,
video_config.target_spatial_index})
.second;
RTC_DCHECK(res) << "Duplicate video_config.stream_label="
<< *video_config.stream_label;
}
// Create peer connection factory.
PeerConnectionFactoryDependencies pcf_deps = CreatePCFDependencies(
std::move(components->pcf_dependencies), params->audio_config,
video_analyzer_helper, components->network_thread, signaling_thread,
std::move(stream_required_spatial_index), video_analyzer_helper,
components->network_thread, signaling_thread,
std::move(audio_output_file_name));
rtc::scoped_refptr<PeerConnectionFactoryInterface> pcf =
CreateModularPeerConnectionFactory(std::move(pcf_deps));

2
test/pc/e2e/test_peer.h
View File

@ -22,7 +22,7 @@
#include "pc/test/mock_peer_connection_observers.h"
#include "rtc_base/network.h"
#include "rtc_base/thread.h"
#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h"
#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h"
#include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h"
#include "test/pc/e2e/api/peerconnection_quality_test_fixture.h"