zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve and re-enable FEC end-to-end tests.

Browse Source 
These tests got flaky under the new jitter buffer.

Enhancements:
- Use send-side BWE.
- Let BWE ramp up before applying packet loss.
- Improve packet loss simulation for ULPFEC.
- Add delay to fake network pipe for FlexFEC.
  (Not added for ULPFEC, since this makes those flaky...?)
- Add FlexFEC+NACK test, using RTX instead of "raw retransmits".
- Tighter checks of received packets' payload types and SSRCs.

TESTED=
$ ninja -C out/Debug video_engine_tests && third_party/gtest-parallel/gtest-parallel -r 1000 out/Debug/video_engine_tests --gtest_filter="*EndToEnd*Ulpfec*:*EndToEnd*Flexfec*"
ninja: Entering directory `out/Debug'
ninja: no work to do.
[12000/12000] TestWithNewVideoJitterBuffer/EndToEndTest.RecoversWithFlexfecAndNack/1 (14935 ms)

BUG=webrtc:7047

Review-Url: https://codereview.webrtc.org/2675573004
Cr-Commit-Position: refs/heads/master@{#16449}
This commit is contained in:
brandtr
2017-02-06 05:54:43 -08:00
committed by Commit bot
parent cb789bb510
commit d40b0f39e0
1 changed files with 124 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

202
webrtc/video/end_to_end_tests.cc
View File

@ -603,14 +603,14 @@ TEST_P(EndToEndTest, ReceivesNackAndRetransmitsAudio) {
RunBaseTest(&test); RunBaseTest(&test);
} }
// Disable due to failure, see bugs.webrtc.org/7050 for TEST_P(EndToEndTest, ReceivesUlpfec) {
// details.
TEST_P(EndToEndTest, DISABLED_CanReceiveUlpfec) {
class UlpfecRenderObserver : public test::EndToEndTest, class UlpfecRenderObserver : public test::EndToEndTest,
public rtc::VideoSinkInterface<VideoFrame> { public rtc::VideoSinkInterface<VideoFrame> {
public: public:
UlpfecRenderObserver() UlpfecRenderObserver()
: EndToEndTest(kDefaultTimeoutMs), state_(kFirstPacket) {} : EndToEndTest(kDefaultTimeoutMs),
random_(0xcafef00d1),
num_packets_sent_(0) {}
private: private:
Action OnSendRtp(const uint8_t* packet, size_t length) override { Action OnSendRtp(const uint8_t* packet, size_t length) override {
@ -618,45 +618,36 @@ TEST_P(EndToEndTest, DISABLED_CanReceiveUlpfec) {
RTPHeader header; RTPHeader header;
EXPECT_TRUE(parser_->Parse(packet, length, &header)); EXPECT_TRUE(parser_->Parse(packet, length, &header));
EXPECT_TRUE(header.payloadType == kFakeVideoSendPayloadType ||
header.payloadType == kRedPayloadType)
<< "Unknown payload type received.";
EXPECT_EQ(kVideoSendSsrcs[0], header.ssrc) << "Unknown SSRC received.";
// Parse RED header.
int encapsulated_payload_type = -1; int encapsulated_payload_type = -1;
if (header.payloadType == kRedPayloadType) { if (header.payloadType == kRedPayloadType) {
encapsulated_payload_type = encapsulated_payload_type =
static_cast<int>(packet[header.headerLength]); static_cast<int>(packet[header.headerLength]);
if (encapsulated_payload_type != kFakeVideoSendPayloadType)
EXPECT_EQ(kUlpfecPayloadType, encapsulated_payload_type); EXPECT_TRUE(encapsulated_payload_type == kFakeVideoSendPayloadType ||
} else { encapsulated_payload_type == kUlpfecPayloadType)
EXPECT_EQ(kFakeVideoSendPayloadType, header.payloadType); << "Unknown encapsulated payload type received.";
} }
if (protected_sequence_numbers_.count(header.sequenceNumber) != 0) { // To reduce test flakiness, always let ULPFEC packets through.
// Retransmitted packet, should not count. if (encapsulated_payload_type == kUlpfecPayloadType) {
protected_sequence_numbers_.erase(header.sequenceNumber);
auto ts_it = protected_timestamps_.find(header.timestamp);
EXPECT_NE(ts_it, protected_timestamps_.end());
protected_timestamps_.erase(ts_it);
return SEND_PACKET; return SEND_PACKET;
} }
switch (state_) { // Simulate 5% video packet loss after rampup period. Record the
case kFirstPacket: // corresponding timestamps that were dropped.
state_ = kDropEveryOtherPacketUntilUlpfec; if (num_packets_sent_++ > 100 && random_.Rand(1, 100) <= 5) {
break; if (encapsulated_payload_type == kFakeVideoSendPayloadType) {
case kDropEveryOtherPacketUntilUlpfec: dropped_sequence_numbers_.insert(header.sequenceNumber);
if (encapsulated_payload_type == kUlpfecPayloadType) { dropped_timestamps_.insert(header.timestamp);
state_ = kDropNextMediaPacket; }
return SEND_PACKET;
} return DROP_PACKET;
if (header.sequenceNumber % 2 == 0)
return DROP_PACKET;
break;
case kDropNextMediaPacket:
if (encapsulated_payload_type == kFakeVideoSendPayloadType) {
protected_sequence_numbers_.insert(header.sequenceNumber);
protected_timestamps_.insert(header.timestamp);
state_ = kDropEveryOtherPacketUntilUlpfec;
return DROP_PACKET;
}
break;
} }
return SEND_PACKET; return SEND_PACKET;
@ -666,27 +657,23 @@ TEST_P(EndToEndTest, DISABLED_CanReceiveUlpfec) {
rtc::CritScope lock(&crit_); rtc::CritScope lock(&crit_);
// Rendering frame with timestamp of packet that was dropped -> FEC // Rendering frame with timestamp of packet that was dropped -> FEC
// protection worked. // protection worked.
if (protected_timestamps_.count(video_frame.timestamp()) != 0) auto it = dropped_timestamps_.find(video_frame.timestamp());
if (it != dropped_timestamps_.end()) {
observation_complete_.Set(); observation_complete_.Set();
}
} }
enum {
kFirstPacket,
kDropEveryOtherPacketUntilUlpfec,
kDropNextMediaPacket,
} state_;
void ModifyVideoConfigs( void ModifyVideoConfigs(
VideoSendStream::Config* send_config, VideoSendStream::Config* send_config,
std::vector<VideoReceiveStream::Config>* receive_configs, std::vector<VideoReceiveStream::Config>* receive_configs,
VideoEncoderConfig* encoder_config) override { VideoEncoderConfig* encoder_config) override {
// TODO(pbos): Run this test with combined NACK/ULPFEC enabled as well. send_config->rtp.extensions.push_back(
// int rtp_history_ms = 1000; RtpExtension(RtpExtension::kTransportSequenceNumberUri,
// (*receive_configs)[0].rtp.nack.rtp_history_ms = rtp_history_ms; test::kTransportSequenceNumberExtensionId));
// send_config->rtp.nack.rtp_history_ms = rtp_history_ms;
send_config->rtp.ulpfec.red_payload_type = kRedPayloadType; send_config->rtp.ulpfec.red_payload_type = kRedPayloadType;
send_config->rtp.ulpfec.ulpfec_payload_type = kUlpfecPayloadType; send_config->rtp.ulpfec.ulpfec_payload_type = kUlpfecPayloadType;
(*receive_configs)[0].rtp.transport_cc = true;
(*receive_configs)[0].rtp.ulpfec.red_payload_type = kRedPayloadType; (*receive_configs)[0].rtp.ulpfec.red_payload_type = kRedPayloadType;
(*receive_configs)[0].rtp.ulpfec.ulpfec_payload_type = kUlpfecPayloadType; (*receive_configs)[0].rtp.ulpfec.ulpfec_payload_type = kUlpfecPayloadType;
(*receive_configs)[0].renderer = this; (*receive_configs)[0].renderer = this;
@ -698,10 +685,11 @@ TEST_P(EndToEndTest, DISABLED_CanReceiveUlpfec) {
} }
rtc::CriticalSection crit_; rtc::CriticalSection crit_;
std::set<uint32_t> protected_sequence_numbers_ GUARDED_BY(crit_); std::set<uint32_t> dropped_sequence_numbers_ GUARDED_BY(crit_);
// Since several packets can have the same timestamp a multiset is used // Several packets can have the same timestamp.
// instead of a set. std::multiset<uint32_t> dropped_timestamps_ GUARDED_BY(crit_);
std::multiset<uint32_t> protected_timestamps_ GUARDED_BY(crit_); Random random_;
int num_packets_sent_;
} test; } test;
RunBaseTest(&test); RunBaseTest(&test);
@ -713,11 +701,13 @@ class FlexfecRenderObserver : public test::EndToEndTest,
static constexpr uint32_t kVideoLocalSsrc = 123; static constexpr uint32_t kVideoLocalSsrc = 123;
static constexpr uint32_t kFlexfecLocalSsrc = 456; static constexpr uint32_t kFlexfecLocalSsrc = 456;
explicit FlexfecRenderObserver(bool expect_flexfec_rtcp) explicit FlexfecRenderObserver(bool enable_nack, bool expect_flexfec_rtcp)
: test::EndToEndTest(test::CallTest::kDefaultTimeoutMs), : test::EndToEndTest(test::CallTest::kDefaultTimeoutMs),
enable_nack_(enable_nack),
expect_flexfec_rtcp_(expect_flexfec_rtcp), expect_flexfec_rtcp_(expect_flexfec_rtcp),
received_flexfec_rtcp_(false), received_flexfec_rtcp_(false),
random_(0xcafef00d1) {} random_(0xcafef00d1),
num_packets_sent_(0) {}
size_t GetNumFlexfecStreams() const override { return 1; } size_t GetNumFlexfecStreams() const override { return 1; }
@ -727,33 +717,55 @@ class FlexfecRenderObserver : public test::EndToEndTest,
RTPHeader header; RTPHeader header;
EXPECT_TRUE(parser_->Parse(packet, length, &header)); EXPECT_TRUE(parser_->Parse(packet, length, &header));
uint8_t payload_type = header.payloadType; EXPECT_TRUE(header.payloadType ==
if (payload_type != test::CallTest::kFakeVideoSendPayloadType) { test::CallTest::kFakeVideoSendPayloadType ||
EXPECT_EQ(test::CallTest::kFlexfecPayloadType, payload_type); header.payloadType == test::CallTest::kFlexfecPayloadType ||
(enable_nack_ &&
header.payloadType == test::CallTest::kSendRtxPayloadType))
<< "Unknown payload type received.";
EXPECT_TRUE(
header.ssrc == test::CallTest::kVideoSendSsrcs[0] ||
header.ssrc == test::CallTest::kFlexfecSendSsrc ||
(enable_nack_ && header.ssrc == test::CallTest::kSendRtxSsrcs[0]))
<< "Unknown SSRC received.";
// To reduce test flakiness, always let FlexFEC packets through.
if (header.payloadType == test::CallTest::kFlexfecPayloadType) {
EXPECT_EQ(test::CallTest::kFlexfecSendSsrc, header.ssrc);
return SEND_PACKET;
} }
// Is this a retransmitted media packet? From the perspective of FEC, this // To reduce test flakiness, always let RTX packets through.
// packet is then no longer dropped, so remove it from the list of if (header.payloadType == test::CallTest::kSendRtxPayloadType) {
// dropped packets. EXPECT_EQ(test::CallTest::kSendRtxSsrcs[0], header.ssrc);
if (payload_type == test::CallTest::kFakeVideoSendPayloadType) {
auto seq_num_it = dropped_sequence_numbers_.find(header.sequenceNumber); // Parse RTX header.
uint16_t original_sequence_number =
ByteReader<uint16_t>::ReadBigEndian(&packet[header.headerLength]);
// From the perspective of FEC, a retransmitted packet is no longer
// dropped, so remove it from list of dropped packets.
auto seq_num_it =
dropped_sequence_numbers_.find(original_sequence_number);
if (seq_num_it != dropped_sequence_numbers_.end()) { if (seq_num_it != dropped_sequence_numbers_.end()) {
dropped_sequence_numbers_.erase(seq_num_it); dropped_sequence_numbers_.erase(seq_num_it);
auto ts_it = dropped_timestamps_.find(header.timestamp); auto ts_it = dropped_timestamps_.find(header.timestamp);
EXPECT_NE(ts_it, dropped_timestamps_.end()); EXPECT_NE(ts_it, dropped_timestamps_.end());
dropped_timestamps_.erase(ts_it); dropped_timestamps_.erase(ts_it);
return SEND_PACKET;
} }
return SEND_PACKET;
} }
// Simulate 5% packet loss. Record what media packets, and corresponding // Simulate 5% video packet loss after rampup period. Record the
// timestamps, that were dropped. // corresponding timestamps that were dropped.
if (random_.Rand(1, 100) <= 5) { if (num_packets_sent_++ > 100 && random_.Rand(1, 100) <= 5) {
if (payload_type == test::CallTest::kFakeVideoSendPayloadType) { EXPECT_EQ(test::CallTest::kFakeVideoSendPayloadType, header.payloadType);
dropped_sequence_numbers_.insert(header.sequenceNumber); EXPECT_EQ(test::CallTest::kVideoSendSsrcs[0], header.ssrc);
dropped_timestamps_.insert(header.timestamp);
} dropped_sequence_numbers_.insert(header.sequenceNumber);
dropped_timestamps_.insert(header.timestamp);
return DROP_PACKET; return DROP_PACKET;
} }
@ -781,6 +793,15 @@ class FlexfecRenderObserver : public test::EndToEndTest,
return SEND_PACKET; return SEND_PACKET;
} }
test::PacketTransport* CreateSendTransport(Call* sender_call) override {
// At low RTT (< kLowRttNackMs) -> NACK only, no FEC.
const int kNetworkDelayMs = 100;
FakeNetworkPipe::Config config;
config.queue_delay_ms = kNetworkDelayMs;
return new test::PacketTransport(sender_call, this,
test::PacketTransport::kSender, config);
}
void OnFrame(const VideoFrame& video_frame) override { void OnFrame(const VideoFrame& video_frame) override {
rtc::CritScope lock(&crit_); rtc::CritScope lock(&crit_);
// Rendering frame with timestamp of packet that was dropped -> FEC // Rendering frame with timestamp of packet that was dropped -> FEC
@ -797,8 +818,31 @@ class FlexfecRenderObserver : public test::EndToEndTest,
VideoSendStream::Config* send_config, VideoSendStream::Config* send_config,
std::vector<VideoReceiveStream::Config>* receive_configs, std::vector<VideoReceiveStream::Config>* receive_configs,
VideoEncoderConfig* encoder_config) override { VideoEncoderConfig* encoder_config) override {
send_config->rtp.extensions.push_back(
RtpExtension(RtpExtension::kTransportSequenceNumberUri,
test::kTransportSequenceNumberExtensionId));
(*receive_configs)[0].rtp.local_ssrc = kVideoLocalSsrc; (*receive_configs)[0].rtp.local_ssrc = kVideoLocalSsrc;
(*receive_configs)[0].rtp.transport_cc = true;
(*receive_configs)[0].renderer = this; (*receive_configs)[0].renderer = this;
if (enable_nack_) {
send_config->rtp.nack.rtp_history_ms = test::CallTest::kNackRtpHistoryMs;
send_config->rtp.ulpfec.red_rtx_payload_type =
test::CallTest::kRtxRedPayloadType;
send_config->rtp.rtx.ssrcs.push_back(test::CallTest::kSendRtxSsrcs[0]);
send_config->rtp.rtx.payload_type = test::CallTest::kSendRtxPayloadType;
(*receive_configs)[0].rtp.nack.rtp_history_ms =
test::CallTest::kNackRtpHistoryMs;
(*receive_configs)[0].rtp.ulpfec.red_rtx_payload_type =
test::CallTest::kRtxRedPayloadType;
(*receive_configs)[0].rtp.rtx_ssrc = test::CallTest::kSendRtxSsrcs[0];
(*receive_configs)[0]
.rtp.rtx_payload_types[test::CallTest::kVideoSendPayloadType] =
test::CallTest::kSendRtxPayloadType;
}
} }
void ModifyFlexfecConfigs( void ModifyFlexfecConfigs(
@ -813,25 +857,27 @@ class FlexfecRenderObserver : public test::EndToEndTest,
rtc::CriticalSection crit_; rtc::CriticalSection crit_;
std::set<uint32_t> dropped_sequence_numbers_ GUARDED_BY(crit_); std::set<uint32_t> dropped_sequence_numbers_ GUARDED_BY(crit_);
// Since several packets can have the same timestamp a multiset is used // Several packets can have the same timestamp.
// instead of a set.
std::multiset<uint32_t> dropped_timestamps_ GUARDED_BY(crit_); std::multiset<uint32_t> dropped_timestamps_ GUARDED_BY(crit_);
const bool enable_nack_;
const bool expect_flexfec_rtcp_; const bool expect_flexfec_rtcp_;
bool received_flexfec_rtcp_ GUARDED_BY(crit_); bool received_flexfec_rtcp_ GUARDED_BY(crit_);
Random random_; Random random_;
int num_packets_sent_;
}; };
// Disable due to failure, see bugs.webrtc.org/7050 for TEST_P(EndToEndTest, RecoversWithFlexfec) {
// details. FlexfecRenderObserver test(false, false);
TEST_P(EndToEndTest, DISABLED_ReceivesFlexfec) {
FlexfecRenderObserver test(false);
RunBaseTest(&test); RunBaseTest(&test);
} }
// Disable due to failure, see bugs.webrtc.org/7050 for TEST_P(EndToEndTest, RecoversWithFlexfecAndNack) {
// details. FlexfecRenderObserver test(true, false);
TEST_P(EndToEndTest, DISABLED_ReceivesFlexfecAndSendsCorrespondingRtcp) { RunBaseTest(&test);
FlexfecRenderObserver test(true); }
TEST_P(EndToEndTest, RecoversWithFlexfecAndSendsCorrespondingRtcp) {
FlexfecRenderObserver test(false, true);
RunBaseTest(&test); RunBaseTest(&test);
} }