Persist RTP state for FlexFEC.

Before this CL, the RTP state would be re-randomized after a
recreation of VideoSendStream. That might lead to us sending
a non-compliant RTP stream, which is avoided after the
changes in this CL.

BUG=webrtc:5654
TBR=pbos@webrtc.org  # Trivial change to fuzzer.

Review-Url: https://codereview.webrtc.org/2912713002
Cr-Commit-Position: refs/heads/master@{#18322}
This commit is contained in:
brandtr
2017-05-30 02:32:12 -07:00
committed by Commit Bot
parent bb6f7524ba
commit 48d21a23c6
7 changed files with 238 additions and 24 deletions

View File

@ -17,10 +17,10 @@
#include "webrtc/base/array_view.h"
#include "webrtc/base/basictypes.h"
#include "webrtc/base/random.h"
#include "webrtc/base/sequenced_task_checker.h"
#include "webrtc/config.h"
#include "webrtc/modules/include/module_common_types.h"
#include "webrtc/modules/rtp_rtcp/include/flexfec_sender.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "webrtc/modules/rtp_rtcp/source/rtp_header_extension.h"
#include "webrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h"
#include "webrtc/modules/rtp_rtcp/source/ulpfec_generator.h"
@ -40,6 +40,7 @@ class FlexfecSender {
uint32_t protected_media_ssrc,
const std::vector<RtpExtension>& rtp_header_extensions,
rtc::ArrayView<const RtpExtensionSize> extension_sizes,
const RtpState* rtp_state,
Clock* clock);
~FlexfecSender();
@ -64,6 +65,9 @@ class FlexfecSender {
// Returns the overhead, per packet, for FlexFEC.
size_t MaxPacketOverhead() const;
// Only called on the VideoSendStream queue, after operation has shut down.
RtpState GetRtpState();
private:
// Utility.
Clock* const clock_;

View File

@ -65,17 +65,21 @@ FlexfecSender::FlexfecSender(
uint32_t protected_media_ssrc,
const std::vector<RtpExtension>& rtp_header_extensions,
rtc::ArrayView<const RtpExtensionSize> extension_sizes,
const RtpState* rtp_state,
Clock* clock)
: clock_(clock),
random_(clock_->TimeInMicroseconds()),
last_generated_packet_ms_(-1),
payload_type_(payload_type),
// Initialize the timestamp offset and RTP sequence numbers randomly.
// (This is not intended to be cryptographically strong.)
timestamp_offset_(random_.Rand<uint32_t>()),
// Reset RTP state if this is not the first time we are operating.
// Otherwise, randomize the initial timestamp offset and RTP sequence
// numbers. (This is not intended to be cryptographically strong.)
timestamp_offset_(rtp_state ? rtp_state->start_timestamp
: random_.Rand<uint32_t>()),
ssrc_(ssrc),
protected_media_ssrc_(protected_media_ssrc),
seq_num_(random_.Rand(1, kMaxInitRtpSeqNumber)),
seq_num_(rtp_state ? rtp_state->sequence_number
: random_.Rand(1, kMaxInitRtpSeqNumber)),
ulpfec_generator_(ForwardErrorCorrection::CreateFlexfec()),
rtp_header_extension_map_(RegisterBweExtensions(rtp_header_extensions)),
header_extensions_size_(
@ -154,4 +158,11 @@ size_t FlexfecSender::MaxPacketOverhead() const {
return header_extensions_size_ + kFlexfecMaxHeaderSize;
}
RtpState FlexfecSender::GetRtpState() {
RtpState rtp_state;
rtp_state.sequence_number = seq_num_;
rtp_state.start_timestamp = timestamp_offset_;
return rtp_state;
}
} // namespace webrtc

View File

@ -78,7 +78,7 @@ TEST(FlexfecSenderTest, Ssrc) {
SimulatedClock clock(kInitialSimulatedClockTime);
FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc,
kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
&clock);
nullptr /* rtp_state */, &clock);
EXPECT_EQ(kFlexfecSsrc, sender.ssrc());
}
@ -87,7 +87,7 @@ TEST(FlexfecSenderTest, NoFecAvailableBeforeMediaAdded) {
SimulatedClock clock(kInitialSimulatedClockTime);
FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc,
kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
&clock);
nullptr /* rtp_state */, &clock);
EXPECT_FALSE(sender.FecAvailable());
auto fec_packets = sender.GetFecPackets();
@ -98,7 +98,7 @@ TEST(FlexfecSenderTest, ProtectOneFrameWithOneFecPacket) {
SimulatedClock clock(kInitialSimulatedClockTime);
FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc,
kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
&clock);
nullptr /* rtp_state */, &clock);
auto fec_packet = GenerateSingleFlexfecPacket(&sender);
EXPECT_EQ(kRtpHeaderSize, fec_packet->headers_size());
@ -121,7 +121,7 @@ TEST(FlexfecSenderTest, ProtectTwoFramesWithOneFecPacket) {
SimulatedClock clock(kInitialSimulatedClockTime);
FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc,
kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
&clock);
nullptr /* rtp_state */, &clock);
sender.SetFecParameters(params);
AugmentedPacketGenerator packet_generator(kMediaSsrc);
@ -161,7 +161,7 @@ TEST(FlexfecSenderTest, ProtectTwoFramesWithTwoFecPackets) {
SimulatedClock clock(kInitialSimulatedClockTime);
FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc,
kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
&clock);
nullptr /* rtp_state */, &clock);
sender.SetFecParameters(params);
AugmentedPacketGenerator packet_generator(kMediaSsrc);
@ -197,7 +197,7 @@ TEST(FlexfecSenderTest, NoRtpHeaderExtensionsForBweByDefault) {
SimulatedClock clock(kInitialSimulatedClockTime);
FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc,
kRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
&clock);
nullptr /* rtp_state */, &clock);
auto fec_packet = GenerateSingleFlexfecPacket(&sender);
EXPECT_FALSE(fec_packet->HasExtension<AbsoluteSendTime>());
@ -211,7 +211,7 @@ TEST(FlexfecSenderTest, RegisterAbsoluteSendTimeRtpHeaderExtension) {
SimulatedClock clock(kInitialSimulatedClockTime);
FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc,
kRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
&clock);
nullptr /* rtp_state */, &clock);
auto fec_packet = GenerateSingleFlexfecPacket(&sender);
EXPECT_TRUE(fec_packet->HasExtension<AbsoluteSendTime>());
@ -225,7 +225,7 @@ TEST(FlexfecSenderTest, RegisterTransmissionOffsetRtpHeaderExtension) {
SimulatedClock clock(kInitialSimulatedClockTime);
FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc,
kRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
&clock);
nullptr /* rtp_state */, &clock);
auto fec_packet = GenerateSingleFlexfecPacket(&sender);
EXPECT_FALSE(fec_packet->HasExtension<AbsoluteSendTime>());
@ -239,7 +239,7 @@ TEST(FlexfecSenderTest, RegisterTransportSequenceNumberRtpHeaderExtension) {
SimulatedClock clock(kInitialSimulatedClockTime);
FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc,
kRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
&clock);
nullptr /* rtp_state */, &clock);
auto fec_packet = GenerateSingleFlexfecPacket(&sender);
EXPECT_FALSE(fec_packet->HasExtension<AbsoluteSendTime>());
@ -255,7 +255,7 @@ TEST(FlexfecSenderTest, RegisterAllRtpHeaderExtensionsForBwe) {
SimulatedClock clock(kInitialSimulatedClockTime);
FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc,
kRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
&clock);
nullptr /* rtp_state */, &clock);
auto fec_packet = GenerateSingleFlexfecPacket(&sender);
EXPECT_TRUE(fec_packet->HasExtension<AbsoluteSendTime>());
@ -267,7 +267,7 @@ TEST(FlexfecSenderTest, MaxPacketOverhead) {
SimulatedClock clock(kInitialSimulatedClockTime);
FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc,
kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
&clock);
nullptr /* rtp_state */, &clock);
EXPECT_EQ(kFlexfecMaxHeaderSize, sender.MaxPacketOverhead());
}
@ -287,10 +287,37 @@ TEST(FlexfecSenderTest, MaxPacketOverheadWithExtensions) {
kExtensionHeaderLength + TransportSequenceNumber::kValueSizeBytes);
FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc,
kRtpHeaderExtensions, RTPSender::FecExtensionSizes(),
&clock);
nullptr /* rtp_state */, &clock);
EXPECT_EQ(kExtensionsTotalSize + kFlexfecMaxHeaderSize,
sender.MaxPacketOverhead());
}
TEST(FlexfecSenderTest, SetsAndGetsRtpState) {
RtpState initial_rtp_state;
initial_rtp_state.sequence_number = 100;
initial_rtp_state.start_timestamp = 200;
SimulatedClock clock(kInitialSimulatedClockTime);
FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc,
kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
&initial_rtp_state, &clock);
auto fec_packet = GenerateSingleFlexfecPacket(&sender);
EXPECT_EQ(initial_rtp_state.sequence_number, fec_packet->SequenceNumber());
EXPECT_EQ(initial_rtp_state.start_timestamp, fec_packet->Timestamp());
clock.AdvanceTimeMilliseconds(1000);
fec_packet = GenerateSingleFlexfecPacket(&sender);
EXPECT_EQ(initial_rtp_state.sequence_number + 1,
fec_packet->SequenceNumber());
EXPECT_EQ(initial_rtp_state.start_timestamp + 1 * kVideoPayloadTypeFrequency,
fec_packet->Timestamp());
RtpState updated_rtp_state = sender.GetRtpState();
EXPECT_EQ(initial_rtp_state.sequence_number + 2,
updated_rtp_state.sequence_number);
EXPECT_EQ(initial_rtp_state.start_timestamp,
updated_rtp_state.start_timestamp);
}
} // namespace webrtc

View File

@ -846,7 +846,7 @@ TEST_P(RtpSenderTest, SendFlexfecPackets) {
const std::vector<RtpExtensionSize> kNoRtpExtensionSizes;
FlexfecSender flexfec_sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc,
kNoRtpExtensions, kNoRtpExtensionSizes,
&fake_clock_);
nullptr /* rtp_state */, &fake_clock_);
// Reset |rtp_sender_| to use FlexFEC.
rtp_sender_.reset(new RTPSender(
@ -903,7 +903,7 @@ TEST_P(RtpSenderTestWithoutPacer, SendFlexfecPackets) {
const std::vector<RtpExtensionSize> kNoRtpExtensionSizes;
FlexfecSender flexfec_sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc,
kNoRtpExtensions, kNoRtpExtensionSizes,
&fake_clock_);
nullptr /* rtp_state */, &fake_clock_);
// Reset |rtp_sender_| to use FlexFEC.
rtp_sender_.reset(new RTPSender(false, &fake_clock_, &transport_, nullptr,
@ -944,7 +944,7 @@ TEST_P(RtpSenderTest, FecOverheadRate) {
const std::vector<RtpExtensionSize> kNoRtpExtensionSizes;
FlexfecSender flexfec_sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc,
kNoRtpExtensions, kNoRtpExtensionSizes,
&fake_clock_);
nullptr /* rtp_state */, &fake_clock_);
// Reset |rtp_sender_| to use FlexFEC.
rtp_sender_.reset(new RTPSender(

View File

@ -37,7 +37,7 @@ void FuzzOneInput(const uint8_t* data, size_t size) {
SimulatedClock clock(1 + data[i++]);
FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc,
kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
&clock);
nullptr /* rtp_state */, &clock);
FecProtectionParams params = {
data[i++], static_cast<int>(data[i++] % 100),
data[i++] <= 127 ? kFecMaskRandom : kFecMaskBursty};

View File

@ -4145,6 +4145,165 @@ TEST_F(EndToEndTest, MAYBE_PictureIdStateRetainedAfterReinitingVp8) {
TestPictureIdStatePreservation(encoder.get());
}
TEST_F(EndToEndTest, TestFlexfecRtpStatePreservation) {
class RtpSequenceObserver : public test::RtpRtcpObserver {
public:
RtpSequenceObserver()
: test::RtpRtcpObserver(kDefaultTimeoutMs),
num_flexfec_packets_sent_(0) {}
void ResetPacketCount() {
rtc::CritScope lock(&crit_);
num_flexfec_packets_sent_ = 0;
}
private:
Action OnSendRtp(const uint8_t* packet, size_t length) override {
rtc::CritScope lock(&crit_);
RTPHeader header;
EXPECT_TRUE(parser_->Parse(packet, length, &header));
const uint16_t sequence_number = header.sequenceNumber;
const uint32_t timestamp = header.timestamp;
const uint32_t ssrc = header.ssrc;
if (ssrc == kVideoSendSsrcs[0] || ssrc == kSendRtxSsrcs[0]) {
return SEND_PACKET;
}
EXPECT_EQ(kFlexfecSendSsrc, ssrc) << "Unknown SSRC sent.";
++num_flexfec_packets_sent_;
// If this is the first packet, we have nothing to compare to.
if (!last_observed_sequence_number_) {
last_observed_sequence_number_.emplace(sequence_number);
last_observed_timestamp_.emplace(timestamp);
return SEND_PACKET;
}
// Verify continuity and monotonicity of RTP sequence numbers.
EXPECT_EQ(static_cast<uint16_t>(*last_observed_sequence_number_ + 1),
sequence_number);
last_observed_sequence_number_.emplace(sequence_number);
// Timestamps should be non-decreasing...
const bool timestamp_is_same_or_newer =
timestamp == *last_observed_timestamp_ ||
IsNewerTimestamp(timestamp, *last_observed_timestamp_);
EXPECT_TRUE(timestamp_is_same_or_newer);
// ...but reasonably close in time.
const int k10SecondsInRtpTimestampBase = 10 * kVideoPayloadTypeFrequency;
EXPECT_TRUE(IsNewerTimestamp(
*last_observed_timestamp_ + k10SecondsInRtpTimestampBase, timestamp));
last_observed_timestamp_.emplace(timestamp);
// Pass test when enough packets have been let through.
if (num_flexfec_packets_sent_ >= 10) {
observation_complete_.Set();
}
return SEND_PACKET;
}
rtc::Optional<uint16_t> last_observed_sequence_number_ GUARDED_BY(crit_);
rtc::Optional<uint32_t> last_observed_timestamp_ GUARDED_BY(crit_);
size_t num_flexfec_packets_sent_ GUARDED_BY(crit_);
rtc::CriticalSection crit_;
} observer;
Call::Config config(event_log_.get());
CreateCalls(config, config);
FakeNetworkPipe::Config lossy_delayed_link;
lossy_delayed_link.loss_percent = 2;
lossy_delayed_link.queue_delay_ms = 50;
test::PacketTransport send_transport(sender_call_.get(), &observer,
test::PacketTransport::kSender,
payload_type_map_, lossy_delayed_link);
send_transport.SetReceiver(receiver_call_->Receiver());
FakeNetworkPipe::Config flawless_link;
test::PacketTransport receive_transport(nullptr, &observer,
test::PacketTransport::kReceiver,
payload_type_map_, flawless_link);
receive_transport.SetReceiver(sender_call_->Receiver());
// For reduced flakyness, we use a real VP8 encoder together with NACK
// and RTX.
const int kNumVideoStreams = 1;
const int kNumFlexfecStreams = 1;
CreateSendConfig(kNumVideoStreams, 0, kNumFlexfecStreams, &send_transport);
std::unique_ptr<VideoEncoder> encoder(VP8Encoder::Create());
video_send_config_.encoder_settings.encoder = encoder.get();
video_send_config_.encoder_settings.payload_name = "VP8";
video_send_config_.encoder_settings.payload_type = kVideoSendPayloadType;
video_send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
video_send_config_.rtp.rtx.ssrcs.push_back(kSendRtxSsrcs[0]);
video_send_config_.rtp.rtx.payload_type = kSendRtxPayloadType;
CreateMatchingReceiveConfigs(&receive_transport);
video_receive_configs_[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
video_receive_configs_[0].rtp.rtx_ssrc = kSendRtxSsrcs[0];
video_receive_configs_[0].rtp.rtx_payload_types[kVideoSendPayloadType] =
kSendRtxPayloadType;
// The matching FlexFEC receive config is not created by
// CreateMatchingReceiveConfigs since this is not a test::BaseTest.
// Set up the receive config manually instead.
FlexfecReceiveStream::Config flexfec_receive_config(&receive_transport);
flexfec_receive_config.payload_type =
video_send_config_.rtp.flexfec.payload_type;
flexfec_receive_config.remote_ssrc = video_send_config_.rtp.flexfec.ssrc;
flexfec_receive_config.protected_media_ssrcs =
video_send_config_.rtp.flexfec.protected_media_ssrcs;
flexfec_receive_config.local_ssrc = kReceiverLocalVideoSsrc;
flexfec_receive_config.transport_cc = true;
flexfec_receive_config.rtp_header_extensions.emplace_back(
RtpExtension::kTransportSequenceNumberUri,
test::kTransportSequenceNumberExtensionId);
flexfec_receive_configs_.push_back(flexfec_receive_config);
CreateFlexfecStreams();
CreateVideoStreams();
// RTCP might be disabled if the network is "down".
sender_call_->SignalChannelNetworkState(MediaType::VIDEO, kNetworkUp);
receiver_call_->SignalChannelNetworkState(MediaType::VIDEO, kNetworkUp);
const int kFrameMaxWidth = 320;
const int kFrameMaxHeight = 180;
const int kFrameRate = 15;
CreateFrameGeneratorCapturer(kFrameRate, kFrameMaxWidth, kFrameMaxHeight);
// Initial test.
Start();
EXPECT_TRUE(observer.Wait()) << "Timed out waiting for packets.";
// Ensure monotonicity when the VideoSendStream is restarted.
Stop();
observer.ResetPacketCount();
Start();
EXPECT_TRUE(observer.Wait()) << "Timed out waiting for packets.";
// Ensure monotonicity when the VideoSendStream is recreated.
frame_generator_capturer_->Stop();
sender_call_->DestroyVideoSendStream(video_send_stream_);
observer.ResetPacketCount();
video_send_stream_ = sender_call_->CreateVideoSendStream(
video_send_config_.Copy(), video_encoder_config_.Copy());
video_send_stream_->Start();
CreateFrameGeneratorCapturer(kFrameRate, kFrameMaxWidth, kFrameMaxHeight);
frame_generator_capturer_->Start();
EXPECT_TRUE(observer.Wait()) << "Timed out waiting for packets.";
// Cleanup.
send_transport.StopSending();
receive_transport.StopSending();
Stop();
DestroyStreams();
}
TEST_F(EndToEndTest,
MAYBE_PictureIdStateRetainedAfterReinitingSimulcastEncoderAdapter) {
class VideoEncoderFactoryAdapter : public webrtc::VideoEncoderFactory {

View File

@ -95,7 +95,8 @@ std::vector<RtpRtcp*> CreateRtpRtcpModules(
// TODO(brandtr): Update this function when we support multistream protection.
std::unique_ptr<FlexfecSender> MaybeCreateFlexfecSender(
const VideoSendStream::Config& config) {
const VideoSendStream::Config& config,
const std::map<uint32_t, RtpState>& suspended_ssrcs) {
if (config.rtp.flexfec.payload_type < 0) {
return nullptr;
}
@ -128,11 +129,17 @@ std::unique_ptr<FlexfecSender> MaybeCreateFlexfecSender(
return nullptr;
}
const RtpState* rtp_state = nullptr;
auto it = suspended_ssrcs.find(config.rtp.flexfec.ssrc);
if (it != suspended_ssrcs.end()) {
rtp_state = &it->second;
}
RTC_DCHECK_EQ(1U, config.rtp.flexfec.protected_media_ssrcs.size());
return std::unique_ptr<FlexfecSender>(new FlexfecSender(
config.rtp.flexfec.payload_type, config.rtp.flexfec.ssrc,
config.rtp.flexfec.protected_media_ssrcs[0], config.rtp.extensions,
RTPSender::FecExtensionSizes(), Clock::GetRealTimeClock()));
RTPSender::FecExtensionSizes(), rtp_state, Clock::GetRealTimeClock()));
}
} // namespace
@ -762,7 +769,7 @@ VideoSendStreamImpl::VideoSendStreamImpl(
call_stats_(call_stats),
transport_(transport),
bitrate_allocator_(bitrate_allocator),
flexfec_sender_(MaybeCreateFlexfecSender(*config_)),
flexfec_sender_(MaybeCreateFlexfecSender(*config_, suspended_ssrcs_)),
max_padding_bitrate_(0),
encoder_min_bitrate_bps_(0),
encoder_max_bitrate_bps_(initial_encoder_max_bitrate),
@ -1184,6 +1191,7 @@ void VideoSendStreamImpl::ConfigureSsrcs() {
std::map<uint32_t, RtpState> VideoSendStreamImpl::GetRtpStates() const {
RTC_DCHECK_RUN_ON(worker_queue_);
std::map<uint32_t, RtpState> rtp_states;
for (size_t i = 0; i < config_->rtp.ssrcs.size(); ++i) {
uint32_t ssrc = config_->rtp.ssrcs[i];
RTC_DCHECK_EQ(ssrc, rtp_rtcp_modules_[i]->SSRC());
@ -1195,6 +1203,11 @@ std::map<uint32_t, RtpState> VideoSendStreamImpl::GetRtpStates() const {
rtp_states[ssrc] = rtp_rtcp_modules_[i]->GetRtxState();
}
if (flexfec_sender_) {
uint32_t ssrc = config_->rtp.flexfec.ssrc;
rtp_states[ssrc] = flexfec_sender_->GetRtpState();
}
return rtp_states;
}