zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow providing the absolute capture time extension when packetizing a frame.

Browse Source 
Bug: b/150859541
Change-Id: Iffb6ee84f49ffa64fdb0633248864d2dfd6e9ff3
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/234868
Commit-Queue: Paul Hallak <phallak@google.com>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35194}
This commit is contained in:
Paul Hallak
2021-10-13 12:44:02 +02:00
committed by WebRTC LUCI CQ
parent ab9ed5c305
commit af1038d97c
5 changed files with 110 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
modules/rtp_rtcp/BUILD.gn
View File

@ -398,6 +398,7 @@ rtc_library("rtp_video_header") {
"source/rtp_video_header.h",
]
deps = [
"../../api:rtp_headers",
"../../api/transport/rtp:dependency_descriptor",
"../../api/video:video_frame",
"../../api/video:video_frame_type",

41
modules/rtp_rtcp/source/rtp_sender_video.cc
View File

@ -315,9 +315,7 @@ void RTPSenderVideo::SetVideoLayersAllocationInternal(
allocation_ = std::move(allocation);
}
void RTPSenderVideo::AddRtpHeaderExtensions(
const RTPVideoHeader& video_header,
const absl::optional<AbsoluteCaptureTime>& absolute_capture_time,
void RTPSenderVideo::AddRtpHeaderExtensions(const RTPVideoHeader& video_header,
bool first_packet,
bool last_packet,
RtpPacketToSend* packet) const {
@ -369,8 +367,9 @@ void RTPSenderVideo::AddRtpHeaderExtensions(
packet->SetExtension<PlayoutDelayLimits>(current_playout_delay_);
}
if (first_packet && absolute_capture_time) {
packet->SetExtension<AbsoluteCaptureTimeExtension>(*absolute_capture_time);
if (first_packet && video_header.absolute_capture_time.has_value()) {
packet->SetExtension<AbsoluteCaptureTimeExtension>(
*video_header.absolute_capture_time);
}
if (video_header.generic) {
@ -477,8 +476,7 @@ bool RTPSenderVideo::SendVideo(
int64_t capture_time_ms,
rtc::ArrayView<const uint8_t> payload,
RTPVideoHeader video_header,
absl::optional<int64_t> expected_retransmission_time_ms,
absl::optional<int64_t> estimated_capture_clock_offset_ms) {
absl::optional<int64_t> expected_retransmission_time_ms) {
#if RTC_TRACE_EVENTS_ENABLED
TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", capture_time_ms, "Send", "type",
FrameTypeToString(video_header.frame_type));
@ -538,22 +536,31 @@ bool RTPSenderVideo::SendVideo(
single_packet->SetTimestamp(rtp_timestamp);
single_packet->set_capture_time_ms(capture_time_ms);
const absl::optional<AbsoluteCaptureTime> absolute_capture_time =
// Construct the absolute capture time extension if not provided.
if (!video_header.absolute_capture_time.has_value()) {
video_header.absolute_capture_time.emplace();
video_header.absolute_capture_time->absolute_capture_timestamp =
Int64MsToUQ32x32(
clock_->ConvertTimestampToNtpTimeInMilliseconds(capture_time_ms));
if (include_capture_clock_offset_) {
video_header.absolute_capture_time->estimated_capture_clock_offset = 0;
}
}
// Let `absolute_capture_time_sender_` decide if the extension should be sent.
video_header.absolute_capture_time =
absolute_capture_time_sender_.OnSendPacket(
AbsoluteCaptureTimeSender::GetSource(single_packet->Ssrc(),
single_packet->Csrcs()),
single_packet->Timestamp(), kVideoPayloadTypeFrequency,
Int64MsToUQ32x32(
clock_->ConvertTimestampToNtpTimeInMilliseconds(capture_time_ms)),
/*estimated_capture_clock_offset=*/
include_capture_clock_offset_ ? estimated_capture_clock_offset_ms
: absl::nullopt);
video_header.absolute_capture_time->absolute_capture_timestamp,
video_header.absolute_capture_time->estimated_capture_clock_offset);
auto first_packet = std::make_unique<RtpPacketToSend>(*single_packet);
auto middle_packet = std::make_unique<RtpPacketToSend>(*single_packet);
auto last_packet = std::make_unique<RtpPacketToSend>(*single_packet);
// Simplest way to estimate how much extensions would occupy is to set them.
AddRtpHeaderExtensions(video_header, absolute_capture_time,
AddRtpHeaderExtensions(video_header,
/*first_packet=*/true, /*last_packet=*/true,
single_packet.get());
if (video_structure_ != nullptr &&
@ -568,13 +575,13 @@ bool RTPSenderVideo::SendVideo(
video_structure_ = nullptr;
}
AddRtpHeaderExtensions(video_header, absolute_capture_time,
AddRtpHeaderExtensions(video_header,
/*first_packet=*/true, /*last_packet=*/false,
first_packet.get());
AddRtpHeaderExtensions(video_header, absolute_capture_time,
AddRtpHeaderExtensions(video_header,
/*first_packet=*/false, /*last_packet=*/false,
middle_packet.get());
AddRtpHeaderExtensions(video_header, absolute_capture_time,
AddRtpHeaderExtensions(video_header,
/*first_packet=*/false, /*last_packet=*/true,
last_packet.get());

14
modules/rtp_rtcp/source/rtp_sender_video.h
View File

@ -90,20 +90,14 @@ class RTPSenderVideo {
// expected_retransmission_time_ms.has_value() -> retransmission allowed.
// `capture_time_ms` and `clock::CurrentTime` should be using the same epoch.
// Calls to this method is assumed to be externally serialized.
// `estimated_capture_clock_offset_ms` is an estimated clock offset between
// this sender and the original capturer, for this video packet. See
// http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time for more
// details. If the sender and the capture has the same clock, it is supposed
// to be zero valued, which is given as the default.
// Calls to this method are assumed to be externally serialized.
bool SendVideo(int payload_type,
absl::optional<VideoCodecType> codec_type,
uint32_t rtp_timestamp,
int64_t capture_time_ms,
rtc::ArrayView<const uint8_t> payload,
RTPVideoHeader video_header,
absl::optional<int64_t> expected_retransmission_time_ms,
absl::optional<int64_t> estimated_capture_clock_offset_ms = 0);
absl::optional<int64_t> expected_retransmission_time_ms);
bool SendEncodedImage(
int payload_type,
@ -170,9 +164,7 @@ class RTPSenderVideo {
const FrameDependencyStructure* video_structure);
void SetVideoLayersAllocationInternal(VideoLayersAllocation allocation);
void AddRtpHeaderExtensions(
const RTPVideoHeader& video_header,
const absl::optional<AbsoluteCaptureTime>& absolute_capture_time,
void AddRtpHeaderExtensions(const RTPVideoHeader& video_header,
bool first_packet,
bool last_packet,
RtpPacketToSend* packet) const

83
modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
View File

@ -16,6 +16,7 @@
#include <vector>
#include "absl/memory/memory.h"
#include "api/rtp_headers.h"
#include "api/test/mock_frame_encryptor.h"
#include "api/transport/field_trial_based_config.h"
#include "api/transport/rtp/dependency_descriptor.h"
@ -1181,15 +1182,21 @@ TEST_P(RtpSenderVideoTest, AbsoluteCaptureTime) {
kAbsoluteCaptureTimestampMs, kFrame, hdr,
kDefaultExpectedRetransmissionTimeMs);
absl::optional<AbsoluteCaptureTime> absolute_capture_time;
// It is expected that one and only one of the packets sent on this video
// frame has absolute capture time header extension. And no absolute capture
// time header extensions include capture clock offset.
int packets_with_abs_capture_time = 0;
// frame has absolute capture time header extension.
for (const RtpPacketReceived& packet : transport_.sent_packets()) {
auto absolute_capture_time =
if (absolute_capture_time.has_value()) {
EXPECT_FALSE(packet.HasExtension<AbsoluteCaptureTimeExtension>());
} else {
absolute_capture_time =
packet.GetExtension<AbsoluteCaptureTimeExtension>();
if (absolute_capture_time) {
++packets_with_abs_capture_time;
}
}
// Verify the capture timestamp and that the clock offset is not set.
ASSERT_TRUE(absolute_capture_time.has_value());
EXPECT_EQ(
absolute_capture_time->absolute_capture_timestamp,
Int64MsToUQ32x32(fake_clock_.ConvertTimestampToNtpTimeInMilliseconds(
@ -1197,9 +1204,6 @@ TEST_P(RtpSenderVideoTest, AbsoluteCaptureTime) {
EXPECT_FALSE(
absolute_capture_time->estimated_capture_clock_offset.has_value());
}
}
EXPECT_EQ(packets_with_abs_capture_time, 1);
}
// Essentially the same test as AbsoluteCaptureTime but with a field trial.
// After the field trial is experimented, we will remove AbsoluteCaptureTime.
@ -1214,31 +1218,64 @@ TEST_P(RtpSenderVideoTest, AbsoluteCaptureTimeWithCaptureClockOffset) {
kAbsoluteCaptureTimeExtensionId);
RTPVideoHeader hdr;
const absl::optional<int64_t> kExpectedCaptureClockOffset =
absl::make_optional(1234);
hdr.frame_type = VideoFrameType::kVideoFrameKey;
rtp_sender_video_->SendVideo(
kPayload, kType, kTimestamp, kAbsoluteCaptureTimestampMs, kFrame, hdr,
kDefaultExpectedRetransmissionTimeMs, kExpectedCaptureClockOffset);
rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp,
kAbsoluteCaptureTimestampMs, kFrame, hdr,
kDefaultExpectedRetransmissionTimeMs);
absl::optional<AbsoluteCaptureTime> absolute_capture_time;
// It is expected that one and only one of the packets sent on this video
// frame has absolute capture time header extension. And it includes capture
// clock offset.
int packets_with_abs_capture_time = 0;
// frame has absolute capture time header extension.
for (const RtpPacketReceived& packet : transport_.sent_packets()) {
auto absolute_capture_time =
if (absolute_capture_time.has_value()) {
EXPECT_FALSE(packet.HasExtension<AbsoluteCaptureTimeExtension>());
} else {
absolute_capture_time =
packet.GetExtension<AbsoluteCaptureTimeExtension>();
if (absolute_capture_time) {
++packets_with_abs_capture_time;
}
}
// Verify the capture timestamp and that the clock offset is set to zero.
ASSERT_TRUE(absolute_capture_time.has_value());
EXPECT_EQ(
absolute_capture_time->absolute_capture_timestamp,
Int64MsToUQ32x32(fake_clock_.ConvertTimestampToNtpTimeInMilliseconds(
kAbsoluteCaptureTimestampMs)));
EXPECT_EQ(kExpectedCaptureClockOffset,
absolute_capture_time->estimated_capture_clock_offset);
EXPECT_EQ(absolute_capture_time->estimated_capture_clock_offset, 0);
}
TEST_P(RtpSenderVideoTest, AbsoluteCaptureTimeWithExtensionProvided) {
constexpr AbsoluteCaptureTime kAbsoluteCaptureTime = {
123,
absl::optional<int64_t>(456),
};
uint8_t kFrame[kMaxPacketLength];
rtp_module_->RegisterRtpHeaderExtension(AbsoluteCaptureTimeExtension::Uri(),
kAbsoluteCaptureTimeExtensionId);
RTPVideoHeader hdr;
hdr.frame_type = VideoFrameType::kVideoFrameKey;
hdr.absolute_capture_time = kAbsoluteCaptureTime;
rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp,
/*capture_time_ms=*/789, kFrame, hdr,
kDefaultExpectedRetransmissionTimeMs);
absl::optional<AbsoluteCaptureTime> absolute_capture_time;
// It is expected that one and only one of the packets sent on this video
// frame has absolute capture time header extension.
for (const RtpPacketReceived& packet : transport_.sent_packets()) {
if (absolute_capture_time.has_value()) {
EXPECT_FALSE(packet.HasExtension<AbsoluteCaptureTimeExtension>());
} else {
absolute_capture_time =
packet.GetExtension<AbsoluteCaptureTimeExtension>();
}
}
EXPECT_EQ(packets_with_abs_capture_time, 1);
// Verify the extension.
EXPECT_EQ(absolute_capture_time, kAbsoluteCaptureTime);
}
TEST_P(RtpSenderVideoTest, PopulatesPlayoutDelay) {

6
modules/rtp_rtcp/source/rtp_video_header.h
View File

@ -16,6 +16,7 @@
#include "absl/container/inlined_vector.h"
#include "absl/types/optional.h"
#include "absl/types/variant.h"
#include "api/rtp_headers.h"
#include "api/transport/rtp/dependency_descriptor.h"
#include "api/video/color_space.h"
#include "api/video/video_codec_type.h"
@ -81,6 +82,11 @@ struct RTPVideoHeader {
// carries the webrtc::VideoFrame id field from the sender to the receiver.
absl::optional<uint16_t> video_frame_tracking_id;
RTPVideoTypeHeader video_type_header;
// When provided, is sent as is as an RTP header extension according to
// http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time.
// Otherwise, it is derived from other relevant information.
absl::optional<AbsoluteCaptureTime> absolute_capture_time;
};
} // namespace webrtc