zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Android: Support externally aligned timestamps

Browse Source 
This support is needed if there is a big delay between the creation of
frames and the time they are delivered to the WebRTC C++ layer in
AndroidVideoTrackSource. This is the case if e.g. some heavy video
processing is applied to the frames that takes a couple of hundred
milliseconds. Currently, timestamps coming from Android video sources
are aligned to rtc::TimeMicros() once they reach the WebRTC C++ layer in
AndroidVideoTrackSource. At this point, we "forget" any latency that
might occur before this point, and audio/video sync consequently
suffers.

Bug: webrtc:9991
Change-Id: I7b1aaca9a60a978b9195dd5e5eed4779a0055607
Reviewed-on: https://webrtc-review.googlesource.com/c/110783
Commit-Queue: Magnus Jedvert <magjed@webrtc.org>
Reviewed-by: Sami Kalliomäki <sakal@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#25654}
This commit is contained in:
Magnus Jedvert
2018-11-15 12:07:32 +01:00
committed by Commit Bot
parent 2277ac6718
commit 9514071500
18 changed files with 298 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
sdk/android/native_unittests/org/webrtc/JavaVideoSourceTestHelper.java
View File

@ -24,8 +24,9 @@ public class JavaVideoSourceTestHelper {
}
@CalledByNative
public static void deliverFrame(int width, int height, int rotation, CapturerObserver observer) {
public static void deliverFrame(
int width, int height, int rotation, long timestampNs, CapturerObserver observer) {
observer.onFrameCaptured(
new VideoFrame(JavaI420Buffer.allocate(width, height), rotation, 0 /* timestampNs= */));
new VideoFrame(JavaI420Buffer.allocate(width, height), rotation, timestampNs));
}
}

67
sdk/android/native_unittests/video/videosource_unittest.cc
View File

@ -40,9 +40,9 @@ TEST(JavaVideoSourceTest, CreateJavaVideoSource) {
rtc::ThreadManager::Instance()->WrapCurrentThread();
rtc::scoped_refptr<JavaVideoTrackSourceInterface> video_track_source =
CreateJavaVideoSource(env,
rtc::ThreadManager::Instance()->CurrentThread(),
false /* is_screencast */);
CreateJavaVideoSource(
env, rtc::ThreadManager::Instance()->CurrentThread(),
false /* is_screencast */, true /* align_timestamps */);
ASSERT_NE(nullptr, video_track_source);
EXPECT_NE(nullptr,
@ -57,9 +57,9 @@ TEST(JavaVideoSourceTest, OnFrameCapturedFrameIsDeliveredToSink) {
rtc::ThreadManager::Instance()->WrapCurrentThread();
rtc::scoped_refptr<JavaVideoTrackSourceInterface> video_track_source =
CreateJavaVideoSource(env,
rtc::ThreadManager::Instance()->CurrentThread(),
false /* is_screencast */);
CreateJavaVideoSource(
env, rtc::ThreadManager::Instance()->CurrentThread(),
false /* is_screencast */, true /* align_timestamps */);
video_track_source->AddOrUpdateSink(&test_video_sink, rtc::VideoSinkWants());
jni::Java_JavaVideoSourceTestHelper_startCapture(
@ -68,8 +68,9 @@ TEST(JavaVideoSourceTest, OnFrameCapturedFrameIsDeliveredToSink) {
const int width = 20;
const int height = 32;
const int rotation = 180;
const int64_t timestamp = 987654321;
jni::Java_JavaVideoSourceTestHelper_deliverFrame(
env, width, height, rotation,
env, width, height, rotation, timestamp,
video_track_source->GetJavaVideoCapturerObserver(env));
std::vector<VideoFrame> frames = test_video_sink.GetFrames();
@ -80,15 +81,49 @@ TEST(JavaVideoSourceTest, OnFrameCapturedFrameIsDeliveredToSink) {
EXPECT_EQ(rotation, frame.rotation());
}
TEST(JavaVideoSourceTest,
OnFrameCapturedFrameIsDeliveredToSinkWithPreservedTimestamp) {
TestVideoSink test_video_sink;
JNIEnv* env = AttachCurrentThreadIfNeeded();
// Wrap test thread so it can be used as the signaling thread.
rtc::ThreadManager::Instance()->WrapCurrentThread();
rtc::scoped_refptr<JavaVideoTrackSourceInterface> video_track_source =
CreateJavaVideoSource(
env, rtc::ThreadManager::Instance()->CurrentThread(),
false /* is_screencast */, false /* align_timestamps */);
video_track_source->AddOrUpdateSink(&test_video_sink, rtc::VideoSinkWants());
jni::Java_JavaVideoSourceTestHelper_startCapture(
env, video_track_source->GetJavaVideoCapturerObserver(env),
true /* success */);
const int width = 20;
const int height = 32;
const int rotation = 180;
const int64_t timestamp = 987654321;
jni::Java_JavaVideoSourceTestHelper_deliverFrame(
env, width, height, rotation, 987654321,
video_track_source->GetJavaVideoCapturerObserver(env));
std::vector<VideoFrame> frames = test_video_sink.GetFrames();
ASSERT_EQ(1u, frames.size());
webrtc::VideoFrame frame = frames[0];
EXPECT_EQ(width, frame.width());
EXPECT_EQ(height, frame.height());
EXPECT_EQ(rotation, frame.rotation());
EXPECT_EQ(timestamp / 1000, frame.timestamp_us());
}
TEST(JavaVideoSourceTest, CapturerStartedSuccessStateBecomesLive) {
JNIEnv* env = AttachCurrentThreadIfNeeded();
// Wrap test thread so it can be used as the signaling thread.
rtc::ThreadManager::Instance()->WrapCurrentThread();
rtc::scoped_refptr<JavaVideoTrackSourceInterface> video_track_source =
CreateJavaVideoSource(env,
rtc::ThreadManager::Instance()->CurrentThread(),
false /* is_screencast */);
CreateJavaVideoSource(
env, rtc::ThreadManager::Instance()->CurrentThread(),
false /* is_screencast */, true /* align_timestamps */);
jni::Java_JavaVideoSourceTestHelper_startCapture(
env, video_track_source->GetJavaVideoCapturerObserver(env),
@ -104,9 +139,9 @@ TEST(JavaVideoSourceTest, CapturerStartedFailureStateBecomesEnded) {
rtc::ThreadManager::Instance()->WrapCurrentThread();
rtc::scoped_refptr<JavaVideoTrackSourceInterface> video_track_source =
CreateJavaVideoSource(env,
rtc::ThreadManager::Instance()->CurrentThread(),
false /* is_screencast */);
CreateJavaVideoSource(
env, rtc::ThreadManager::Instance()->CurrentThread(),
false /* is_screencast */, true /* align_timestamps */);
jni::Java_JavaVideoSourceTestHelper_startCapture(
env, video_track_source->GetJavaVideoCapturerObserver(env),
@ -122,9 +157,9 @@ TEST(JavaVideoSourceTest, CapturerStoppedStateBecomesEnded) {
rtc::ThreadManager::Instance()->WrapCurrentThread();
rtc::scoped_refptr<JavaVideoTrackSourceInterface> video_track_source =
CreateJavaVideoSource(env,
rtc::ThreadManager::Instance()->CurrentThread(),
false /* is_screencast */);
CreateJavaVideoSource(
env, rtc::ThreadManager::Instance()->CurrentThread(),
false /* is_screencast */, true /* align_timestamps */);
jni::Java_JavaVideoSourceTestHelper_startCapture(
env, video_track_source->GetJavaVideoCapturerObserver(env),