Add an option to soft reset HW decoder.

Soft reset can be used when input frame resolution changes
to avoid re creating MediaCodec instance.
Instead MediaCodec is flushed and some variables are reset.

R=pbos@webrtc.org, perkj@webrtc.org

Review URL: https://codereview.webrtc.org/1732533002 .

Cr-Commit-Position: refs/heads/master@{#11878}

webrtc/api/java/jni/androidmediadecoder_jni.cc

@@ -99,6 +99,7 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
   void CheckOnCodecThread();
 
   int32_t InitDecodeOnCodecThread();
+  int32_t ResetDecodeOnCodecThread();
   int32_t ReleaseOnCodecThread();
   int32_t DecodeOnCodecThread(const EncodedImage& inputImage);
   // Deliver any outputs pending in the MediaCodec to our |callback_| and return
@@ -106,6 +107,7 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
   bool DeliverPendingOutputs(JNIEnv* jni, int dequeue_timeout_us);
   int32_t ProcessHWErrorOnCodecThread();
   void EnableFrameLogOnWarning();
+  void ResetVariables();
 
   // Type of video codec.
   VideoCodecType codecType_;
@@ -139,6 +141,7 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
   ScopedGlobalRef<jclass> j_media_codec_video_decoder_class_;
   ScopedGlobalRef<jobject> j_media_codec_video_decoder_;
   jmethodID j_init_decode_method_;
+  jmethodID j_reset_method_;
   jmethodID j_release_method_;
   jmethodID j_dequeue_input_buffer_method_;
   jmethodID j_queue_input_buffer_method_;
@@ -200,6 +203,8 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder(
       jni, *j_media_codec_video_decoder_class_, "initDecode",
       "(Lorg/webrtc/MediaCodecVideoDecoder$VideoCodecType;"
       "IILorg/webrtc/SurfaceTextureHelper;)Z");
+  j_reset_method_ =
+      GetMethodID(jni, *j_media_codec_video_decoder_class_, "reset", "(II)V");
   j_release_method_ =
       GetMethodID(jni, *j_media_codec_video_decoder_class_, "release", "()V");
   j_dequeue_input_buffer_method_ = GetMethodID(
@@ -306,6 +311,20 @@ int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst,
       Bind(&MediaCodecVideoDecoder::InitDecodeOnCodecThread, this));
 }
 
+void MediaCodecVideoDecoder::ResetVariables() {
+  CheckOnCodecThread();
+
+  key_frame_required_ = true;
+  frames_received_ = 0;
+  frames_decoded_ = 0;
+  frames_decoded_logged_ = kMaxDecodedLogFrames;
+  start_time_ms_ = GetCurrentTimeMs();
+  current_frames_ = 0;
+  current_bytes_ = 0;
+  current_decoding_time_ms_ = 0;
+  current_delay_time_ms_ = 0;
+}
+
 int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() {
   CheckOnCodecThread();
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
@@ -322,11 +341,7 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() {
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
 
-  // Always start with a complete key frame.
+  ResetVariables();
-  key_frame_required_ = true;
-  frames_received_ = 0;
-  frames_decoded_ = 0;
-  frames_decoded_logged_ = kMaxDecodedLogFrames;
 
   jobject java_surface_texture_helper_ = nullptr;
   if (use_surface_) {
@@ -352,6 +367,7 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() {
       codec_.width,
       codec_.height,
       java_surface_texture_helper_);
+
   if (CheckException(jni) || !success) {
     ALOGE << "Codec initialization error - fallback to SW codec.";
     sw_fallback_required_ = true;
@@ -372,16 +388,11 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() {
     default:
       max_pending_frames_ = 0;
   }
-  start_time_ms_ = GetCurrentTimeMs();
+  ALOGD << "Maximum amount of pending frames: " << max_pending_frames_;
-  current_frames_ = 0;
-  current_bytes_ = 0;
-  current_decoding_time_ms_ = 0;
-  current_delay_time_ms_ = 0;
 
   jobjectArray input_buffers = (jobjectArray)GetObjectField(
       jni, *j_media_codec_video_decoder_, j_input_buffers_field_);
   size_t num_input_buffers = jni->GetArrayLength(input_buffers);
-  ALOGD << "Maximum amount of pending frames: " << max_pending_frames_;
   input_buffers_.resize(num_input_buffers);
   for (size_t i = 0; i < num_input_buffers; ++i) {
     input_buffers_[i] =
@@ -398,6 +409,37 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() {
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
+int32_t MediaCodecVideoDecoder::ResetDecodeOnCodecThread() {
+  CheckOnCodecThread();
+  JNIEnv* jni = AttachCurrentThreadIfNeeded();
+  ScopedLocalRefFrame local_ref_frame(jni);
+  ALOGD << "ResetDecodeOnCodecThread Type: " << (int)codecType_ << ". "
+      << codec_.width << " x " << codec_.height;
+  ALOGD << "  Frames received: " << frames_received_ <<
+      ". Frames decoded: " << frames_decoded_;
+
+  inited_ = false;
+  rtc::MessageQueueManager::Clear(this);
+  ResetVariables();
+
+  jni->CallVoidMethod(
+      *j_media_codec_video_decoder_,
+      j_reset_method_,
+      codec_.width,
+      codec_.height);
+
+  if (CheckException(jni)) {
+    ALOGE << "Soft reset error - fallback to SW codec.";
+    sw_fallback_required_ = true;
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+  inited_ = true;
+
+  codec_thread_->PostDelayed(kMediaCodecPollMs, this);
+
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
 int32_t MediaCodecVideoDecoder::Release() {
   ALOGD << "DecoderRelease request";
   return codec_thread_->Invoke<int32_t>(
@@ -493,9 +535,22 @@ int32_t MediaCodecVideoDecoder::Decode(
   if ((inputImage._encodedWidth * inputImage._encodedHeight > 0) &&
       (inputImage._encodedWidth != codec_.width ||
       inputImage._encodedHeight != codec_.height)) {
+    ALOGW << "Input resolution changed from " <<
+        codec_.width << " x " << codec_.height << " to " <<
+        inputImage._encodedWidth << " x " << inputImage._encodedHeight;
     codec_.width = inputImage._encodedWidth;
     codec_.height = inputImage._encodedHeight;
-    int32_t ret = InitDecode(&codec_, 1);
+    int32_t ret;
+    if (use_surface_ && codecType_ == kVideoCodecVP8) {
+      // Soft codec reset - only for VP8 and surface decoding.
+      // TODO(glaznev): try to use similar approach for H.264
+      // and buffer decoding.
+      ret = codec_thread_->Invoke<int32_t>(Bind(
+          &MediaCodecVideoDecoder::ResetDecodeOnCodecThread, this));
+    } else {
+      // Hard codec reset.
+      ret = InitDecode(&codec_, 1);
+    }
     if (ret < 0) {
       ALOGE << "InitDecode failure: " << ret << " - fallback to SW codec";
       sw_fallback_required_ = true;
@@ -531,7 +586,9 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread(
 
   // Try to drain the decoder and wait until output is not too
   // much behind the input.
-  if (frames_received_ > frames_decoded_ + max_pending_frames_) {
+  if (codecType_ == kVideoCodecH264 &&
+      frames_received_ > frames_decoded_ + max_pending_frames_) {
+    // Print warning for H.264 only - for VP8/VP9 one frame delay is ok.
     ALOGW << "Decoder is too far behind. Try to drain. Received: " <<
         frames_received_ << ". Decoded: " << frames_decoded_;
     EnableFrameLogOnWarning();

webrtc/api/java/src/org/webrtc/MediaCodecVideoDecoder.java

@@ -239,12 +239,14 @@ public class MediaCodecVideoDecoder {
 
   // Pass null in |surfaceTextureHelper| to configure the codec for ByteBuffer output.
   private boolean initDecode(
-      VideoCodecType type, int width, int height, SurfaceTextureHelper surfaceTextureHelper) {
+      VideoCodecType type, int width, int height,
+      SurfaceTextureHelper surfaceTextureHelper) {
     if (mediaCodecThread != null) {
-      throw new RuntimeException("Forgot to release()?");
+      throw new RuntimeException("initDecode: Forgot to release()?");
     }
-    useSurface = (surfaceTextureHelper != null);
+
     String mime = null;
+    useSurface = (surfaceTextureHelper != null);
     String[] supportedCodecPrefixes = null;
     if (type == VideoCodecType.VIDEO_CODEC_VP8) {
       mime = VP8_MIME_TYPE;
@@ -256,15 +258,17 @@ public class MediaCodecVideoDecoder {
       mime = H264_MIME_TYPE;
       supportedCodecPrefixes = supportedH264HwCodecPrefixes;
     } else {
-      throw new RuntimeException("Non supported codec " + type);
+      throw new RuntimeException("initDecode: Non-supported codec " + type);
     }
     DecoderProperties properties = findDecoder(mime, supportedCodecPrefixes);
     if (properties == null) {
       throw new RuntimeException("Cannot find HW decoder for " + type);
     }
+
     Logging.d(TAG, "Java initDecode: " + type + " : "+ width + " x " + height +
         ". Color: 0x" + Integer.toHexString(properties.colorFormat) +
         ". Use Surface: " + useSurface);
+
     runningInstance = this; // Decoder is now running and can be queried for stack traces.
     mediaCodecThread = Thread.currentThread();
     try {
@@ -283,14 +287,14 @@ public class MediaCodecVideoDecoder {
         format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat);
       }
       Logging.d(TAG, "  Format: " + format);
-      mediaCodec =
+      mediaCodec = MediaCodecVideoEncoder.createByCodecName(properties.codecName);
-          MediaCodecVideoEncoder.createByCodecName(properties.codecName);
       if (mediaCodec == null) {
         Logging.e(TAG, "Can not create media decoder");
         return false;
       }
       mediaCodec.configure(format, surface, null, 0);
       mediaCodec.start();
+
       colorFormat = properties.colorFormat;
       outputBuffers = mediaCodec.getOutputBuffers();
       inputBuffers = mediaCodec.getInputBuffers();
@@ -307,6 +311,24 @@ public class MediaCodecVideoDecoder {
     }
   }
 
+  // Resets the decoder so it can start decoding frames with new resolution.
+  // Flushes MediaCodec and clears decoder output buffers.
+  private void reset(int width, int height) {
+    if (mediaCodecThread == null || mediaCodec == null) {
+      throw new RuntimeException("Incorrect reset call for non-initialized decoder.");
+    }
+    Logging.d(TAG, "Java reset: " + width + " x " + height);
+
+    mediaCodec.flush();
+
+    this.width = width;
+    this.height = height;
+    decodeStartTimeMs.clear();
+    dequeuedSurfaceOutputBuffers.clear();
+    hasDecodedFirstFrame = false;
+    droppedFrames = 0;
+  }
+
   private void release() {
     Logging.d(TAG, "Java releaseDecoder. Total number of dropped frames: " + droppedFrames);
     checkOnMediaCodecThread();

webrtc/examples/androidtests/src/org/appspot/apprtc/test/PeerConnectionClientTest.java

@@ -40,6 +40,7 @@ public class PeerConnectionClientTest extends InstrumentationTestCase
   private static final int WAIT_TIMEOUT = 7000;
   private static final int CAMERA_SWITCH_ATTEMPTS = 3;
   private static final int VIDEO_RESTART_ATTEMPTS = 3;
+  private static final int CAPTURE_FORMAT_CHANGE_ATTEMPTS = 3;
   private static final int VIDEO_RESTART_TIMEOUT = 500;
   private static final int EXPECTED_VIDEO_FRAMES = 10;
   private static final String VIDEO_CODEC_VP8 = "VP8";
@@ -49,6 +50,12 @@ public class PeerConnectionClientTest extends InstrumentationTestCase
   private static final String LOCAL_RENDERER_NAME = "Local renderer";
   private static final String REMOTE_RENDERER_NAME = "Remote renderer";
 
+  private static final int MAX_VIDEO_FPS = 30;
+  private static final int WIDTH_VGA = 640;
+  private static final int HEIGHT_VGA = 480;
+  private static final int WIDTH_QVGA = 320;
+  private static final int HEIGHT_QVGA = 240;
+
   // The peer connection client is assumed to be thread safe in itself; the
   // reference is written by the test thread and read by worker threads.
   private volatile PeerConnectionClient pcClient;
@@ -551,4 +558,55 @@ public class PeerConnectionClientTest extends InstrumentationTestCase
     assertTrue(waitForPeerConnectionClosed(WAIT_TIMEOUT));
     Log.d(TAG, "testVideoSourceRestart done.");
   }
+
+  // Checks if capture format can be changed on fly and decoder can be reset properly.
+  public void testCaptureFormatChange() throws InterruptedException {
+    Log.d(TAG, "testCaptureFormatChange");
+    loopback = true;
+
+    MockRenderer localRenderer = new MockRenderer(EXPECTED_VIDEO_FRAMES, LOCAL_RENDERER_NAME);
+    MockRenderer remoteRenderer = new MockRenderer(EXPECTED_VIDEO_FRAMES, REMOTE_RENDERER_NAME);
+
+    pcClient = createPeerConnectionClient(
+        localRenderer, remoteRenderer, createParametersForVideoCall(VIDEO_CODEC_VP8, false), null);
+
+    // Wait for local SDP, rename it to answer and set as remote SDP.
+    assertTrue("Local SDP was not set.", waitForLocalSDP(WAIT_TIMEOUT));
+    SessionDescription remoteSdp = new SessionDescription(
+        SessionDescription.Type.fromCanonicalForm("answer"),
+        localSdp.description);
+    pcClient.setRemoteDescription(remoteSdp);
+
+    // Wait for ICE connection.
+    assertTrue("ICE connection failure.", waitForIceConnected(ICE_CONNECTION_WAIT_TIMEOUT));
+
+    // Check that local and remote video frames were rendered.
+    assertTrue("Local video frames were not rendered before camera resolution change.",
+        localRenderer.waitForFramesRendered(WAIT_TIMEOUT));
+    assertTrue("Remote video frames were not rendered before camera resolution change.",
+        remoteRenderer.waitForFramesRendered(WAIT_TIMEOUT));
+
+    // Change capture output format a few times.
+    for (int i = 0; i < 2 * CAPTURE_FORMAT_CHANGE_ATTEMPTS; i++) {
+      if (i % 2 == 0) {
+        pcClient.changeCaptureFormat(WIDTH_VGA, HEIGHT_VGA, MAX_VIDEO_FPS);
+      } else {
+        pcClient.changeCaptureFormat(WIDTH_QVGA, HEIGHT_QVGA, MAX_VIDEO_FPS);
+      }
+
+      // Reset video renders and check that local and remote video frames
+      // were rendered after capture format change.
+      localRenderer.reset(EXPECTED_VIDEO_FRAMES);
+      remoteRenderer.reset(EXPECTED_VIDEO_FRAMES);
+      assertTrue("Local video frames were not rendered after capture format change.",
+          localRenderer.waitForFramesRendered(WAIT_TIMEOUT));
+      assertTrue("Remote video frames were not rendered after capture format change.",
+          remoteRenderer.waitForFramesRendered(WAIT_TIMEOUT));
+    }
+
+    pcClient.close();
+    assertTrue(waitForPeerConnectionClosed(WAIT_TIMEOUT));
+    Log.d(TAG, "testCaptureFormatChange done.");
+  }
+
 }