Android AppRTCDemo: Add slider for changing camera capture quality during call

This CL adds a slider that can change capture resolution and fps during a call. The camera will no be reconfigured, but the frames will be downscaled/dropped in software by cricket::VideoAdapter in the cricket::VideoCapturer. This is controlled with VideoCapturerAndroid.onOutputFormatRequest(). The slider is turned off by default and can be enabled with a checkbox under 'WebRTC Video Settings'.

R=glaznev@webrtc.org

Review URL: https://codereview.webrtc.org/1361083002 .

Cr-Commit-Position: refs/heads/master@{#10067}

webrtc/examples/androidapp/res/layout/fragment_call.xml

@@ -18,7 +18,8 @@
     <LinearLayout
            android:id="@+id/buttons_call_container"
            android:orientation="horizontal"
-           android:layout_alignParentBottom="true"
+           android:layout_above="@+id/capture_format_text_call"
+           android:layout_alignWithParentIfMissing="true"
            android:layout_marginBottom="32dp"
            android:layout_centerHorizontal="true"
            android:layout_width="wrap_content"
@@ -48,4 +49,22 @@
            android:layout_height="48dp"/>
     </LinearLayout>
 
+    <TextView
+        android:id="@+id/capture_format_text_call"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:layout_centerHorizontal="true"
+        android:layout_above="@+id/capture_format_slider_call"
+        android:textSize="16sp"
+        android:text="Slide to change capture format"/>
+
+    <SeekBar
+        android:id="@+id/capture_format_slider_call"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:layout_centerHorizontal="true"
+        android:layout_alignParentBottom="true"
+        android:progress="50"
+        android:layout_margin="8dp"/>
+
 </RelativeLayout>

webrtc/examples/androidapp/res/values/strings.xml

@@ -46,6 +46,11 @@
     <string name="pref_fps_dlg">Enter local camera fps.</string>
     <string name="pref_fps_default">Default</string>
 
+    <string name="pref_capturequalityslider_key">capturequalityslider_preference</string>
+    <string name="pref_capturequalityslider_title">Capture quality slider.</string>
+    <string name="pref_capturequalityslider_dlg">Enable slider for changing capture quality.</string>
+    <string name="pref_capturequalityslider_default">false</string>
+
     <string name="pref_startvideobitrate_key">startvideobitrate_preference</string>
     <string name="pref_startvideobitrate_title">Start video bitrate setting.</string>
     <string name="pref_startvideobitrate_dlg">Start video bitrate setting.</string>

webrtc/examples/androidapp/res/xml/preferences.xml

@@ -26,6 +26,12 @@
             android:entries="@array/cameraFps"
             android:entryValues="@array/cameraFps" />
 
+        <CheckBoxPreference
+            android:key="@string/pref_capturequalityslider_key"
+            android:title="@string/pref_capturequalityslider_title"
+            android:dialogTitle="@string/pref_capturequalityslider_dlg"
+            android:defaultValue="@string/pref_capturequalityslider_default" />
+
         <ListPreference
             android:key="@string/pref_startvideobitrate_key"
             android:title="@string/pref_startvideobitrate_title"

webrtc/examples/androidapp/src/org/appspot/apprtc/CallActivity.java

@@ -59,6 +59,8 @@ public class CallActivity extends Activity
       "org.appspot.apprtc.VIDEO_HEIGHT";
   public static final String EXTRA_VIDEO_FPS =
       "org.appspot.apprtc.VIDEO_FPS";
+  public static final String EXTRA_VIDEO_CAPTUREQUALITYSLIDER_ENABLED =
+      "org.appsopt.apprtc.VIDEO_CAPTUREQUALITYSLIDER";
   public static final String EXTRA_VIDEO_BITRATE =
       "org.appspot.apprtc.VIDEO_BITRATE";
   public static final String EXTRA_VIDEOCODEC =
@@ -304,6 +306,13 @@ public class CallActivity extends Activity
     updateVideoView();
   }
 
+  @Override
+  public void onCaptureFormatChange(int width, int height, int framerate) {
+    if (peerConnectionClient != null) {
+      peerConnectionClient.changeCaptureFormat(width, height, framerate);
+    }
+  }
+
   // Helper functions.
   private void toggleCallControlFragmentVisibility() {
     if (!iceConnected || !callFragment.isAdded()) {

webrtc/examples/androidapp/src/org/appspot/apprtc/CallFragment.java

@@ -17,6 +17,7 @@ import android.view.LayoutInflater;
 import android.view.View;
 import android.view.ViewGroup;
 import android.widget.ImageButton;
+import android.widget.SeekBar;
 import android.widget.TextView;
 
 import org.webrtc.RendererCommon.ScalingType;
@@ -30,6 +31,8 @@ public class CallFragment extends Fragment {
   private ImageButton disconnectButton;
   private ImageButton cameraSwitchButton;
   private ImageButton videoScalingButton;
+  private TextView captureFormatText;
+  private SeekBar captureFormatSlider;
   private OnCallEvents callEvents;
   private ScalingType scalingType;
   private boolean videoCallEnabled = true;
@@ -41,6 +44,7 @@ public class CallFragment extends Fragment {
     public void onCallHangUp();
     public void onCameraSwitch();
     public void onVideoScalingSwitch(ScalingType scalingType);
+    public void onCaptureFormatChange(int width, int height, int framerate);
   }
 
   @Override
@@ -58,6 +62,10 @@ public class CallFragment extends Fragment {
         (ImageButton) controlView.findViewById(R.id.button_call_switch_camera);
     videoScalingButton =
         (ImageButton) controlView.findViewById(R.id.button_call_scaling_mode);
+    captureFormatText =
+        (TextView) controlView.findViewById(R.id.capture_format_text_call);
+    captureFormatSlider =
+        (SeekBar) controlView.findViewById(R.id.capture_format_slider_call);
 
     // Add buttons click events.
     disconnectButton.setOnClickListener(new View.OnClickListener() {
@@ -98,15 +106,25 @@ public class CallFragment extends Fragment {
   public void onStart() {
     super.onStart();
 
+    boolean captureSliderEnabled = false;
     Bundle args = getArguments();
     if (args != null) {
       String contactName = args.getString(CallActivity.EXTRA_ROOMID);
       contactView.setText(contactName);
       videoCallEnabled = args.getBoolean(CallActivity.EXTRA_VIDEO_CALL, true);
+      captureSliderEnabled = videoCallEnabled
+          && args.getBoolean(CallActivity.EXTRA_VIDEO_CAPTUREQUALITYSLIDER_ENABLED, false);
     }
     if (!videoCallEnabled) {
       cameraSwitchButton.setVisibility(View.INVISIBLE);
     }
+    if (captureSliderEnabled) {
+      captureFormatSlider.setOnSeekBarChangeListener(
+          new CaptureQualityController(captureFormatText, callEvents));
+    } else {
+      captureFormatText.setVisibility(View.GONE);
+      captureFormatSlider.setVisibility(View.GONE);
+    }
   }
 
   @Override

webrtc/examples/androidapp/src/org/appspot/apprtc/CaptureQualityController.java

@@ -0,0 +1,112 @@
+/*
+ *  Copyright 2015 The WebRTC Project Authors. All rights reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.appspot.apprtc;
+
+import android.widget.SeekBar;
+import android.widget.TextView;
+
+import org.webrtc.CameraEnumerationAndroid.CaptureFormat;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+/**
+ * Control capture format based on a seekbar listener.
+ */
+public class CaptureQualityController implements SeekBar.OnSeekBarChangeListener {
+  private final List<CaptureFormat> formats = Arrays.asList(
+      new CaptureFormat(1280, 720, 0, 30000),
+      new CaptureFormat(640, 480, 0, 30000),
+      new CaptureFormat(320, 240, 0, 30000),
+      new CaptureFormat(256, 144, 0, 30000));
+  // Prioritize framerate below this threshold and resolution above the threshold.
+  private static final int FRAMERATE_THRESHOLD = 15;
+  private TextView captureFormatText;
+  private CallFragment.OnCallEvents callEvents;
+  private int width = 0;
+  private int height = 0;
+  private int framerate = 0;
+  private double targetBandwidth = 0;
+
+  public CaptureQualityController(
+      TextView captureFormatText, CallFragment.OnCallEvents callEvents) {
+    this.captureFormatText = captureFormatText;
+    this.callEvents = callEvents;
+  }
+
+  private final Comparator<CaptureFormat> compareFormats = new Comparator<CaptureFormat>() {
+    @Override
+    public int compare(CaptureFormat first, CaptureFormat second) {
+      int firstFps = calculateFramerate(targetBandwidth, first);
+      int secondFps = calculateFramerate(targetBandwidth, second);
+
+     if (firstFps >= FRAMERATE_THRESHOLD && secondFps >= FRAMERATE_THRESHOLD
+         || firstFps == secondFps) {
+        // Compare resolution.
+        return first.width * first.height - second.width * second.height;
+     } else {
+        // Compare fps.
+        return firstFps - secondFps;
+     }
+    }
+  };
+
+  @Override
+  public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) {
+    if (progress == 0) {
+      width = 0;
+      height = 0;
+      framerate = 0;
+      captureFormatText.setText("Muted");
+      return;
+    }
+
+    // Extract max bandwidth (in millipixels / second).
+    long maxCaptureBandwidth = java.lang.Long.MIN_VALUE;
+    for (CaptureFormat format : formats) {
+      maxCaptureBandwidth = Math.max(maxCaptureBandwidth,
+          (long) format.width * format.height * format.maxFramerate);
+    }
+
+    // Fraction between 0 and 1.
+    double bandwidthFraction = (double) progress / 100.0;
+    // Make a log-scale transformation, still between 0 and 1.
+    final double kExpConstant = 3.0;
+    bandwidthFraction =
+        (Math.exp(kExpConstant * bandwidthFraction) - 1) / (Math.exp(kExpConstant) - 1);
+    targetBandwidth = bandwidthFraction * maxCaptureBandwidth;
+
+    // Choose the best format given a target bandwidth.
+    final CaptureFormat bestFormat = Collections.max(formats, compareFormats);
+    width = bestFormat.width;
+    height = bestFormat.height;
+    framerate = calculateFramerate(targetBandwidth, bestFormat);
+    captureFormatText.setText(width + "x" + height + " @ " + framerate + "fps");
+  }
+
+  @Override
+  public void onStartTrackingTouch(SeekBar seekBar) {
+  }
+
+  @Override
+  public void onStopTrackingTouch(SeekBar seekBar) {
+    callEvents.onCaptureFormatChange(width, height, framerate);
+  }
+
+  // Return the highest frame rate possible based on bandwidth and format.
+  private int calculateFramerate(double bandwidth, CaptureFormat format) {
+    return (int) Math.round(Math.min(format.maxFramerate,
+        (int) Math.round(bandwidth / (format.width * format.height))) / 1000.0);
+  }
+}
+

webrtc/examples/androidapp/src/org/appspot/apprtc/ConnectActivity.java

@@ -57,6 +57,7 @@ public class ConnectActivity extends Activity {
   private String keyprefVideoCallEnabled;
   private String keyprefResolution;
   private String keyprefFps;
+  private String keyprefCaptureQualitySlider;
   private String keyprefVideoBitrateType;
   private String keyprefVideoBitrateValue;
   private String keyprefVideoCodec;
@@ -83,6 +84,7 @@ public class ConnectActivity extends Activity {
     keyprefVideoCallEnabled = getString(R.string.pref_videocall_key);
     keyprefResolution = getString(R.string.pref_resolution_key);
     keyprefFps = getString(R.string.pref_fps_key);
+    keyprefCaptureQualitySlider = getString(R.string.pref_capturequalityslider_key);
     keyprefVideoBitrateType = getString(R.string.pref_startvideobitrate_key);
     keyprefVideoBitrateValue = getString(R.string.pref_startvideobitratevalue_key);
     keyprefVideoCodec = getString(R.string.pref_videocodec_key);
@@ -286,6 +288,10 @@ public class ConnectActivity extends Activity {
       }
     }
 
+    // Check capture quality slider flag.
+    boolean captureQualitySlider = sharedPref.getBoolean(keyprefCaptureQualitySlider,
+        Boolean.valueOf(getString(R.string.pref_capturequalityslider_default)));
+
     // Get video and audio start bitrate.
     int videoStartBitrate = 0;
     String bitrateTypeDefault = getString(
@@ -329,6 +335,8 @@ public class ConnectActivity extends Activity {
       intent.putExtra(CallActivity.EXTRA_VIDEO_WIDTH, videoWidth);
       intent.putExtra(CallActivity.EXTRA_VIDEO_HEIGHT, videoHeight);
       intent.putExtra(CallActivity.EXTRA_VIDEO_FPS, cameraFps);
+      intent.putExtra(CallActivity.EXTRA_VIDEO_CAPTUREQUALITYSLIDER_ENABLED,
+          captureQualitySlider);
       intent.putExtra(CallActivity.EXTRA_VIDEO_BITRATE, videoStartBitrate);
       intent.putExtra(CallActivity.EXTRA_VIDEOCODEC, videoCodec);
       intent.putExtra(CallActivity.EXTRA_HWCODEC_ENABLED, hwCodec);

webrtc/examples/androidapp/src/org/appspot/apprtc/PeerConnectionClient.java

@@ -844,6 +844,24 @@ public class PeerConnectionClient {
     });
   }
 
+  public void changeCaptureFormat(final int width, final int height, final int framerate) {
+    executor.execute(new Runnable() {
+      @Override
+      public void run() {
+        changeCaptureFormatInternal(width, height, framerate);
+      }
+    });
+  }
+
+  private void changeCaptureFormatInternal(int width, int height, int framerate) {
+    if (!videoCallEnabled || isError || videoCapturer == null) {
+      Log.e(TAG, "Failed to change capture format. Video: " + videoCallEnabled + ". Error : "
+          + isError);
+      return;
+    }
+    videoCapturer.onOutputFormatRequest(width, height, framerate);
+  }
+
   // Implementation detail: observe ICE & stream changes and react accordingly.
   private class PCObserver implements PeerConnection.Observer {
     @Override

webrtc/examples/androidapp/src/org/appspot/apprtc/SettingsActivity.java

@@ -25,6 +25,7 @@ public class SettingsActivity extends Activity
   private String keyprefVideoCall;
   private String keyprefResolution;
   private String keyprefFps;
+  private String keyprefCaptureQualitySlider;
   private String keyprefStartVideoBitrateType;
   private String keyprefStartVideoBitrateValue;
   private String keyPrefVideoCodec;
@@ -45,6 +46,7 @@ public class SettingsActivity extends Activity
     keyprefVideoCall = getString(R.string.pref_videocall_key);
     keyprefResolution = getString(R.string.pref_resolution_key);
     keyprefFps = getString(R.string.pref_fps_key);
+    keyprefCaptureQualitySlider = getString(R.string.pref_capturequalityslider_key);
     keyprefStartVideoBitrateType = getString(R.string.pref_startvideobitrate_key);
     keyprefStartVideoBitrateValue = getString(R.string.pref_startvideobitratevalue_key);
     keyPrefVideoCodec = getString(R.string.pref_videocodec_key);
@@ -76,6 +78,7 @@ public class SettingsActivity extends Activity
     updateSummaryB(sharedPreferences, keyprefVideoCall);
     updateSummary(sharedPreferences, keyprefResolution);
     updateSummary(sharedPreferences, keyprefFps);
+    updateSummaryB(sharedPreferences, keyprefCaptureQualitySlider);
     updateSummary(sharedPreferences, keyprefStartVideoBitrateType);
     updateSummaryBitrate(sharedPreferences, keyprefStartVideoBitrateValue);
     setVideoBitrateEnable(sharedPreferences);
@@ -116,6 +119,7 @@ public class SettingsActivity extends Activity
         || key.equals(keyprefStartAudioBitrateValue)) {
       updateSummaryBitrate(sharedPreferences, key);
     } else if (key.equals(keyprefVideoCall)
+        || key.equals(keyprefCaptureQualitySlider)
         || key.equals(keyprefHwCodec)
         || key.equals(keyprefNoAudioProcessing)
         || key.equals(keyprefCpuUsageDetection)