From 67e0cf15d323642516f40b619f6474db333524a7 Mon Sep 17 00:00:00 2001 From: Magnus Jedvert Date: Fri, 25 Sep 2015 08:23:39 +0200 Subject: [PATCH] Android AppRTCDemo: Add slider for changing camera capture quality during call This CL adds a slider that can change capture resolution and fps during a call. The camera will no be reconfigured, but the frames will be downscaled/dropped in software by cricket::VideoAdapter in the cricket::VideoCapturer. This is controlled with VideoCapturerAndroid.onOutputFormatRequest(). The slider is turned off by default and can be enabled with a checkbox under 'WebRTC Video Settings'. R=glaznev@webrtc.org Review URL: https://codereview.webrtc.org/1361083002 . Cr-Commit-Position: refs/heads/master@{#10067} --- .../androidapp/res/layout/fragment_call.xml | 21 +++- .../androidapp/res/values/strings.xml | 5 + .../androidapp/res/xml/preferences.xml | 6 + .../src/org/appspot/apprtc/CallActivity.java | 9 ++ .../src/org/appspot/apprtc/CallFragment.java | 18 +++ .../apprtc/CaptureQualityController.java | 112 ++++++++++++++++++ .../org/appspot/apprtc/ConnectActivity.java | 8 ++ .../appspot/apprtc/PeerConnectionClient.java | 18 +++ .../org/appspot/apprtc/SettingsActivity.java | 4 + 9 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 webrtc/examples/androidapp/src/org/appspot/apprtc/CaptureQualityController.java diff --git a/webrtc/examples/androidapp/res/layout/fragment_call.xml b/webrtc/examples/androidapp/res/layout/fragment_call.xml index 70a9a28197..5d71d73f50 100644 --- a/webrtc/examples/androidapp/res/layout/fragment_call.xml +++ b/webrtc/examples/androidapp/res/layout/fragment_call.xml @@ -18,7 +18,8 @@ + + + + diff --git a/webrtc/examples/androidapp/res/values/strings.xml b/webrtc/examples/androidapp/res/values/strings.xml index 4f2f3777e2..b3c55b438e 100644 --- a/webrtc/examples/androidapp/res/values/strings.xml +++ b/webrtc/examples/androidapp/res/values/strings.xml @@ -46,6 +46,11 @@ Enter local camera fps. Default + capturequalityslider_preference + Capture quality slider. + Enable slider for changing capture quality. + false + startvideobitrate_preference Start video bitrate setting. Start video bitrate setting. diff --git a/webrtc/examples/androidapp/res/xml/preferences.xml b/webrtc/examples/androidapp/res/xml/preferences.xml index 73d8d5e254..c580e0cb77 100644 --- a/webrtc/examples/androidapp/res/xml/preferences.xml +++ b/webrtc/examples/androidapp/res/xml/preferences.xml @@ -26,6 +26,12 @@ android:entries="@array/cameraFps" android:entryValues="@array/cameraFps" /> + + formats = Arrays.asList( + new CaptureFormat(1280, 720, 0, 30000), + new CaptureFormat(640, 480, 0, 30000), + new CaptureFormat(320, 240, 0, 30000), + new CaptureFormat(256, 144, 0, 30000)); + // Prioritize framerate below this threshold and resolution above the threshold. + private static final int FRAMERATE_THRESHOLD = 15; + private TextView captureFormatText; + private CallFragment.OnCallEvents callEvents; + private int width = 0; + private int height = 0; + private int framerate = 0; + private double targetBandwidth = 0; + + public CaptureQualityController( + TextView captureFormatText, CallFragment.OnCallEvents callEvents) { + this.captureFormatText = captureFormatText; + this.callEvents = callEvents; + } + + private final Comparator compareFormats = new Comparator() { + @Override + public int compare(CaptureFormat first, CaptureFormat second) { + int firstFps = calculateFramerate(targetBandwidth, first); + int secondFps = calculateFramerate(targetBandwidth, second); + + if (firstFps >= FRAMERATE_THRESHOLD && secondFps >= FRAMERATE_THRESHOLD + || firstFps == secondFps) { + // Compare resolution. + return first.width * first.height - second.width * second.height; + } else { + // Compare fps. + return firstFps - secondFps; + } + } + }; + + @Override + public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) { + if (progress == 0) { + width = 0; + height = 0; + framerate = 0; + captureFormatText.setText("Muted"); + return; + } + + // Extract max bandwidth (in millipixels / second). + long maxCaptureBandwidth = java.lang.Long.MIN_VALUE; + for (CaptureFormat format : formats) { + maxCaptureBandwidth = Math.max(maxCaptureBandwidth, + (long) format.width * format.height * format.maxFramerate); + } + + // Fraction between 0 and 1. + double bandwidthFraction = (double) progress / 100.0; + // Make a log-scale transformation, still between 0 and 1. + final double kExpConstant = 3.0; + bandwidthFraction = + (Math.exp(kExpConstant * bandwidthFraction) - 1) / (Math.exp(kExpConstant) - 1); + targetBandwidth = bandwidthFraction * maxCaptureBandwidth; + + // Choose the best format given a target bandwidth. + final CaptureFormat bestFormat = Collections.max(formats, compareFormats); + width = bestFormat.width; + height = bestFormat.height; + framerate = calculateFramerate(targetBandwidth, bestFormat); + captureFormatText.setText(width + "x" + height + " @ " + framerate + "fps"); + } + + @Override + public void onStartTrackingTouch(SeekBar seekBar) { + } + + @Override + public void onStopTrackingTouch(SeekBar seekBar) { + callEvents.onCaptureFormatChange(width, height, framerate); + } + + // Return the highest frame rate possible based on bandwidth and format. + private int calculateFramerate(double bandwidth, CaptureFormat format) { + return (int) Math.round(Math.min(format.maxFramerate, + (int) Math.round(bandwidth / (format.width * format.height))) / 1000.0); + } +} + diff --git a/webrtc/examples/androidapp/src/org/appspot/apprtc/ConnectActivity.java b/webrtc/examples/androidapp/src/org/appspot/apprtc/ConnectActivity.java index 7c007901c1..0bdaebb5b0 100644 --- a/webrtc/examples/androidapp/src/org/appspot/apprtc/ConnectActivity.java +++ b/webrtc/examples/androidapp/src/org/appspot/apprtc/ConnectActivity.java @@ -57,6 +57,7 @@ public class ConnectActivity extends Activity { private String keyprefVideoCallEnabled; private String keyprefResolution; private String keyprefFps; + private String keyprefCaptureQualitySlider; private String keyprefVideoBitrateType; private String keyprefVideoBitrateValue; private String keyprefVideoCodec; @@ -83,6 +84,7 @@ public class ConnectActivity extends Activity { keyprefVideoCallEnabled = getString(R.string.pref_videocall_key); keyprefResolution = getString(R.string.pref_resolution_key); keyprefFps = getString(R.string.pref_fps_key); + keyprefCaptureQualitySlider = getString(R.string.pref_capturequalityslider_key); keyprefVideoBitrateType = getString(R.string.pref_startvideobitrate_key); keyprefVideoBitrateValue = getString(R.string.pref_startvideobitratevalue_key); keyprefVideoCodec = getString(R.string.pref_videocodec_key); @@ -286,6 +288,10 @@ public class ConnectActivity extends Activity { } } + // Check capture quality slider flag. + boolean captureQualitySlider = sharedPref.getBoolean(keyprefCaptureQualitySlider, + Boolean.valueOf(getString(R.string.pref_capturequalityslider_default))); + // Get video and audio start bitrate. int videoStartBitrate = 0; String bitrateTypeDefault = getString( @@ -329,6 +335,8 @@ public class ConnectActivity extends Activity { intent.putExtra(CallActivity.EXTRA_VIDEO_WIDTH, videoWidth); intent.putExtra(CallActivity.EXTRA_VIDEO_HEIGHT, videoHeight); intent.putExtra(CallActivity.EXTRA_VIDEO_FPS, cameraFps); + intent.putExtra(CallActivity.EXTRA_VIDEO_CAPTUREQUALITYSLIDER_ENABLED, + captureQualitySlider); intent.putExtra(CallActivity.EXTRA_VIDEO_BITRATE, videoStartBitrate); intent.putExtra(CallActivity.EXTRA_VIDEOCODEC, videoCodec); intent.putExtra(CallActivity.EXTRA_HWCODEC_ENABLED, hwCodec); diff --git a/webrtc/examples/androidapp/src/org/appspot/apprtc/PeerConnectionClient.java b/webrtc/examples/androidapp/src/org/appspot/apprtc/PeerConnectionClient.java index 4a406d73b7..a31558ac4a 100644 --- a/webrtc/examples/androidapp/src/org/appspot/apprtc/PeerConnectionClient.java +++ b/webrtc/examples/androidapp/src/org/appspot/apprtc/PeerConnectionClient.java @@ -844,6 +844,24 @@ public class PeerConnectionClient { }); } + public void changeCaptureFormat(final int width, final int height, final int framerate) { + executor.execute(new Runnable() { + @Override + public void run() { + changeCaptureFormatInternal(width, height, framerate); + } + }); + } + + private void changeCaptureFormatInternal(int width, int height, int framerate) { + if (!videoCallEnabled || isError || videoCapturer == null) { + Log.e(TAG, "Failed to change capture format. Video: " + videoCallEnabled + ". Error : " + + isError); + return; + } + videoCapturer.onOutputFormatRequest(width, height, framerate); + } + // Implementation detail: observe ICE & stream changes and react accordingly. private class PCObserver implements PeerConnection.Observer { @Override diff --git a/webrtc/examples/androidapp/src/org/appspot/apprtc/SettingsActivity.java b/webrtc/examples/androidapp/src/org/appspot/apprtc/SettingsActivity.java index ce7d989bd6..9ad6e4d8e4 100644 --- a/webrtc/examples/androidapp/src/org/appspot/apprtc/SettingsActivity.java +++ b/webrtc/examples/androidapp/src/org/appspot/apprtc/SettingsActivity.java @@ -25,6 +25,7 @@ public class SettingsActivity extends Activity private String keyprefVideoCall; private String keyprefResolution; private String keyprefFps; + private String keyprefCaptureQualitySlider; private String keyprefStartVideoBitrateType; private String keyprefStartVideoBitrateValue; private String keyPrefVideoCodec; @@ -45,6 +46,7 @@ public class SettingsActivity extends Activity keyprefVideoCall = getString(R.string.pref_videocall_key); keyprefResolution = getString(R.string.pref_resolution_key); keyprefFps = getString(R.string.pref_fps_key); + keyprefCaptureQualitySlider = getString(R.string.pref_capturequalityslider_key); keyprefStartVideoBitrateType = getString(R.string.pref_startvideobitrate_key); keyprefStartVideoBitrateValue = getString(R.string.pref_startvideobitratevalue_key); keyPrefVideoCodec = getString(R.string.pref_videocodec_key); @@ -76,6 +78,7 @@ public class SettingsActivity extends Activity updateSummaryB(sharedPreferences, keyprefVideoCall); updateSummary(sharedPreferences, keyprefResolution); updateSummary(sharedPreferences, keyprefFps); + updateSummaryB(sharedPreferences, keyprefCaptureQualitySlider); updateSummary(sharedPreferences, keyprefStartVideoBitrateType); updateSummaryBitrate(sharedPreferences, keyprefStartVideoBitrateValue); setVideoBitrateEnable(sharedPreferences); @@ -116,6 +119,7 @@ public class SettingsActivity extends Activity || key.equals(keyprefStartAudioBitrateValue)) { updateSummaryBitrate(sharedPreferences, key); } else if (key.equals(keyprefVideoCall) + || key.equals(keyprefCaptureQualitySlider) || key.equals(keyprefHwCodec) || key.equals(keyprefNoAudioProcessing) || key.equals(keyprefCpuUsageDetection)