Add support for saving local audio input to file in AppRTCMobile

Uses new WebRtcAudioRecordSamplesReadyCallback which was added recently in
https://webrtc-review.googlesource.com/c/src/+/49981.

This CL:
- Serves as a test of new WebRtcAudioRecordSamplesReadyCallback.
- Useful for debugging purposes since it records the most native raw audio.

Bug: None
Change-Id: I57375cbf237c171e045b0bdb05f7ae1401930fbc
Reviewed-on: https://webrtc-review.googlesource.com/53120
Commit-Queue: Henrik Andreassson <henrika@webrtc.org>
Reviewed-by: Sami Kalliomäki <sakal@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#22128}

examples/BUILD.gn

@@ -84,6 +84,7 @@ if (is_android) {
       "androidapp/src/org/appspot/apprtc/PeerConnectionClient.java",
       "androidapp/src/org/appspot/apprtc/RoomParametersFetcher.java",
       "androidapp/src/org/appspot/apprtc/RtcEventLog.java",
+      "androidapp/src/org/appspot/apprtc/RecordedAudioToFileController.java",
       "androidapp/src/org/appspot/apprtc/SettingsActivity.java",
       "androidapp/src/org/appspot/apprtc/SettingsFragment.java",
       "androidapp/src/org/appspot/apprtc/TCPChannelClient.java",

examples/androidapp/res/values/strings.xml

@@ -125,6 +125,11 @@
     <string name="pref_aecdump_dlg">Enable diagnostic audio recordings.</string>
     <string name="pref_aecdump_default">false</string>
 
+    <string name="pref_enable_save_input_audio_to_file_key">enable_key</string>
+    <string name="pref_enable_save_input_audio_to_file_title">Save input audio to file.</string>
+    <string name="pref_enable_save_input_audio_to_file_dlg">Save input audio to file.</string>
+    <string name="pref_enable_save_input_audio_to_file_default">false</string>
+
     <string name="pref_opensles_key">opensles_preference</string>
     <string name="pref_opensles_title">Use OpenSL ES for audio playback.</string>
     <string name="pref_opensles_dlg">Use OpenSL ES for audio playback.</string>

examples/androidapp/res/xml/preferences.xml

@@ -123,6 +123,12 @@
             android:dialogTitle="@string/pref_aecdump_dlg"
             android:defaultValue="@string/pref_aecdump_default" />
 
+        <CheckBoxPreference
+            android:key="@string/pref_enable_save_input_audio_to_file_key"
+            android:title="@string/pref_enable_save_input_audio_to_file_title"
+            android:dialogTitle="@string/pref_enable_save_input_audio_to_file_dlg"
+            android:defaultValue="@string/pref_enable_save_input_audio_to_file_default" />
+
         <CheckBoxPreference
             android:key="@string/pref_opensles_key"
             android:title="@string/pref_opensles_title"

examples/androidapp/src/org/appspot/apprtc/CallActivity.java

@@ -90,6 +90,8 @@ public class CallActivity extends Activity implements AppRTCClient.SignalingEven
   public static final String EXTRA_NOAUDIOPROCESSING_ENABLED =
       "org.appspot.apprtc.NOAUDIOPROCESSING";
   public static final String EXTRA_AECDUMP_ENABLED = "org.appspot.apprtc.AECDUMP";
+  public static final String EXTRA_SAVE_INPUT_AUDIO_TO_FILE_ENABLED =
+      "org.appspot.apprtc.SAVE_INPUT_AUDIO_TO_FILE";
   public static final String EXTRA_OPENSLES_ENABLED = "org.appspot.apprtc.OPENSLES";
   public static final String EXTRA_DISABLE_BUILT_IN_AEC = "org.appspot.apprtc.DISABLE_BUILT_IN_AEC";
   public static final String EXTRA_DISABLE_BUILT_IN_AGC = "org.appspot.apprtc.DISABLE_BUILT_IN_AGC";
@@ -331,6 +333,7 @@ public class CallActivity extends Activity implements AppRTCClient.SignalingEven
             intent.getIntExtra(EXTRA_AUDIO_BITRATE, 0), intent.getStringExtra(EXTRA_AUDIOCODEC),
             intent.getBooleanExtra(EXTRA_NOAUDIOPROCESSING_ENABLED, false),
             intent.getBooleanExtra(EXTRA_AECDUMP_ENABLED, false),
+            intent.getBooleanExtra(EXTRA_SAVE_INPUT_AUDIO_TO_FILE_ENABLED, false),
             intent.getBooleanExtra(EXTRA_OPENSLES_ENABLED, false),
             intent.getBooleanExtra(EXTRA_DISABLE_BUILT_IN_AEC, false),
             intent.getBooleanExtra(EXTRA_DISABLE_BUILT_IN_AGC, false),

examples/androidapp/src/org/appspot/apprtc/ConnectActivity.java

@@ -315,10 +315,14 @@ public class ConnectActivity extends Activity {
         CallActivity.EXTRA_NOAUDIOPROCESSING_ENABLED, R.string.pref_noaudioprocessing_default,
         useValuesFromIntent);
 
-    // Check Disable Audio Processing flag.
     boolean aecDump = sharedPrefGetBoolean(R.string.pref_aecdump_key,
         CallActivity.EXTRA_AECDUMP_ENABLED, R.string.pref_aecdump_default, useValuesFromIntent);
 
+    boolean saveInputAudioToFile =
+        sharedPrefGetBoolean(R.string.pref_enable_save_input_audio_to_file_key,
+            CallActivity.EXTRA_SAVE_INPUT_AUDIO_TO_FILE_ENABLED,
+            R.string.pref_enable_save_input_audio_to_file_default, useValuesFromIntent);
+
     // Check OpenSL ES enabled flag.
     boolean useOpenSLES = sharedPrefGetBoolean(R.string.pref_opensles_key,
         CallActivity.EXTRA_OPENSLES_ENABLED, R.string.pref_opensles_default, useValuesFromIntent);
@@ -476,6 +480,7 @@ public class ConnectActivity extends Activity {
       intent.putExtra(CallActivity.EXTRA_FLEXFEC_ENABLED, flexfecEnabled);
       intent.putExtra(CallActivity.EXTRA_NOAUDIOPROCESSING_ENABLED, noAudioProcessing);
       intent.putExtra(CallActivity.EXTRA_AECDUMP_ENABLED, aecDump);
+      intent.putExtra(CallActivity.EXTRA_SAVE_INPUT_AUDIO_TO_FILE_ENABLED, saveInputAudioToFile);
       intent.putExtra(CallActivity.EXTRA_OPENSLES_ENABLED, useOpenSLES);
       intent.putExtra(CallActivity.EXTRA_DISABLE_BUILT_IN_AEC, disableBuiltInAEC);
       intent.putExtra(CallActivity.EXTRA_DISABLE_BUILT_IN_AGC, disableBuiltInAGC);

examples/androidapp/src/org/appspot/apprtc/PeerConnectionClient.java

@@ -35,6 +35,7 @@ import java.util.concurrent.Executors;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import org.appspot.apprtc.AppRTCClient.SignalingParameters;
+import org.appspot.apprtc.RecordedAudioToFileController;
 import org.webrtc.AudioSource;
 import org.webrtc.AudioTrack;
 import org.webrtc.CameraVideoCapturer;
@@ -163,6 +164,9 @@ public class PeerConnectionClient {
   private boolean dataChannelEnabled;
   // Enable RtcEventLog.
   private RtcEventLog rtcEventLog;
+  // Implements the WebRtcAudioRecordSamplesReadyCallback interface and writes
+  // recorded audio samples to an output file.
+  private RecordedAudioToFileController saveRecordedAudioToFile = null;
 
   /**
    * Peer connection parameters.
@@ -204,6 +208,7 @@ public class PeerConnectionClient {
     public final String audioCodec;
     public final boolean noAudioProcessing;
     public final boolean aecDump;
+    public final boolean saveInputAudioToFile;
     public final boolean useOpenSLES;
     public final boolean disableBuiltInAEC;
     public final boolean disableBuiltInAGC;
@@ -216,22 +221,10 @@ public class PeerConnectionClient {
     public PeerConnectionParameters(boolean videoCallEnabled, boolean loopback, boolean tracing,
         int videoWidth, int videoHeight, int videoFps, int videoMaxBitrate, String videoCodec,
         boolean videoCodecHwAcceleration, boolean videoFlexfecEnabled, int audioStartBitrate,
-        String audioCodec, boolean noAudioProcessing, boolean aecDump, boolean useOpenSLES,
+        String audioCodec, boolean noAudioProcessing, boolean aecDump, boolean saveInputAudioToFile,
-        boolean disableBuiltInAEC, boolean disableBuiltInAGC, boolean disableBuiltInNS,
+        boolean useOpenSLES, boolean disableBuiltInAEC, boolean disableBuiltInAGC,
-        boolean enableLevelControl, boolean disableWebRtcAGCAndHPF, boolean enableRtcEventLog) {
+        boolean disableBuiltInNS, boolean enableLevelControl, boolean disableWebRtcAGCAndHPF,
-      this(videoCallEnabled, loopback, tracing, videoWidth, videoHeight, videoFps, videoMaxBitrate,
+        boolean enableRtcEventLog, DataChannelParameters dataChannelParameters) {
-          videoCodec, videoCodecHwAcceleration, videoFlexfecEnabled, audioStartBitrate, audioCodec,
-          noAudioProcessing, aecDump, useOpenSLES, disableBuiltInAEC, disableBuiltInAGC,
-          disableBuiltInNS, enableLevelControl, disableWebRtcAGCAndHPF, enableRtcEventLog, null);
-    }
-
-    public PeerConnectionParameters(boolean videoCallEnabled, boolean loopback, boolean tracing,
-        int videoWidth, int videoHeight, int videoFps, int videoMaxBitrate, String videoCodec,
-        boolean videoCodecHwAcceleration, boolean videoFlexfecEnabled, int audioStartBitrate,
-        String audioCodec, boolean noAudioProcessing, boolean aecDump, boolean useOpenSLES,
-        boolean disableBuiltInAEC, boolean disableBuiltInAGC, boolean disableBuiltInNS,
-        boolean enableLevelControl, boolean disableWebRtcAGCAndHPF, boolean enableRtcEventLog,
-        DataChannelParameters dataChannelParameters) {
       this.videoCallEnabled = videoCallEnabled;
       this.loopback = loopback;
       this.tracing = tracing;
@@ -246,6 +239,7 @@ public class PeerConnectionClient {
       this.audioCodec = audioCodec;
       this.noAudioProcessing = noAudioProcessing;
       this.aecDump = aecDump;
+      this.saveInputAudioToFile = saveInputAudioToFile;
       this.useOpenSLES = useOpenSLES;
       this.disableBuiltInAEC = disableBuiltInAEC;
       this.disableBuiltInAGC = disableBuiltInAGC;
@@ -508,6 +502,22 @@ public class PeerConnectionClient {
       }
     });
 
+    // It is possible to save a copy in raw PCM format on a file by checking
+    // the "Save input audio to file" checkbox in the Settings UI. A callback
+    // interface is set when this flag is enabled. As a result, a copy of recorded
+    // audio samples are provided to this client directly from the native audio
+    // layer in Java.
+    if (peerConnectionParameters.saveInputAudioToFile) {
+      if (!peerConnectionParameters.useOpenSLES) {
+        Log.d(TAG, "Enable recording of microphone input audio to file");
+        saveRecordedAudioToFile = new RecordedAudioToFileController(executor);
+      } else {
+        // TODO(henrika): ensure that the UI reflects that if OpenSL ES is selected,
+        // then the "Save inut audio to file" option shall be grayed out.
+        Log.e(TAG, "Recording of input audio is not supported for OpenSL ES");
+      }
+    }
+
     WebRtcAudioTrack.setErrorCallback(new WebRtcAudioTrack.ErrorCallback() {
       @Override
       public void onWebRtcAudioTrackInitError(String errorMessage) {
@@ -677,6 +687,11 @@ public class PeerConnectionClient {
       }
     }
 
+    if (saveRecordedAudioToFile != null) {
+      if (saveRecordedAudioToFile.start()) {
+        Log.d(TAG, "Recording input audio to file is activated");
+      }
+    }
     Log.d(TAG, "Peer connection created.");
   }
 
@@ -740,6 +755,11 @@ public class PeerConnectionClient {
       videoSource.dispose();
       videoSource = null;
     }
+    if (saveRecordedAudioToFile != null) {
+      Log.d(TAG, "Closing audio file for recorded input audio.");
+      saveRecordedAudioToFile.stop();
+      saveRecordedAudioToFile = null;
+    }
     localRender = null;
     remoteRenders = null;
     Log.d(TAG, "Closing peer connection factory.");

examples/androidapp/src/org/appspot/apprtc/RecordedAudioToFileController.java

@@ -0,0 +1,138 @@
+/*
+ *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.appspot.apprtc;
+
+import android.media.AudioFormat;
+import android.os.Environment;
+import android.util.Log;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.concurrent.ExecutorService;
+import org.webrtc.voiceengine.WebRtcAudioRecord;
+import org.webrtc.voiceengine.WebRtcAudioRecord.AudioSamples;
+import org.webrtc.voiceengine.WebRtcAudioRecord.WebRtcAudioRecordSamplesReadyCallback;
+
+/**
+ * Implements the WebRtcAudioRecordSamplesReadyCallback interface and writes
+ * recorded raw audio samples to an output file.
+ */
+public class RecordedAudioToFileController implements WebRtcAudioRecordSamplesReadyCallback {
+  private static final String TAG = "RecordedAudioToFile";
+  private static final long MAX_FILE_SIZE_IN_BYTES = 58348800L;
+
+  private final Object lock = new Object();
+  private final ExecutorService executor;
+  private OutputStream rawAudioFileOutputStream = null;
+  private long fileSizeInBytes = 0;
+
+  public RecordedAudioToFileController(ExecutorService executor) {
+    Log.d(TAG, "ctor");
+    this.executor = executor;
+  }
+
+  /**
+   * Should be called on the same executor thread as the one provided at
+   * construction.
+   */
+  public boolean start() {
+    Log.d(TAG, "start");
+    if (!isExternalStorageWritable()) {
+      Log.e(TAG, "Writing to external media is not possible");
+      return false;
+    }
+    // Register this class as receiver of recorded audio samples for storage.
+    WebRtcAudioRecord.setOnAudioSamplesReady(this);
+    return true;
+  }
+
+  /**
+   * Should be called on the same executor thread as the one provided at
+   * construction.
+   */
+  public void stop() {
+    Log.d(TAG, "stop");
+    // De-register this class as receiver of recorded audio samples for storage.
+    WebRtcAudioRecord.setOnAudioSamplesReady(null);
+    synchronized (lock) {
+      if (rawAudioFileOutputStream != null) {
+        try {
+          rawAudioFileOutputStream.close();
+        } catch (IOException e) {
+          Log.e(TAG, "Failed to close file with saved input audio: " + e);
+        }
+        rawAudioFileOutputStream = null;
+      }
+      fileSizeInBytes = 0;
+    }
+  }
+
+  // Checks if external storage is available for read and write.
+  private boolean isExternalStorageWritable() {
+    String state = Environment.getExternalStorageState();
+    if (Environment.MEDIA_MOUNTED.equals(state)) {
+      return true;
+    }
+    return false;
+  }
+
+  // Utilizes audio parameters to create a file name which contains sufficient
+  // information so that the file can be played using an external file player.
+  // Example: /sdcard/recorded_audio_16bits_48000Hz_mono.pcm.
+  private void openRawAudioOutputFile(int sampleRate, int channelCount) {
+    final String fileName = Environment.getExternalStorageDirectory().getPath() + File.separator
+        + "recorded_audio_16bits_" + String.valueOf(sampleRate) + "Hz"
+        + ((channelCount == 1) ? "_mono" : "_stereo") + ".pcm";
+    final File outputFile = new File(fileName);
+    try {
+      rawAudioFileOutputStream = new FileOutputStream(outputFile);
+    } catch (FileNotFoundException e) {
+      Log.e(TAG, "Failed to open audio output file: " + e.getMessage());
+    }
+    Log.d(TAG, "Opened file for recording: " + fileName);
+  }
+
+  // Called when new audio samples are ready.
+  @Override
+  public void onWebRtcAudioRecordSamplesReady(AudioSamples samples) {
+    // The native audio layer on Android should use 16-bit PCM format.
+    if (samples.getAudioFormat() != AudioFormat.ENCODING_PCM_16BIT) {
+      Log.e(TAG, "Invalid audio format");
+      return;
+    }
+    // Open a new file for the first callback only since it allows us to add
+    // audio parameters to the file name.
+    synchronized (lock) {
+      if (rawAudioFileOutputStream == null) {
+        openRawAudioOutputFile(samples.getSampleRate(), samples.getChannelCount());
+        fileSizeInBytes = 0;
+      }
+    }
+    // Append the recorded 16-bit audio samples to the open output file.
+    executor.execute(() -> {
+      if (rawAudioFileOutputStream != null) {
+        try {
+          // Set a limit on max file size. 58348800 bytes corresponds to
+          // approximately 10 minutes of recording in mono at 48kHz.
+          if (fileSizeInBytes < MAX_FILE_SIZE_IN_BYTES) {
+            // Writes samples.getData().length bytes to output stream.
+            rawAudioFileOutputStream.write(samples.getData());
+            fileSizeInBytes += samples.getData().length;
+          }
+        } catch (IOException e) {
+          Log.e(TAG, "Failed to write audio to file: " + e.getMessage());
+        }
+      }
+    });
+  }
+}

examples/androidapp/src/org/appspot/apprtc/SettingsActivity.java

@@ -42,6 +42,7 @@ public class SettingsActivity extends Activity implements OnSharedPreferenceChan
   private String keyPrefAudioCodec;
   private String keyprefNoAudioProcessing;
   private String keyprefAecDump;
+  private String keyprefEnableSaveInputAudioToFile;
   private String keyprefOpenSLES;
   private String keyprefDisableBuiltInAEC;
   private String keyprefDisableBuiltInAGC;
@@ -84,6 +85,8 @@ public class SettingsActivity extends Activity implements OnSharedPreferenceChan
     keyPrefAudioCodec = getString(R.string.pref_audiocodec_key);
     keyprefNoAudioProcessing = getString(R.string.pref_noaudioprocessing_key);
     keyprefAecDump = getString(R.string.pref_aecdump_key);
+    keyprefEnableSaveInputAudioToFile =
+        getString(R.string.pref_enable_save_input_audio_to_file_key);
     keyprefOpenSLES = getString(R.string.pref_opensles_key);
     keyprefDisableBuiltInAEC = getString(R.string.pref_disable_built_in_aec_key);
     keyprefDisableBuiltInAGC = getString(R.string.pref_disable_built_in_agc_key);
@@ -140,6 +143,7 @@ public class SettingsActivity extends Activity implements OnSharedPreferenceChan
     updateSummary(sharedPreferences, keyPrefAudioCodec);
     updateSummaryB(sharedPreferences, keyprefNoAudioProcessing);
     updateSummaryB(sharedPreferences, keyprefAecDump);
+    updateSummaryB(sharedPreferences, keyprefEnableSaveInputAudioToFile);
     updateSummaryB(sharedPreferences, keyprefOpenSLES);
     updateSummaryB(sharedPreferences, keyprefDisableBuiltInAEC);
     updateSummaryB(sharedPreferences, keyprefDisableBuiltInAGC);
@@ -235,6 +239,7 @@ public class SettingsActivity extends Activity implements OnSharedPreferenceChan
         || key.equals(keyprefFlexfec)
         || key.equals(keyprefNoAudioProcessing)
         || key.equals(keyprefAecDump)
+        || key.equals(keyprefEnableSaveInputAudioToFile)
         || key.equals(keyprefOpenSLES)
         || key.equals(keyprefDisableBuiltInAEC)
         || key.equals(keyprefDisableBuiltInAGC)

examples/androidtests/src/org/appspot/apprtc/test/PeerConnectionClientTest.java

@@ -344,9 +344,10 @@ public class PeerConnectionClientTest implements PeerConnectionEvents {
         "OPUS", /* audioCodec */
         false, /* noAudioProcessing */
         false, /* aecDump */
+        false, /* saveInputAudioToFile */
         false /* useOpenSLES */, false /* disableBuiltInAEC */, false /* disableBuiltInAGC */,
         false /* disableBuiltInNS */, false /* enableLevelControl */, false /* disableWebRtcAGC */,
-        false /* enableRtcEventLog */);
+        false /* enableRtcEventLog */, null /*dataChannelParameters */);
   }
 
   private VideoCapturer createCameraCapturer(boolean captureToTexture) {
@@ -380,9 +381,10 @@ public class PeerConnectionClientTest implements PeerConnectionEvents {
         "OPUS", /* audioCodec */
         false, /* noAudioProcessing */
         false, /* aecDump */
+        false, /* saveInputAudioToFile */
         false /* useOpenSLES */, false /* disableBuiltInAEC */, false /* disableBuiltInAGC */,
         false /* disableBuiltInNS */, false /* enableLevelControl */, false /* disableWebRtcAGC */,
-        false /* enableRtcEventLog */);
+        false /* enableRtcEventLog */, null /*dataChannelParameters */);
   }
 
   @Before