Simulation controller (shell arguments parser and simulation runner) and libraries (data access, noise generators, evaluation scores).

Execution flag added to the .py and .sh scripts.
BUILD.gn files adapted (see :lib), APM config files moved.

BUG=webrtc:7218
NOTRY=True

Review-Url: https://codereview.webrtc.org/2715943002
Cr-Commit-Position: refs/heads/master@{#17007}

webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn

@@ -33,6 +33,10 @@ copy("lib") {
   testonly = true
   sources = [
     "quality_assessment/__init__.py",
+    "quality_assessment/data_access.py",
+    "quality_assessment/eval_scores.py",
+    "quality_assessment/noise_generation.py",
+    "quality_assessment/simulation.py",
   ]
   visibility = [ ":*" ]  # Only targets in this file can depend on this.
   outputs = [
@@ -46,7 +50,7 @@ copy("lib") {
 copy("apm_configs") {
   testonly = true
   sources = [
-    "quality_assessment/apm_configs/default.json",
+    "apm_configs/default.json",
   ]
   visibility = [ ":*" ]  # Only targets in this file can depend on this.
   outputs = [

webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment-export.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 # Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
 #
 # Use of this source code is governed by a BSD-style license

webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment-gencfgs.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 # Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
 #
 # Use of this source code is governed by a BSD-style license

webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 # Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
 #
 # Use of this source code is governed by a BSD-style license
@@ -6,3 +6,83 @@
 # tree. An additional intellectual property rights grant can be found
 # in the file PATENTS.  All contributing project authors may
 # be found in the AUTHORS file in the root of the source tree.
+
+"""Perform APM module quality assessment on one or more input files using one or
+   more audioproc_f configuration files and one or more noise generators.
+
+Usage: apm_quality_assessment.py -i audio1.wav [audio2.wav ...]
+                                 -c cfg1.json [cfg2.json ...]
+                                 -n white [echo ...]
+                                 -e audio_level [polqa ...]
+                                 -o /path/to/output
+"""
+
+import argparse
+import logging
+import sys
+
+import quality_assessment.eval_scores as eval_scores
+import quality_assessment.noise_generation as noise_generation
+import quality_assessment.simulation as simulation
+
+_NOISE_GENERATOR_CLASSES = noise_generation.NoiseGenerator.REGISTERED_CLASSES
+_NOISE_GENERATORS_NAMES = _NOISE_GENERATOR_CLASSES.keys()
+_EVAL_SCORE_WORKER_CLASSES = eval_scores.EvaluationScore.REGISTERED_CLASSES
+_EVAL_SCORE_WORKER_NAMES = _EVAL_SCORE_WORKER_CLASSES.keys()
+
+_DEFAULT_CONFIG_FILE = 'apm_configs/default.json'
+
+def _instance_arguments_parser():
+  parser = argparse.ArgumentParser(description=(
+      'Perform APM module quality assessment on one or more input files using '
+      'one or more audioproc_f configuration files and one or more noise '
+      'generators.'))
+
+  parser.add_argument('-c', '--config_files', nargs='+', required=False,
+                      help=('path to the configuration files defining the '
+                            'arguments with which the audioproc_f tool is '
+                            'called'),
+                      default=[_DEFAULT_CONFIG_FILE])
+
+  parser.add_argument('-i', '--input_files', nargs='+', required=True,
+                      help='path to the input wav files (one or more)')
+
+  parser.add_argument('-n', '--noise_generators', nargs='+', required=False,
+                      help='custom list of noise generators to use',
+                      choices=_NOISE_GENERATORS_NAMES,
+                      default=_NOISE_GENERATORS_NAMES)
+
+  parser.add_argument('-e', '--eval_scores', nargs='+', required=False,
+                      help='custom list of evaluation scores to use',
+                      choices=_EVAL_SCORE_WORKER_NAMES,
+                      default=_EVAL_SCORE_WORKER_NAMES)
+
+  parser.add_argument('-o', '--output_dir', required=False,
+                      help=('base path to the output directory in which the '
+                            'output wav files and the evaluation outcomes '
+                            'are saved'),
+                      default='output')
+
+  return parser
+
+
+def main():
+  # TODO(alessiob): level = logging.INFO once debugged.
+  logging.basicConfig(level=logging.DEBUG)
+
+  parser = _instance_arguments_parser()
+  args = parser.parse_args()
+
+  simulator = simulation.ApmModuleSimulator()
+  simulator.run(
+      config_filepaths=args.config_files,
+      input_filepaths=args.input_files,
+      noise_generator_names=args.noise_generators,
+      eval_score_names=args.eval_scores,
+      output_dir=args.output_dir)
+
+  sys.exit(0)
+
+
+if __name__ == '__main__':
+  main()

webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.sh

@@ -8,7 +8,7 @@
 # be found in the AUTHORS file in the root of the source tree.
 
 # Customize probing signals, noise sources and scores if needed.
-PROBING_SIGNALS=(py_quality_assessment/probing_signals/*.wav)
+PROBING_SIGNALS=(probing_signals/*.wav)
 NOISE_SOURCES=( \
     "identity" \
     "white" \
@@ -26,7 +26,7 @@ chmod +x apm_quality_assessment-gencfgs.py
 ./apm_quality_assessment-gencfgs.py
 
 # Customize APM configurations if needed.
-APM_CONFIGS=(py_quality_assessment/apm_configs/*.json)
+APM_CONFIGS=(apm_configs/*.json)
 
 # Add output path if missing.
 if [ ! -d ${OUTPUT_PATH} ]; then
@@ -56,6 +56,7 @@ done
 wait
 
 # Export results.
+chmod +x ./apm_quality_assessment-export.py
 ./apm_quality_assessment-export.py -o ${OUTPUT_PATH}
 
 # Show results in the browser.

webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/data_access.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import os
+
+def make_directory(path):
+  """
+  Recursively make a directory without rising exceptions if it already exists.
+  """
+  if os.path.exists(path):
+    return
+  os.makedirs(path)

webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores.py

@@ -0,0 +1,56 @@
+# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+class EvaluationScore(object):
+
+  NAME = None
+  REGISTERED_CLASSES = {}
+
+  def __init__(self):
+    pass
+
+  @classmethod
+  def register_class(cls, class_to_register):
+    """
+    Decorator to automatically register the classes that extend EvaluationScore.
+    """
+    cls.REGISTERED_CLASSES[class_to_register.NAME] = class_to_register
+
+
+@EvaluationScore.register_class
+class AudioLevelScore(EvaluationScore):
+  """
+  Compute the difference between the average audio level of the tested and
+  the reference signals.
+
+  Unit: dB
+  Ideal: 0 dB
+  Worst case: +/-inf dB
+  """
+
+  NAME = 'audio_level'
+
+  def __init__(self):
+    super(AudioLevelScore, self).__init__()
+
+
+@EvaluationScore.register_class
+class PolqaScore(EvaluationScore):
+  """
+  Compute the POLQA score. It requires that the POLQA_PATH environment variable
+  points to the PolqaOem64 executable.
+
+  Unit: MOS
+  Ideal: 4.5
+  Worst case: 1.0
+  """
+
+  NAME = 'polqa'
+
+  def __init__(self):
+    super(PolqaScore, self).__init__()

webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/noise_generation.py

@@ -0,0 +1,84 @@
+# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+class NoiseGenerator(object):
+
+  NAME = None
+  REGISTERED_CLASSES = {}
+
+  def __init__(self):
+    pass
+
+  @classmethod
+  def register_class(cls, class_to_register):
+    """
+    Decorator to automatically register the classes that extend NoiseGenerator.
+    """
+    cls.REGISTERED_CLASSES[class_to_register.NAME] = class_to_register
+
+
+# Identity generator.
+@NoiseGenerator.register_class
+class IdentityGenerator(NoiseGenerator):
+  """
+  Generator that adds no noise, therefore both the noisy and the reference
+  signals are the input signal.
+  """
+
+  NAME = 'identity'
+
+  def __init__(self):
+    super(IdentityGenerator, self).__init__()
+
+
+@NoiseGenerator.register_class
+class WhiteNoiseGenerator(NoiseGenerator):
+  """
+  Additive white noise generator.
+  """
+
+  NAME = 'white'
+
+  def __init__(self):
+    super(WhiteNoiseGenerator, self).__init__()
+
+
+@NoiseGenerator.register_class
+class NarrowBandNoiseGenerator(NoiseGenerator):
+  """
+  Additive narrow-band noise generator.
+  """
+
+  NAME = 'narrow_band'
+
+  def __init__(self):
+    super(NarrowBandNoiseGenerator, self).__init__()
+
+
+@NoiseGenerator.register_class
+class EnvironmentalNoiseGenerator(NoiseGenerator):
+  """
+  Additive environmental noise generator.
+  """
+
+  NAME = 'environmental'
+
+  def __init__(self):
+    super(EnvironmentalNoiseGenerator, self).__init__()
+
+
+@NoiseGenerator.register_class
+class EchoNoiseGenerator(NoiseGenerator):
+  """
+  Echo noise generator.
+  """
+
+  NAME = 'echo'
+
+  def __init__(self):
+    super(EchoNoiseGenerator, self).__init__()

webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py

@@ -0,0 +1,115 @@
+# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import logging
+import os
+
+from . import data_access
+from . import eval_scores
+from . import noise_generation
+
+class ApmModuleSimulator(object):
+
+  _NOISE_GENERATOR_CLASSES = noise_generation.NoiseGenerator.REGISTERED_CLASSES
+  _EVAL_SCORE_WORKER_CLASSES = eval_scores.EvaluationScore.REGISTERED_CLASSES
+
+  def __init__(self):
+    # TODO(alessio): instance when implementation is ready.
+    self._audioproc_wrapper = None
+    self._evaluator = None
+
+    self._base_output_path = None
+    self._noise_generators = None
+    self._evaluation_score_workers = None
+    self._config_filepaths = None
+    self._input_filepaths = None
+
+  def run(self, config_filepaths, input_filepaths, noise_generator_names,
+          eval_score_names, output_dir):
+    """
+    Initializes paths and required instances, then runs all the simulations.
+    """
+    self._base_output_path = os.path.abspath(output_dir)
+
+    # Instance noise generators.
+    self._noise_generators = [
+        self._NOISE_GENERATOR_CLASSES[name]() for name in noise_generator_names]
+
+    # Instance evaluation score workers.
+    self._evaluation_score_workers = [
+        self._EVAL_SCORE_WORKER_CLASSES[name]() for name in eval_score_names]
+
+    # Set APM configuration file paths.
+    self._config_filepaths = self._get_paths_collection(config_filepaths)
+
+    # Set probing signal file paths.
+    self._input_filepaths = self._get_paths_collection(input_filepaths)
+
+    self._simulate_all()
+
+  def _simulate_all(self):
+    """
+    Iterates over the combinations of APM configurations, probing signals, and
+    noise generators.
+    """
+    # Try different APM config files.
+    for config_name in self._config_filepaths:
+      config_filepath = self._config_filepaths[config_name]
+
+      # Try different probing signal files.
+      for input_name in self._input_filepaths:
+        input_filepath = self._input_filepaths[input_name]
+
+        # Try different noise generators.
+        for noise_generator in self._noise_generators:
+          logging.info('config: <%s>, input: <%s>, noise: <%s>',
+                       config_name, input_name, noise_generator.NAME)
+
+          # Output path for the input-noise pairs. It is used to cache the noisy
+          # copies of the probing signals (shared across some simulations).
+          input_noise_cache_path = os.path.join(
+              self._base_output_path,
+              '_cache',
+              'input_{}-noise_{}'.format(input_name, noise_generator.NAME))
+          data_access.make_directory(input_noise_cache_path)
+          logging.debug('input-noise cache path: <%s>', input_noise_cache_path)
+
+          # Full output path.
+          output_path = os.path.join(
+              self._base_output_path,
+              'cfg-{}'.format(config_name),
+              'input-{}'.format(input_name),
+              'noise-{}'.format(noise_generator.NAME))
+          data_access.make_directory(output_path)
+          logging.debug('output path: <%s>', output_path)
+
+          self._simulate(noise_generator, input_filepath,
+                         input_noise_cache_path, output_path, config_filepath)
+
+  def _simulate(self, noise_generator, input_filepath, input_noise_cache_path,
+                output_path, config_filepath):
+    """
+    Simulates a given combination of APM configurations, probing signals, and
+    noise generators. It iterates over the noise generator internal
+    configurations.
+    """
+    # TODO(alessio): implement.
+    pass
+
+  @classmethod
+  def _get_paths_collection(cls, filepaths):
+    """
+    Given a list of file paths, makes a collection with one pair for each item
+    in the list where the key is the file name without extension and the value
+    is the path.
+    """
+    filepaths_collection = {}
+    for filepath in filepaths:
+      name = os.path.splitext(os.path.split(filepath)[1])[0]
+      filepaths_collection[name] = os.path.abspath(filepath)
+    return filepaths_collection

webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_evalscores.py

@@ -0,0 +1,19 @@
+# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import unittest
+
+from . import eval_scores
+
+class TestEvalScores(unittest.TestCase):
+
+  def test_registered_classes(self):
+    # Check that there is at least one registered evaluation score worker.
+    classes = eval_scores.EvaluationScore.REGISTERED_CLASSES
+    self.assertIsInstance(classes, dict)
+    self.assertGreater(len(classes), 0)

webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_noisegen.py

@@ -0,0 +1,19 @@
+# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import unittest
+
+from . import noise_generation
+
+class TestNoiseGen(unittest.TestCase):
+
+  def test_registered_classes(self):
+    # Check that there is at least one registered noise generator.
+    classes = noise_generation.NoiseGenerator.REGISTERED_CLASSES
+    self.assertIsInstance(classes, dict)
+    self.assertGreater(len(classes), 0)

webrtc/modules/audio_processing/test/py_quality_assessment/test_simulation.py

@@ -0,0 +1,19 @@
+# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import unittest
+
+import apm_quality_assessment
+
+class TestSimulationScript(unittest.TestCase):
+
+  def test_main(self):
+    # Exit with error code if no arguments are passed.
+    with self.assertRaises(SystemExit) as cm:
+      apm_quality_assessment.main()
+    self.assertGreater(cm.exception.code, 0)