Added ACM_dump protobuf, class for reading/writing and unittest.

This adds a class to read and write ACM_dump protobuf files. In this CL
it is not hooked up to actually store any packets or debug events.
The unittest writes two dummy RTP packets to disk and reads them to see
if they contain the expected data.

BUG=webrtc:4741
R=andrew@webrtc.org, henrik.lundin@webrtc.org, kjellander@webrtc.org, kwiberg@webrtc.org, tina.legrand@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/52059005

Cr-Commit-Position: refs/heads/master@{#9460}

DEPS

@@ -34,6 +34,7 @@ include_rules = [
   # WebRTC production code.
   '-base',
   '-chromium',
+  '+external/webrtc/webrtc',  # Android platform build.
   '+gflags',
   '+libyuv',
   '+net',

webrtc/modules/audio_coding/BUILD.gn

@@ -7,6 +7,7 @@
 # be found in the AUTHORS file in the root of the source tree.
 
 import("//build/config/arm.gni")
+import("//third_party/protobuf/proto_library.gni")
 import("../../build/webrtc.gni")
 
 config("audio_coding_config") {
@@ -79,6 +80,30 @@ source_set("audio_coding") {
   }
 }
 
+proto_library("acm_dump_proto") {
+  sources = [
+    "main/acm2/dump.proto",
+  ]
+  proto_out_dir = "webrtc/audio_coding"
+}
+
+source_set("acm_dump") {
+  sources = [
+    "main/acm2/acm_dump.cc",
+    "main/acm2/acm_dump.h",
+  ]
+
+  defines = []
+
+  deps = [
+    ":acm_dump_proto",
+  ]
+
+  if (rtc_enable_protobuf) {
+    defines += [ "RTC_AUDIOCODING_DEBUG_DUMP" ]
+  }
+}
+
 source_set("audio_decoder_interface") {
   sources = [
     "codecs/audio_decoder.cc",

webrtc/modules/audio_coding/main/acm2/acm_dump.cc

@@ -0,0 +1,220 @@
+/*
+ *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/audio_coding/main/acm2/acm_dump.h"
+
+#include <sstream>
+
+#include "webrtc/base/checks.h"
+#include "webrtc/base/thread_annotations.h"
+#include "webrtc/system_wrappers/interface/clock.h"
+#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
+#include "webrtc/system_wrappers/interface/file_wrapper.h"
+
+// Files generated at build-time by the protobuf compiler.
+#ifdef WEBRTC_ANDROID_PLATFORM_BUILD
+#include "external/webrtc/webrtc/modules/audio_coding/dump.pb.h"
+#else
+#include "webrtc/audio_coding/dump.pb.h"
+#endif
+
+namespace webrtc {
+
+// Noop implementation if flag is not set
+#ifndef RTC_AUDIOCODING_DEBUG_DUMP
+class AcmDumpImpl final : public AcmDump {
+ public:
+  void StartLogging(const std::string& file_name, int duration_ms) override{};
+  void LogRtpPacket(bool incoming,
+                    const uint8_t* packet,
+                    size_t length) override{};
+  void LogDebugEvent(DebugEvent event_type,
+                     const std::string& event_message) override{};
+  void LogDebugEvent(DebugEvent event_type) override{};
+};
+#else
+
+class AcmDumpImpl final : public AcmDump {
+ public:
+  AcmDumpImpl();
+
+  void StartLogging(const std::string& file_name, int duration_ms) override;
+  void LogRtpPacket(bool incoming,
+                    const uint8_t* packet,
+                    size_t length) override;
+  void LogDebugEvent(DebugEvent event_type,
+                     const std::string& event_message) override;
+  void LogDebugEvent(DebugEvent event_type) override;
+
+ private:
+  // Checks if the logging time has expired, and if so stops the logging.
+  void StopIfNecessary() EXCLUSIVE_LOCKS_REQUIRED(crit_);
+  // Stops logging and clears the stored data and buffers.
+  void Clear() EXCLUSIVE_LOCKS_REQUIRED(crit_);
+  // Returns true if the logging is currently active.
+  bool CurrentlyLogging() const EXCLUSIVE_LOCKS_REQUIRED(crit_) {
+    return active_ &&
+           (clock_->TimeInMicroseconds() <= start_time_us_ + duration_us_);
+  }
+  // This function is identical to LogDebugEvent, but requires holding the lock.
+  void LogDebugEventLocked(DebugEvent event_type,
+                           const std::string& event_message)
+      EXCLUSIVE_LOCKS_REQUIRED(crit_);
+
+  rtc::scoped_ptr<webrtc::CriticalSectionWrapper> crit_;
+  rtc::scoped_ptr<webrtc::FileWrapper> file_ GUARDED_BY(crit_);
+  rtc::scoped_ptr<ACMDumpEventStream> stream_ GUARDED_BY(crit_);
+  bool active_ GUARDED_BY(crit_);
+  int64_t start_time_us_ GUARDED_BY(crit_);
+  int64_t duration_us_ GUARDED_BY(crit_);
+  const webrtc::Clock* clock_ GUARDED_BY(crit_);
+};
+
+namespace {
+
+// Convert from AcmDump's debug event enum (runtime format) to the corresponding
+// protobuf enum (serialized format).
+ACMDumpDebugEvent_EventType convertDebugEvent(AcmDump::DebugEvent event_type) {
+  switch (event_type) {
+    case AcmDump::DebugEvent::kLogStart:
+      return ACMDumpDebugEvent::LOG_START;
+    case AcmDump::DebugEvent::kLogEnd:
+      return ACMDumpDebugEvent::LOG_END;
+    case AcmDump::DebugEvent::kAudioPlayout:
+      return ACMDumpDebugEvent::AUDIO_PLAYOUT;
+  }
+  return ACMDumpDebugEvent::UNKNOWN_EVENT;
+}
+
+}  // Anonymous namespace.
+
+// AcmDumpImpl member functions.
+AcmDumpImpl::AcmDumpImpl()
+    : crit_(webrtc::CriticalSectionWrapper::CreateCriticalSection()),
+      file_(webrtc::FileWrapper::Create()),
+      stream_(new webrtc::ACMDumpEventStream()),
+      active_(false),
+      start_time_us_(0),
+      duration_us_(0),
+      clock_(webrtc::Clock::GetRealTimeClock()) {
+}
+
+void AcmDumpImpl::StartLogging(const std::string& file_name, int duration_ms) {
+  CriticalSectionScoped lock(crit_.get());
+  Clear();
+  if (file_->OpenFile(file_name.c_str(), false) != 0) {
+    return;
+  }
+  // Add a single object to the stream that is reused at every log event.
+  stream_->add_stream();
+  active_ = true;
+  start_time_us_ = clock_->TimeInMicroseconds();
+  duration_us_ = static_cast<int64_t>(duration_ms) * 1000;
+  // Log the start event.
+  std::stringstream log_msg;
+  log_msg << "Initial timestamp: " << start_time_us_;
+  LogDebugEventLocked(DebugEvent::kLogStart, log_msg.str());
+}
+
+void AcmDumpImpl::LogRtpPacket(bool incoming,
+                               const uint8_t* packet,
+                               size_t length) {
+  CriticalSectionScoped lock(crit_.get());
+  if (!CurrentlyLogging()) {
+    StopIfNecessary();
+    return;
+  }
+  // Reuse the same object at every log event.
+  auto rtp_event = stream_->mutable_stream(0);
+  rtp_event->clear_debug_event();
+  const int64_t timestamp = clock_->TimeInMicroseconds() - start_time_us_;
+  rtp_event->set_timestamp_us(timestamp);
+  rtp_event->set_type(webrtc::ACMDumpEvent::RTP_EVENT);
+  rtp_event->mutable_packet()->set_direction(
+      incoming ? ACMDumpRTPPacket::INCOMING : ACMDumpRTPPacket::OUTGOING);
+  rtp_event->mutable_packet()->set_rtp_data(packet, length);
+  std::string dump_buffer;
+  stream_->SerializeToString(&dump_buffer);
+  file_->Write(dump_buffer.data(), dump_buffer.size());
+  file_->Flush();
+}
+
+void AcmDumpImpl::LogDebugEvent(DebugEvent event_type,
+                                const std::string& event_message) {
+  CriticalSectionScoped lock(crit_.get());
+  LogDebugEventLocked(event_type, event_message);
+}
+
+void AcmDumpImpl::LogDebugEvent(DebugEvent event_type) {
+  CriticalSectionScoped lock(crit_.get());
+  LogDebugEventLocked(event_type, "");
+}
+
+void AcmDumpImpl::StopIfNecessary() {
+  if (active_) {
+    DCHECK_GT(clock_->TimeInMicroseconds(), start_time_us_ + duration_us_);
+    LogDebugEventLocked(DebugEvent::kLogEnd, "");
+    Clear();
+  }
+}
+
+void AcmDumpImpl::Clear() {
+  if (active_ || file_->Open()) {
+    file_->CloseFile();
+  }
+  active_ = false;
+  stream_->Clear();
+}
+
+void AcmDumpImpl::LogDebugEventLocked(DebugEvent event_type,
+                                      const std::string& event_message) {
+  if (!CurrentlyLogging()) {
+    StopIfNecessary();
+    return;
+  }
+
+  // Reuse the same object at every log event.
+  auto event = stream_->mutable_stream(0);
+  int64_t timestamp = clock_->TimeInMicroseconds() - start_time_us_;
+  event->set_timestamp_us(timestamp);
+  event->set_type(webrtc::ACMDumpEvent::DEBUG_EVENT);
+  event->clear_packet();
+  auto debug_event = event->mutable_debug_event();
+  debug_event->set_type(convertDebugEvent(event_type));
+  debug_event->set_message(event_message);
+  std::string dump_buffer;
+  stream_->SerializeToString(&dump_buffer);
+  file_->Write(dump_buffer.data(), dump_buffer.size());
+}
+
+#endif  // RTC_AUDIOCODING_DEBUG_DUMP
+
+// AcmDump member functions.
+rtc::scoped_ptr<AcmDump> AcmDump::Create() {
+  return rtc::scoped_ptr<AcmDump>(new AcmDumpImpl());
+}
+
+bool AcmDump::ParseAcmDump(const std::string& file_name,
+                           ACMDumpEventStream* result) {
+  char tmp_buffer[1024];
+  int bytes_read = 0;
+  rtc::scoped_ptr<FileWrapper> dump_file(FileWrapper::Create());
+  if (dump_file->OpenFile(file_name.c_str(), true) != 0) {
+    return false;
+  }
+  std::string dump_buffer;
+  while ((bytes_read = dump_file->Read(tmp_buffer, sizeof(tmp_buffer))) > 0) {
+    dump_buffer.append(tmp_buffer, bytes_read);
+  }
+  dump_file->CloseFile();
+  return result->ParseFromString(dump_buffer);
+}
+
+}  // namespace webrtc

webrtc/modules/audio_coding/main/acm2/acm_dump.h

@@ -0,0 +1,59 @@
+/*
+ *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MODULES_AUDIO_CODING_MAIN_ACM2_ACM_DUMP_H_
+#define WEBRTC_MODULES_AUDIO_CODING_MAIN_ACM2_ACM_DUMP_H_
+
+#include <string>
+
+#include "webrtc/base/scoped_ptr.h"
+
+namespace webrtc {
+
+// Forward declaration of storage class that is automatically generated from
+// the protobuf file.
+class ACMDumpEventStream;
+
+class AcmDumpImpl;
+
+class AcmDump {
+ public:
+  // The types of debug events that are currently supported for logging.
+  enum class DebugEvent { kLogStart, kLogEnd, kAudioPlayout };
+
+  virtual ~AcmDump() {}
+
+  static rtc::scoped_ptr<AcmDump> Create();
+
+  // Starts logging for the specified duration to the specified file.
+  // The logging will stop automatically after the specified duration.
+  // If the file already exists it will be overwritten.
+  // The function will return false on failure.
+  virtual void StartLogging(const std::string& file_name, int duration_ms) = 0;
+
+  // Logs an incoming or outgoing RTP packet.
+  virtual void LogRtpPacket(bool incoming,
+                            const uint8_t* packet,
+                            size_t length) = 0;
+
+  // Logs a debug event, with optional message.
+  virtual void LogDebugEvent(DebugEvent event_type,
+                             const std::string& event_message) = 0;
+  virtual void LogDebugEvent(DebugEvent event_type) = 0;
+
+  // Reads an AcmDump file and returns true when reading was successful.
+  // The result is stored in the given ACMDumpEventStream object.
+  static bool ParseAcmDump(const std::string& file_name,
+                           ACMDumpEventStream* result);
+};
+
+}  // namespace webrtc
+
+#endif  // WEBRTC_MODULES_AUDIO_CODING_MAIN_ACM2_ACM_DUMP_H_

webrtc/modules/audio_coding/main/acm2/acm_dump_unittest.cc

@@ -0,0 +1,117 @@
+/*
+ *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifdef RTC_AUDIOCODING_DEBUG_DUMP
+
+#include <stdio.h>
+#include <string>
+#include <vector>
+
+#include "testing/gtest/include/gtest/gtest.h"
+#include "webrtc/base/scoped_ptr.h"
+#include "webrtc/modules/audio_coding/main/acm2/acm_dump.h"
+#include "webrtc/system_wrappers/interface/clock.h"
+#include "webrtc/test/test_suite.h"
+#include "webrtc/test/testsupport/fileutils.h"
+#include "webrtc/test/testsupport/gtest_disable.h"
+
+// Files generated at build-time by the protobuf compiler.
+#ifdef WEBRTC_ANDROID_PLATFORM_BUILD
+#include "external/webrtc/webrtc/modules/audio_coding/dump.pb.h"
+#else
+#include "webrtc/audio_coding/dump.pb.h"
+#endif
+
+namespace webrtc {
+
+// Test for the acm dump class. Dumps some RTP packets to disk, then reads them
+// back to see if they match.
+class AcmDumpTest : public ::testing::Test {
+ public:
+  AcmDumpTest() : log_dumper_(AcmDump::Create()) {}
+  void VerifyResults(const ACMDumpEventStream& parsed_stream,
+                     size_t packet_size) {
+    // Verify the result.
+    EXPECT_EQ(3, parsed_stream.stream_size());
+    const ACMDumpEvent& start_event = parsed_stream.stream(0);
+    ASSERT_TRUE(start_event.has_type());
+    EXPECT_EQ(ACMDumpEvent::DEBUG_EVENT, start_event.type());
+    EXPECT_TRUE(start_event.has_timestamp_us());
+    EXPECT_FALSE(start_event.has_packet());
+    ASSERT_TRUE(start_event.has_debug_event());
+    auto start_debug_event = start_event.debug_event();
+    ASSERT_TRUE(start_debug_event.has_type());
+    EXPECT_EQ(ACMDumpDebugEvent::LOG_START, start_debug_event.type());
+    ASSERT_TRUE(start_debug_event.has_message());
+
+    for (int i = 1; i < parsed_stream.stream_size(); i++) {
+      const ACMDumpEvent& test_event = parsed_stream.stream(i);
+      ASSERT_TRUE(test_event.has_type());
+      EXPECT_EQ(ACMDumpEvent::RTP_EVENT, test_event.type());
+      EXPECT_TRUE(test_event.has_timestamp_us());
+      EXPECT_FALSE(test_event.has_debug_event());
+      ASSERT_TRUE(test_event.has_packet());
+      const ACMDumpRTPPacket& test_packet = test_event.packet();
+      ASSERT_TRUE(test_packet.has_direction());
+      if (i == 1) {
+        EXPECT_EQ(ACMDumpRTPPacket::INCOMING, test_packet.direction());
+      } else if (i == 2) {
+        EXPECT_EQ(ACMDumpRTPPacket::OUTGOING, test_packet.direction());
+      }
+      ASSERT_TRUE(test_packet.has_rtp_data());
+      ASSERT_EQ(packet_size, test_packet.rtp_data().size());
+      for (size_t i = 0; i < packet_size; i++) {
+        EXPECT_EQ(rtp_packet_[i],
+                  static_cast<uint8_t>(test_packet.rtp_data()[i]));
+      }
+    }
+  }
+
+  void Run(int packet_size, int random_seed) {
+    rtp_packet_.clear();
+    rtp_packet_.reserve(packet_size);
+    srand(random_seed);
+    // Fill the packet vector with random data.
+    for (int i = 0; i < packet_size; i++) {
+      rtp_packet_.push_back(rand());
+    }
+    // Find the name of the current test, in order to use it as a temporary
+    // filename.
+    auto test_info = ::testing::UnitTest::GetInstance()->current_test_info();
+    const std::string temp_filename =
+        test::OutputPath() + test_info->test_case_name() + test_info->name();
+
+    log_dumper_->StartLogging(temp_filename, 10000000);
+    log_dumper_->LogRtpPacket(true, rtp_packet_.data(), rtp_packet_.size());
+    log_dumper_->LogRtpPacket(false, rtp_packet_.data(), rtp_packet_.size());
+
+    // Read the generated file from disk.
+    ACMDumpEventStream parsed_stream;
+
+    ASSERT_EQ(true, AcmDump::ParseAcmDump(temp_filename, &parsed_stream));
+
+    VerifyResults(parsed_stream, packet_size);
+
+    // Clean up temporary file - can be pretty slow.
+    remove(temp_filename.c_str());
+  }
+
+  std::vector<uint8_t> rtp_packet_;
+  rtc::scoped_ptr<AcmDump> log_dumper_;
+};
+
+TEST_F(AcmDumpTest, DumpAndRead) {
+  Run(256, 321);
+  Run(256, 123);
+}
+
+}  // namespace webrtc
+
+#endif  // RTC_AUDIOCODING_DEBUG_DUMP

webrtc/modules/audio_coding/main/acm2/audio_coding_module.gypi

@@ -78,6 +78,34 @@
         'nack.h',
       ],
     },
+    {
+      'target_name': 'acm_dump_proto',
+      'type': 'static_library',
+      'sources': ['dump.proto',],
+      'variables': {
+        'proto_in_dir': '.',
+        # Workaround to protect against gyp's pathname relativization when
+        # this file is included by modules.gyp.
+        'proto_out_protected': 'webrtc/audio_coding',
+        'proto_out_dir': '<(proto_out_protected)',
+      },
+      'includes': ['../../../../build/protoc.gypi',],
+    },
+    {
+      'target_name': 'acm_dump',
+      'type': 'static_library',
+      'conditions': [
+        ['enable_protobuf==1', {
+          'defines': ['RTC_AUDIOCODING_DEBUG_DUMP'],
+          }
+        ],
+      ],
+      'sources': [
+        'acm_dump.h',
+        'acm_dump.cc'
+      ],
+      'dependencies': ['acm_dump_proto'],
+    },
   ],
   'conditions': [
     ['include_tests==1', {

webrtc/modules/audio_coding/main/acm2/dump.proto

@@ -0,0 +1,78 @@
+syntax = "proto2";
+option optimize_for = LITE_RUNTIME;
+package webrtc;
+
+// This is the main message to dump to a file, it can contain multiple event
+// messages, but it is possible to append multiple EventStreams (each with a
+// single event) to a file.
+// This has the benefit that there's no need to keep all data in memory.
+message ACMDumpEventStream {
+  repeated ACMDumpEvent stream = 1;
+}
+
+message ACMDumpEvent {
+  // required - Elapsed wallclock time in us since the start of the log.
+  optional int64 timestamp_us = 1;
+
+  // The different types of events that can occur, the UNKNOWN_EVENT entry
+  // is added in case future EventTypes are added, in that case old code will
+  // receive the new events as UNKNOWN_EVENT.
+  enum EventType {
+    UNKNOWN_EVENT = 0;
+    RTP_EVENT = 1;
+    DEBUG_EVENT = 2;
+  }
+
+  // required - Indicates the type of this event
+  optional EventType type = 2;
+
+  // optional - but required if type == RTP_EVENT
+  optional ACMDumpRTPPacket packet = 3;
+
+  // optional - but required if type == DEBUG_EVENT
+  optional ACMDumpDebugEvent debug_event = 4;
+}
+
+message ACMDumpRTPPacket {
+  // Indicates if the packet is incoming or outgoing with respect to the user
+  // that is logging the data.
+  enum Direction {
+    UNKNOWN_DIRECTION = 0;
+    OUTGOING = 1;
+    INCOMING = 2;
+  }
+  enum PayloadType {
+    UNKNOWN_TYPE = 0;
+    AUDIO = 1;
+    VIDEO = 2;
+    RTX = 3;
+  }
+
+  // required
+  optional Direction direction = 1;
+
+  // required
+  optional PayloadType type = 2;
+
+  // required - Contains the whole RTP packet (header+payload).
+  optional bytes RTP_data = 3;
+}
+
+message ACMDumpDebugEvent {
+  // Indicates the type of the debug event.
+  // LOG_START and LOG_END indicate the start and end of the log respectively.
+  // AUDIO_PLAYOUT indicates a call to the PlayoutData10Ms() function in ACM.
+  enum EventType {
+    UNKNOWN_EVENT = 0;
+    LOG_START = 1;
+    LOG_END = 2;
+    AUDIO_PLAYOUT = 3;
+  }
+
+  // required
+  optional EventType type = 1;
+
+  // An optional message that can be used to store additional information about
+  // the debug event.
+  optional string message = 2;
+}

webrtc/modules/modules.gyp

@@ -310,12 +310,17 @@
               'defines': [ 'WEBRTC_AUDIOPROC_FLOAT_PROFILE' ],
             }],
             ['enable_protobuf==1', {
-              'defines': [ 'WEBRTC_AUDIOPROC_DEBUG_DUMP' ],
+              'defines': [
+                'WEBRTC_AUDIOPROC_DEBUG_DUMP',
+                'RTC_AUDIOCODING_DEBUG_DUMP',
+              ],
               'dependencies': [
+                'acm_dump',
                 'audioproc_protobuf_utils',
                 'audioproc_unittest_proto',
               ],
               'sources': [
+                'audio_coding/main/acm2/acm_dump_unittest.cc',
                 'audio_processing/audio_processing_impl_unittest.cc',
                 'audio_processing/test/audio_processing_unittest.cc',
                 'audio_processing/test/test_utils.h',