Add VideoProcessor tests verifying that H.264 keyframes contain SPS/PPS/IDR.

This CL adds an EncodedFrameChecker interface which can be used by users
of the VideoProcessor to inject customized per-frame checks to the
encoding/decoding pipeline. This currently has two uses:
- Verifying that the QP parser works correctly for VP8 and VP9, by comparing the
  parsed QP to that produced by libvpx.
- Verifying that our H.264 encoders always produce SPS/PPS/IDR in tandem.

TESTED=Galaxy S8, Pixel 2 XL, iPhone 7.
BUG=webrtc:8423

Change-Id: Ic3e401546e239a9ffaf2ed2907689cebb1127805
Reviewed-on: https://webrtc-review.googlesource.com/14559
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Commit-Queue: Rasmus Brandt <brandtr@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#20409}

modules/video_coding/codecs/test/test_config.h

@@ -32,6 +32,14 @@ enum ExcludeFrameTypes {
 
 // Test configuration for a test run.
 struct TestConfig {
+  class EncodedFrameChecker {
+   public:
+    virtual ~EncodedFrameChecker() = default;
+
+    virtual void CheckEncodedFrame(webrtc::VideoCodecType codec,
+                                   const EncodedImage& encoded_frame) const = 0;
+  };
+
   void SetCodecSettings(VideoCodecType codec_type,
                         int num_temporal_layers,
                         bool error_concealment_on,
@@ -103,6 +111,9 @@ struct TestConfig {
   // RTP H264 packetization mode.
   H264PacketizationMode packetization_mode =
       webrtc::H264PacketizationMode::NonInterleaved;
+
+  // Custom checker that will be called for each frame.
+  const EncodedFrameChecker* encoded_frame_checker = nullptr;
 };
 
 }  // namespace test

modules/video_coding/codecs/test/videoprocessor.cc

@@ -43,22 +43,6 @@ std::unique_ptr<VideoBitrateAllocator> CreateBitrateAllocator(
                                                     std::move(tl_factory)));
 }
 
-void VerifyQpParser(const EncodedImage& encoded_frame,
-                    const TestConfig& config) {
-  if (config.hw_encoder)
-    return;
-
-  int qp;
-  if (config.codec_settings.codecType == kVideoCodecVP8) {
-    ASSERT_TRUE(vp8::GetQp(encoded_frame._buffer, encoded_frame._length, &qp));
-  } else if (config.codec_settings.codecType == kVideoCodecVP9) {
-    ASSERT_TRUE(vp9::GetQp(encoded_frame._buffer, encoded_frame._length, &qp));
-  } else {
-    return;
-  }
-  EXPECT_EQ(encoded_frame.qp_, qp) << "Encoder QP != parsed bitstream QP.";
-}
-
 rtc::Optional<size_t> GetMaxNaluLength(const EncodedImage& encoded_frame,
                                        const TestConfig& config) {
   if (config.codec_settings.codecType != kVideoCodecH264)
@@ -230,8 +214,9 @@ void VideoProcessor::FrameEncoded(webrtc::VideoCodecType codec,
   // time recordings should wrap the Encode call as tightly as possible.
   int64_t encode_stop_ns = rtc::TimeNanos();
 
-  // Take the opportunity to verify the QP bitstream parser.
-  VerifyQpParser(encoded_image, config_);
+  if (config_.encoded_frame_checker) {
+    config_.encoded_frame_checker->CheckEncodedFrame(codec, encoded_image);
+  }
 
   // Check for dropped frames.
   const int frame_number =

modules/video_coding/codecs/test/videoprocessor_integrationtest.cc

@@ -76,8 +76,43 @@ bool RunEncodeInRealTime(const TestConfig& config) {
   return false;
 #endif
 }
+
 }  // namespace
 
+void VideoProcessorIntegrationTest::H264KeyframeChecker::CheckEncodedFrame(
+    webrtc::VideoCodecType codec,
+    const EncodedImage& encoded_frame) const {
+  EXPECT_EQ(kVideoCodecH264, codec);
+  bool contains_sps = false;
+  bool contains_pps = false;
+  bool contains_idr = false;
+  const std::vector<webrtc::H264::NaluIndex> nalu_indices =
+      webrtc::H264::FindNaluIndices(encoded_frame._buffer,
+                                    encoded_frame._length);
+  for (const webrtc::H264::NaluIndex& index : nalu_indices) {
+    webrtc::H264::NaluType nalu_type = webrtc::H264::ParseNaluType(
+        encoded_frame._buffer[index.payload_start_offset]);
+    if (nalu_type == webrtc::H264::NaluType::kSps) {
+      contains_sps = true;
+    } else if (nalu_type == webrtc::H264::NaluType::kPps) {
+      contains_pps = true;
+    } else if (nalu_type == webrtc::H264::NaluType::kIdr) {
+      contains_idr = true;
+    }
+  }
+  if (encoded_frame._frameType == kVideoFrameKey) {
+    EXPECT_TRUE(contains_sps) << "Keyframe should contain SPS.";
+    EXPECT_TRUE(contains_pps) << "Keyframe should contain PPS.";
+    EXPECT_TRUE(contains_idr) << "Keyframe should contain IDR.";
+  } else if (encoded_frame._frameType == kVideoFrameDelta) {
+    EXPECT_FALSE(contains_sps) << "Delta frame should not contain SPS.";
+    EXPECT_FALSE(contains_pps) << "Delta frame should not contain PPS.";
+    EXPECT_FALSE(contains_idr) << "Delta frame should not contain IDR.";
+  } else {
+    RTC_NOTREACHED();
+  }
+}
+
 class VideoProcessorIntegrationTest::CpuProcessTime final {
  public:
   explicit CpuProcessTime(const TestConfig& config) : config_(config) {}

modules/video_coding/codecs/test/videoprocessor_integrationtest.h

@@ -16,6 +16,7 @@
 #include <vector>
 
 #include "common_types.h"  // NOLINT(build/include)
+#include "common_video/h264/h264_common.h"
 #include "media/engine/webrtcvideodecoderfactory.h"
 #include "media/engine/webrtcvideoencoderfactory.h"
 #include "modules/video_coding/codecs/test/packet_manipulator.h"
@@ -91,6 +92,13 @@ struct VisualizationParams {
 // or breakdown occurs.
 class VideoProcessorIntegrationTest : public testing::Test {
  protected:
+  // Verifies that all H.264 keyframes contain SPS/PPS/IDR NALUs.
+  class H264KeyframeChecker : public TestConfig::EncodedFrameChecker {
+   public:
+    void CheckEncodedFrame(webrtc::VideoCodecType codec,
+                           const EncodedImage& encoded_frame) const override;
+  };
+
   VideoProcessorIntegrationTest();
   ~VideoProcessorIntegrationTest() override;
 
@@ -104,6 +112,9 @@ class VideoProcessorIntegrationTest : public testing::Test {
   // Config.
   TestConfig config_;
 
+  // Can be used by all H.264 tests.
+  const H264KeyframeChecker h264_keyframe_checker_;
+
  private:
   class CpuProcessTime;
   static const int kMaxNumTemporalLayers = 3;

modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc

@@ -12,6 +12,8 @@
 
 #include <vector>
 
+#include "modules/video_coding/codecs/test/test_config.h"
+#include "rtc_base/ptr_util.h"
 #include "test/testsupport/fileutils.h"
 
 namespace webrtc {
@@ -47,7 +49,28 @@ class VideoProcessorIntegrationTestLibvpx
     config_.verbose = false;
     config_.hw_encoder = false;
     config_.hw_decoder = false;
+    config_.encoded_frame_checker = &qp_frame_checker_;
   }
+
+ private:
+  // Verify that the QP parser returns the same QP as the encoder does.
+  const class QpFrameChecker : public TestConfig::EncodedFrameChecker {
+   public:
+    void CheckEncodedFrame(webrtc::VideoCodecType codec,
+                           const EncodedImage& encoded_frame) const override {
+      int qp;
+      if (codec == kVideoCodecVP8) {
+        EXPECT_TRUE(
+            vp8::GetQp(encoded_frame._buffer, encoded_frame._length, &qp));
+      } else if (codec == kVideoCodecVP9) {
+        EXPECT_TRUE(
+            vp9::GetQp(encoded_frame._buffer, encoded_frame._length, &qp));
+      } else {
+        RTC_NOTREACHED();
+      }
+      EXPECT_EQ(encoded_frame.qp_, qp) << "Encoder QP != parsed bitstream QP.";
+    }
+  } qp_frame_checker_;
 };
 
 // Fails on iOS. See webrtc:4755.

modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc

@@ -50,7 +50,27 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsVp8) {
   // implementations pass. If this test fails on the bots, disable it and
   // ping brandtr@.
   std::vector<RateControlThresholds> rc_thresholds = {
-      {20, 95, 22, 11, 10, 0, 1}};
+      {20, 95, 22, 11, 50, 0, 1}};
+
+  QualityThresholds quality_thresholds(30.0, 14.0, 0.86, 0.39);
+
+  ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
+                              &quality_thresholds, nullptr,
+                              kNoVisualizationParams);
+}
+
+TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsH264) {
+  config_.encoded_frame_checker = &h264_keyframe_checker_;
+  config_.SetCodecSettings(kVideoCodecH264, 1, false, false, false, false,
+                           false, 352, 288);
+
+  std::vector<RateProfile> rate_profiles = {{500, 30, kForemanNumFrames + 1}};
+
+  // The thresholds below may have to be tweaked to let even poor MediaCodec
+  // implementations pass. If this test fails on the bots, disable it and
+  // ping brandtr@.
+  std::vector<RateControlThresholds> rc_thresholds = {
+      {20, 95, 22, 11, 20, 0, 1}};
 
   QualityThresholds quality_thresholds(30.0, 14.0, 0.86, 0.39);
 

modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc

@@ -46,6 +46,7 @@ class VideoProcessorIntegrationTestOpenH264
     config_.verbose = false;
     config_.hw_encoder = false;
     config_.hw_decoder = false;
+    config_.encoded_frame_checker = &h264_keyframe_checker_;
   }
 };
 

modules/video_coding/codecs/test/videoprocessor_integrationtest_videotoolbox.cc

@@ -0,0 +1,65 @@
+/*
+ *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/test/videoprocessor_integrationtest.h"
+
+#include <vector>
+
+#include "test/testsupport/fileutils.h"
+
+namespace webrtc {
+namespace test {
+
+#if defined(WEBRTC_IOS)
+
+namespace {
+const int kForemanNumFrames = 300;
+const std::nullptr_t kNoVisualizationParams = nullptr;
+}  // namespace
+
+class VideoProcessorIntegrationTestVideoToolbox
+    : public VideoProcessorIntegrationTest {
+ protected:
+  VideoProcessorIntegrationTestVideoToolbox() {
+    config_.filename = "foreman_cif";
+    config_.input_filename = ResourcePath(config_.filename, "yuv");
+    config_.output_filename = TempFilename(
+        OutputPath(), "videoprocessor_integrationtest_videotoolbox");
+    config_.num_frames = kForemanNumFrames;
+    config_.verbose = false;
+    config_.hw_encoder = true;
+    config_.hw_decoder = true;
+    config_.encoded_frame_checker = &h264_keyframe_checker_;
+  }
+};
+
+// Since we don't currently run the iOS tests on physical devices on the bots,
+// this test is disabled.
+TEST_F(VideoProcessorIntegrationTestVideoToolbox,
+       DISABLED_ForemanCif500kbpsH264) {
+  config_.SetCodecSettings(kVideoCodecH264, 1, false, false, false, false,
+                           false, 352, 288);
+
+  std::vector<RateProfile> rate_profiles = {{500, 30, kForemanNumFrames + 1}};
+
+  std::vector<RateControlThresholds> rc_thresholds = {
+      {20, 95, 60, 60, 10, 0, 1}};
+
+  QualityThresholds quality_thresholds(30.0, 14.0, 0.86, 0.39);
+
+  ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
+                              &quality_thresholds, nullptr,
+                              kNoVisualizationParams);
+}
+
+#endif  // defined(WEBRTC_IOS)
+
+}  // namespace test
+}  // namespace webrtc