Add ability to scale to arbitrary factors

This CL adds a fallback for the case when no optimized scale factor produces a low enough resolution for what was requested. It also ensures that all resolutions provided by the video adapter are divisible by four. This is required by some hardware implementations.

BUG=webrtc:6837

Review-Url: https://codereview.webrtc.org/2555483005
Cr-Commit-Position: refs/heads/master@{#15469}

webrtc/media/base/adaptedvideotracksource.cc

@@ -16,6 +16,11 @@ AdaptedVideoTrackSource::AdaptedVideoTrackSource() {
   thread_checker_.DetachFromThread();
 }
 
+AdaptedVideoTrackSource::AdaptedVideoTrackSource(int required_alignment)
+    : video_adapter_(required_alignment) {
+  thread_checker_.DetachFromThread();
+}
+
 bool AdaptedVideoTrackSource::GetStats(Stats* stats) {
   rtc::CritScope lock(&stats_crit_);
 

webrtc/media/base/adaptedvideotracksource.h

@@ -28,6 +28,9 @@ class AdaptedVideoTrackSource
   AdaptedVideoTrackSource();
 
  protected:
+  // Allows derived classes to initialize |video_adapter_| with a custom
+  // alignment.
+  AdaptedVideoTrackSource(int required_alignment);
   // Checks the apply_rotation() flag. If the frame needs rotation, and it is a
   // plain memory frame, it is rotated. Subclasses producing native frames must
   // handle apply_rotation() themselves.

webrtc/media/base/videoadapter.cc

@@ -11,16 +11,18 @@
 #include "webrtc/media/base/videoadapter.h"
 
 #include <algorithm>
+#include <cmath>
 #include <cstdlib>
 #include <limits>
 
+#include "webrtc/base/arraysize.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
+#include "webrtc/base/optional.h"
 #include "webrtc/media/base/mediaconstants.h"
 #include "webrtc/media/base/videocommon.h"
 
 namespace {
-
 struct Fraction {
   int numerator;
   int denominator;
@@ -37,17 +39,42 @@ const Fraction kScaleFractions[] = {
   {3, 16},
 };
 
-// Round |valueToRound| to a multiple of |multiple|. Prefer rounding upwards,
-// but never more than |maxValue|.
-int roundUp(int valueToRound, int multiple, int maxValue) {
-  const int roundedValue = (valueToRound + multiple - 1) / multiple * multiple;
-  return roundedValue <= maxValue ? roundedValue
-                                  : (maxValue / multiple * multiple);
+// Round |value_to_round| to a multiple of |multiple|. Prefer rounding upwards,
+// but never more than |max_value|.
+int roundUp(int value_to_round, int multiple, int max_value) {
+  const int rounded_value =
+      (value_to_round + multiple - 1) / multiple * multiple;
+  return rounded_value <= max_value ? rounded_value
+                                    : (max_value / multiple * multiple);
 }
 
+// Generates a scale factor that makes |input_num_pixels| smaller than
+// |target_num_pixels|. This should only be used after making sure none
+// of the optimized factors are small enough.
 Fraction FindScaleLessThanOrEqual(int input_num_pixels, int target_num_pixels) {
+  // Start searching from the last of the optimal fractions;
+  Fraction best_scale = kScaleFractions[arraysize(kScaleFractions) - 1];
+  const float target_scale =
+      sqrt(target_num_pixels / static_cast<float>(input_num_pixels));
+  do {
+    if (best_scale.numerator % 3 == 0 && best_scale.denominator % 2 == 0) {
+      // Multiply by 2/3
+      best_scale.numerator /= 3;
+      best_scale.denominator /= 2;
+    } else {
+      // Multiply by 3/4
+      best_scale.numerator *= 3;
+      best_scale.denominator *= 4;
+    }
+  } while (best_scale.numerator > (target_scale * best_scale.denominator));
+  return best_scale;
+}
+
+rtc::Optional<Fraction> FindOptimizedScaleLessThanOrEqual(
+    int input_num_pixels,
+    int target_num_pixels) {
   float best_distance = std::numeric_limits<float>::max();
-  Fraction best_scale = {0, 1};  // Default to 0 if nothing matches.
+  rtc::Optional<Fraction> best_scale;
   for (const auto& fraction : kScaleFractions) {
     const float scale =
         fraction.numerator / static_cast<float>(fraction.denominator);
@@ -58,7 +85,7 @@ Fraction FindScaleLessThanOrEqual(int input_num_pixels, int target_num_pixels) {
     }
     if (diff < best_distance) {
       best_distance = diff;
-      best_scale = fraction;
+      best_scale = rtc::Optional<Fraction>(fraction);
       if (best_distance == 0) {  // Found exact match.
         break;
       }
@@ -67,9 +94,9 @@ Fraction FindScaleLessThanOrEqual(int input_num_pixels, int target_num_pixels) {
   return best_scale;
 }
 
-Fraction FindScaleLargerThan(int input_num_pixels,
-                             int target_num_pixels,
-                             int* resulting_number_of_pixels) {
+Fraction FindOptimizedScaleLargerThan(int input_num_pixels,
+                                      int target_num_pixels,
+                                      int* resulting_number_of_pixels) {
   float best_distance = std::numeric_limits<float>::max();
   Fraction best_scale = {1, 1};  // Default to unscaled if nothing matches.
   // Default to input number of pixels.
@@ -93,35 +120,47 @@ Fraction FindScaleLargerThan(int input_num_pixels,
   return best_scale;
 }
 
-Fraction FindScale(int input_num_pixels,
-                   int max_pixel_count_step_up,
-                   int max_pixel_count) {
+rtc::Optional<Fraction> FindOptimizedScale(int input_num_pixels,
+                                           int max_pixel_count_step_up,
+                                           int max_pixel_count) {
   // Try scale just above |max_pixel_count_step_up_|.
   if (max_pixel_count_step_up > 0) {
     int resulting_pixel_count;
-    const Fraction scale = FindScaleLargerThan(
+    const Fraction scale = FindOptimizedScaleLargerThan(
         input_num_pixels, max_pixel_count_step_up, &resulting_pixel_count);
     if (resulting_pixel_count <= max_pixel_count)
-      return scale;
+      return rtc::Optional<Fraction>(scale);
   }
   // Return largest scale below |max_pixel_count|.
-  return FindScaleLessThanOrEqual(input_num_pixels, max_pixel_count);
+  return FindOptimizedScaleLessThanOrEqual(input_num_pixels, max_pixel_count);
 }
 
+Fraction FindScale(int input_num_pixels,
+                   int max_pixel_count_step_up,
+                   int max_pixel_count) {
+  const rtc::Optional<Fraction> optimized_scale = FindOptimizedScale(
+      input_num_pixels, max_pixel_count_step_up, max_pixel_count);
+  if (optimized_scale)
+    return *optimized_scale;
+  return FindScaleLessThanOrEqual(input_num_pixels, max_pixel_count);
+}
 }  // namespace
 
 namespace cricket {
 
-VideoAdapter::VideoAdapter()
+VideoAdapter::VideoAdapter(int required_resolution_alignment)
     : frames_in_(0),
       frames_out_(0),
       frames_scaled_(0),
       adaption_changes_(0),
       previous_width_(0),
       previous_height_(0),
+      required_resolution_alignment_(required_resolution_alignment),
       resolution_request_max_pixel_count_(std::numeric_limits<int>::max()),
       resolution_request_max_pixel_count_step_up_(0) {}
 
+VideoAdapter::VideoAdapter() : VideoAdapter(1) {}
+
 VideoAdapter::~VideoAdapter() {}
 
 bool VideoAdapter::KeepFrame(int64_t in_timestamp_ns) {
@@ -211,22 +250,26 @@ bool VideoAdapter::AdaptFrameResolution(int in_width,
     *cropped_height =
         std::min(in_height, static_cast<int>(in_width / requested_aspect));
   }
-
-  // Find best scale factor.
   const Fraction scale =
       FindScale(*cropped_width * *cropped_height,
                 resolution_request_max_pixel_count_step_up_, max_pixel_count);
-
   // Adjust cropping slightly to get even integer output size and a perfect
-  // scale factor.
-  *cropped_width = roundUp(*cropped_width, scale.denominator, in_width);
-  *cropped_height = roundUp(*cropped_height, scale.denominator, in_height);
+  // scale factor. Make sure the resulting dimensions are aligned correctly
+  // to be nice to hardware encoders.
+  *cropped_width =
+      roundUp(*cropped_width,
+              scale.denominator * required_resolution_alignment_, in_width);
+  *cropped_height =
+      roundUp(*cropped_height,
+              scale.denominator * required_resolution_alignment_, in_height);
   RTC_DCHECK_EQ(0, *cropped_width % scale.denominator);
   RTC_DCHECK_EQ(0, *cropped_height % scale.denominator);
 
   // Calculate final output size.
   *out_width = *cropped_width / scale.denominator * scale.numerator;
   *out_height = *cropped_height / scale.denominator * scale.numerator;
+  RTC_DCHECK_EQ(0, *out_height % required_resolution_alignment_);
+  RTC_DCHECK_EQ(0, *out_height % required_resolution_alignment_);
 
   ++frames_out_;
   if (scale.numerator != scale.denominator)

webrtc/media/base/videoadapter.h

@@ -25,6 +25,7 @@ namespace cricket {
 class VideoAdapter {
  public:
   VideoAdapter();
+  VideoAdapter(int required_resolution_alignment);
   virtual ~VideoAdapter();
 
   // Return the adapted resolution and cropping parameters given the
@@ -63,6 +64,8 @@ class VideoAdapter {
   int adaption_changes_;  // Number of changes in scale factor.
   int previous_width_;    // Previous adapter output width.
   int previous_height_;   // Previous adapter output height.
+  // Resolution must be divisible by this factor.
+  const int required_resolution_alignment_;
   // The target timestamp for the next frame based on requested format.
   rtc::Optional<int64_t> next_frame_timestamp_ns_ GUARDED_BY(critical_section_);
 

webrtc/sdk/android/src/jni/androidvideotracksource.cc

@@ -12,13 +12,19 @@
 
 #include <utility>
 
+namespace {
+// MediaCodec wants resolution to be divisible by 2.
+const int kRequiredResolutionAlignment = 2;
+}
+
 namespace webrtc {
 
 AndroidVideoTrackSource::AndroidVideoTrackSource(rtc::Thread* signaling_thread,
                                                  JNIEnv* jni,
                                                  jobject j_egl_context,
                                                  bool is_screencast)
-    : signaling_thread_(signaling_thread),
+    : AdaptedVideoTrackSource(kRequiredResolutionAlignment),
+      signaling_thread_(signaling_thread),
       surface_texture_helper_(webrtc_jni::SurfaceTextureHelper::create(
           jni,
           "Camera SurfaceTextureHelper",