Add script for plotting statistics from webrtc integration test logs.
Add tests (plot_videoprocessor_integrationtest.cc) to be used to plot stats from (not yet used). Move VideoProcessorIntegrationTest fixture to separate file. To be used by plot_videoprocessor_integrationtest.cc. BUG=webrtc:6634 Review-Url: https://codereview.webrtc.org/2643853002 Cr-Commit-Position: refs/heads/master@{#16528}
This commit is contained in:
asapersson
@ -0,0 +1,110 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h"
|
||||
|
||||
namespace webrtc {
|
||||
namespace test {
|
||||
namespace {
|
||||
// Codec settings.
|
||||
const int kBitrates[] = {30, 50, 100, 200, 300, 500, 1000};
|
||||
const int kFps[] = {30};
|
||||
const bool kErrorConcealmentOn = false;
|
||||
const bool kDenoisingOn = false;
|
||||
const bool kFrameDropperOn = true;
|
||||
const bool kSpatialResizeOn = false;
|
||||
const VideoCodecType kVideoCodecType = kVideoCodecVP8;
|
||||
|
||||
// Packet loss probability [0.0, 1.0].
|
||||
const float kPacketLoss = 0.0f;
|
||||
|
||||
const bool kVerboseLogging = true;
|
||||
} // namespace
|
||||
|
||||
// Tests for plotting statistics from logs.
|
||||
class PlotVideoProcessorIntegrationTest
|
||||
: public VideoProcessorIntegrationTest,
|
||||
public ::testing::WithParamInterface<::testing::tuple<int, int>> {
|
||||
protected:
|
||||
PlotVideoProcessorIntegrationTest()
|
||||
: bitrate_(::testing::get<0>(GetParam())),
|
||||
framerate_(::testing::get<1>(GetParam())) {}
|
||||
|
||||
virtual ~PlotVideoProcessorIntegrationTest() {}
|
||||
|
||||
void RunTest(int bitrate,
|
||||
int framerate,
|
||||
int width,
|
||||
int height,
|
||||
const std::string& filename) {
|
||||
// Bitrate and frame rate profile.
|
||||
RateProfile rate_profile;
|
||||
SetRateProfilePars(&rate_profile,
|
||||
0, // update_index
|
||||
bitrate, framerate,
|
||||
0); // frame_index_rate_update
|
||||
rate_profile.frame_index_rate_update[1] = kNbrFramesLong + 1;
|
||||
rate_profile.num_frames = kNbrFramesLong;
|
||||
// Codec/network settings.
|
||||
CodecConfigPars process_settings;
|
||||
SetCodecParameters(&process_settings, kVideoCodecType, kPacketLoss,
|
||||
-1, // key_frame_interval
|
||||
1, // num_temporal_layers
|
||||
kErrorConcealmentOn, kDenoisingOn, kFrameDropperOn,
|
||||
kSpatialResizeOn, width, height, filename,
|
||||
kVerboseLogging);
|
||||
// Metrics for expected quality (PSNR avg, PSNR min, SSIM avg, SSIM min).
|
||||
QualityMetrics quality_metrics;
|
||||
SetQualityMetrics(&quality_metrics, 15.0, 10.0, 0.2, 0.1);
|
||||
// Metrics for rate control.
|
||||
RateControlMetrics rc_metrics[1];
|
||||
SetRateControlMetrics(rc_metrics,
|
||||
0, // update_index
|
||||
300, // max_num_dropped_frames,
|
||||
400, // max_key_frame_size_mismatch
|
||||
200, // max_delta_frame_size_mismatch
|
||||
100, // max_encoding_rate_mismatch
|
||||
300, // max_time_hit_target
|
||||
0, // num_spatial_resizes
|
||||
1); // num_key_frames
|
||||
ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings,
|
||||
rc_metrics);
|
||||
}
|
||||
const int bitrate_;
|
||||
const int framerate_;
|
||||
};
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(CodecSettings,
|
||||
PlotVideoProcessorIntegrationTest,
|
||||
::testing::Combine(::testing::ValuesIn(kBitrates),
|
||||
::testing::ValuesIn(kFps)));
|
||||
|
||||
TEST_P(PlotVideoProcessorIntegrationTest, ProcessSQCif) {
|
||||
RunTest(bitrate_, framerate_, 128, 96, "foreman_128x96");
|
||||
}
|
||||
|
||||
TEST_P(PlotVideoProcessorIntegrationTest, ProcessQQVga) {
|
||||
RunTest(bitrate_, framerate_, 160, 120, "foreman_160x120");
|
||||
}
|
||||
|
||||
TEST_P(PlotVideoProcessorIntegrationTest, ProcessQCif) {
|
||||
RunTest(bitrate_, framerate_, 176, 144, "foreman_176x144");
|
||||
}
|
||||
|
||||
TEST_P(PlotVideoProcessorIntegrationTest, ProcessQVga) {
|
||||
RunTest(bitrate_, framerate_, 320, 240, "foreman_320x240");
|
||||
}
|
||||
|
||||
TEST_P(PlotVideoProcessorIntegrationTest, ProcessCif) {
|
||||
RunTest(bitrate_, framerate_, 352, 288, "foreman_cif");
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
} // namespace webrtc
|
||||
428
webrtc/modules/video_coding/codecs/test/plot_webrtc_test_logs.py
Executable file
428
webrtc/modules/video_coding/codecs/test/plot_webrtc_test_logs.py
Executable file
@ -0,0 +1,428 @@
|
||||
# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
|
||||
#
|
||||
# Use of this source code is governed by a BSD-style license
|
||||
# that can be found in the LICENSE file in the root of the source
|
||||
# tree. An additional intellectual property rights grant can be found
|
||||
# in the file PATENTS. All contributing project authors may
|
||||
# be found in the AUTHORS file in the root of the source tree.
|
||||
|
||||
"""Plots statistics from WebRTC integration test logs.
|
||||
|
||||
Usage: $ python plot_webrtc_test_logs.py filename.txt
|
||||
"""
|
||||
|
||||
import numpy
|
||||
import sys
|
||||
import re
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
# Log events.
|
||||
EVENT_START = 'RUN ] CodecSettings/PlotVideoProcessorIntegrationTest.'
|
||||
EVENT_END = 'OK ] CodecSettings/PlotVideoProcessorIntegrationTest.'
|
||||
|
||||
# Metrics to plot, tuple: (name to parse in file, label to use when plotting).
|
||||
BITRATE = ('Target Bitrate', 'bitrate (kbps)')
|
||||
WIDTH = ('Width', 'width')
|
||||
HEIGHT = ('Height', 'height')
|
||||
FILENAME = ('Filename', 'clip')
|
||||
CODEC_TYPE = ('Codec type', 'Codec')
|
||||
ENCODER_IMPLEMENTATION_NAME = ('Encoder implementation name', 'enc name')
|
||||
DECODER_IMPLEMENTATION_NAME = ('Decoder implementation name', 'dec name')
|
||||
NUM_FRAMES = ('Total # of frames', 'num frames')
|
||||
CORES = ('#CPU cores used', 'cores')
|
||||
DENOISING = ('Denoising', 'denoising')
|
||||
RESILIENCE = ('Resilience', 'resilience')
|
||||
ERROR_CONCEALMENT = ('Error concealment', 'error concealment')
|
||||
PSNR = ('PSNR avg', 'PSNR (dB)')
|
||||
SSIM = ('SSIM avg', 'SSIM')
|
||||
ENC_BITRATE = ('Encoding bitrate', 'encoded bitrate (kbps)')
|
||||
FRAMERATE = ('Frame rate', 'fps')
|
||||
NUM_DROPPED_FRAMES = ('Number of dropped frames', 'num dropped frames')
|
||||
NUM_FRAMES_TO_TARGET = ('Number of frames to approach target rate',
|
||||
'frames to reach target rate')
|
||||
ENCODE_TIME = ('Encoding time', 'encode time (us)')
|
||||
ENCODE_TIME_AVG = ('Encoding time', 'encode time (us) avg')
|
||||
DECODE_TIME = ('Decoding time', 'decode time (us)')
|
||||
DECODE_TIME_AVG = ('Decoding time', 'decode time (us) avg')
|
||||
FRAME_SIZE = ('Frame sizes', 'frame size (bytes)')
|
||||
FRAME_SIZE_AVG = ('Frame sizes', 'frame size (bytes) avg')
|
||||
AVG_KEY_FRAME_SIZE = ('Average key frame size', 'avg key frame size (bytes)')
|
||||
AVG_NON_KEY_FRAME_SIZE = ('Average non-key frame size',
|
||||
'avg non-key frame size (bytes)')
|
||||
|
||||
# Settings.
|
||||
SETTINGS = [
|
||||
WIDTH,
|
||||
HEIGHT,
|
||||
FILENAME,
|
||||
CODEC_TYPE,
|
||||
NUM_FRAMES,
|
||||
ENCODER_IMPLEMENTATION_NAME,
|
||||
DECODER_IMPLEMENTATION_NAME,
|
||||
ENCODE_TIME,
|
||||
DECODE_TIME,
|
||||
FRAME_SIZE,
|
||||
]
|
||||
|
||||
# Settings, options for x-axis.
|
||||
X_SETTINGS = [
|
||||
CORES,
|
||||
FRAMERATE,
|
||||
DENOISING,
|
||||
RESILIENCE,
|
||||
ERROR_CONCEALMENT,
|
||||
BITRATE, # TODO(asapersson): Needs to be last.
|
||||
]
|
||||
|
||||
# Results.
|
||||
RESULTS = [
|
||||
PSNR,
|
||||
SSIM,
|
||||
ENC_BITRATE,
|
||||
NUM_DROPPED_FRAMES,
|
||||
NUM_FRAMES_TO_TARGET,
|
||||
ENCODE_TIME_AVG,
|
||||
DECODE_TIME_AVG,
|
||||
AVG_KEY_FRAME_SIZE,
|
||||
AVG_NON_KEY_FRAME_SIZE,
|
||||
]
|
||||
|
||||
METRICS_TO_PARSE = SETTINGS + X_SETTINGS + RESULTS
|
||||
|
||||
Y_METRICS = [res[1] for res in RESULTS]
|
||||
|
||||
# Parameters for plotting.
|
||||
FIG_SIZE_SCALE_FACTOR_X = 2
|
||||
FIG_SIZE_SCALE_FACTOR_Y = 2.8
|
||||
GRID_COLOR = [0.45, 0.45, 0.45]
|
||||
|
||||
|
||||
def ParseSetting(filename, setting):
|
||||
"""Parses setting from file.
|
||||
|
||||
Args:
|
||||
filename: The name of the file.
|
||||
setting: Name of setting to parse (e.g. width).
|
||||
|
||||
Returns:
|
||||
A list holding parsed settings, e.g. ['width: 128.0', 'width: 160.0'] """
|
||||
|
||||
settings = []
|
||||
|
||||
f = open(filename)
|
||||
while True:
|
||||
line = f.readline()
|
||||
if not line:
|
||||
break
|
||||
if re.search(r'%s' % EVENT_START, line):
|
||||
# Parse event.
|
||||
parsed = {}
|
||||
while True:
|
||||
line = f.readline()
|
||||
if not line:
|
||||
break
|
||||
if re.search(r'%s' % EVENT_END, line):
|
||||
# Add parsed setting to list.
|
||||
if setting in parsed:
|
||||
s = setting + ': ' + str(parsed[setting])
|
||||
if s not in settings:
|
||||
settings.append(s)
|
||||
break
|
||||
|
||||
TryFindMetric(parsed, line, f)
|
||||
|
||||
f.close()
|
||||
return settings
|
||||
|
||||
|
||||
def ParseMetrics(filename, setting1, setting2):
|
||||
"""Parses metrics from file.
|
||||
|
||||
Args:
|
||||
filename: The name of the file.
|
||||
setting1: First setting for sorting metrics (e.g. width).
|
||||
setting2: Second setting for sorting metrics (e.g. cores).
|
||||
|
||||
Returns:
|
||||
A dictionary holding parsed metrics.
|
||||
|
||||
For example:
|
||||
metrics[key1][key2][measurement]
|
||||
|
||||
metrics = {
|
||||
"width: 352": {
|
||||
"cores: 1.0": {
|
||||
"encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642],
|
||||
"PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
|
||||
"bitrate (kbps)": [50, 100, 300, 500, 1000]
|
||||
},
|
||||
"cores: 2.0": {
|
||||
"encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642],
|
||||
"PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
|
||||
"bitrate (kbps)": [50, 100, 300, 500, 1000]
|
||||
},
|
||||
},
|
||||
"width: 176": {
|
||||
"cores: 1.0": {
|
||||
"encode time (us)": [0.857897, 0.91608, 0.959173, 0.971116, 0.980961],
|
||||
"PSNR (dB)": [30.243646, 33.375592, 37.574387, 39.42184, 41.437897],
|
||||
"bitrate (kbps)": [50, 100, 300, 500, 1000]
|
||||
},
|
||||
}
|
||||
} """
|
||||
|
||||
metrics = {}
|
||||
|
||||
# Parse events.
|
||||
f = open(filename)
|
||||
while True:
|
||||
line = f.readline()
|
||||
if not line:
|
||||
break
|
||||
if re.search(r'%s' % EVENT_START, line):
|
||||
# Parse event.
|
||||
parsed = {}
|
||||
while True:
|
||||
line = f.readline()
|
||||
if not line:
|
||||
break
|
||||
if re.search(r'%s' % EVENT_END, line):
|
||||
# Add parsed values to metrics.
|
||||
key1 = setting1 + ': ' + str(parsed[setting1])
|
||||
key2 = setting2 + ': ' + str(parsed[setting2])
|
||||
if key1 not in metrics:
|
||||
metrics[key1] = {}
|
||||
if key2 not in metrics[key1]:
|
||||
metrics[key1][key2] = {}
|
||||
|
||||
for label in parsed:
|
||||
if label not in metrics[key1][key2]:
|
||||
metrics[key1][key2][label] = []
|
||||
metrics[key1][key2][label].append(parsed[label])
|
||||
|
||||
break
|
||||
|
||||
TryFindMetric(parsed, line, f)
|
||||
|
||||
f.close()
|
||||
return metrics
|
||||
|
||||
|
||||
def TryFindMetric(parsed, line, f):
|
||||
for metric in METRICS_TO_PARSE:
|
||||
name = metric[0]
|
||||
label = metric[1]
|
||||
if re.search(r'%s' % name, line):
|
||||
found, value = GetMetric(name, line)
|
||||
if not found:
|
||||
# TODO(asapersson): Change format.
|
||||
# Try find min, max, average stats.
|
||||
found, minimum = GetMetric("Min", f.readline())
|
||||
if not found:
|
||||
return
|
||||
found, maximum = GetMetric("Max", f.readline())
|
||||
if not found:
|
||||
return
|
||||
found, average = GetMetric("Average", f.readline())
|
||||
if not found:
|
||||
return
|
||||
|
||||
parsed[label + ' min'] = minimum
|
||||
parsed[label + ' max'] = maximum
|
||||
parsed[label + ' avg'] = average
|
||||
|
||||
parsed[label] = value
|
||||
return
|
||||
|
||||
|
||||
def GetMetric(name, string):
|
||||
# Float (e.g. bitrate = 98.8253).
|
||||
pattern = r'%s\s*[:=]\s*([+-]?\d+\.*\d*)' % name
|
||||
m = re.search(r'%s' % pattern, string)
|
||||
if m is not None:
|
||||
return StringToFloat(m.group(1))
|
||||
|
||||
# Alphanumeric characters (e.g. codec type : VP8).
|
||||
pattern = r'%s\s*[:=]\s*(\w+)' % name
|
||||
m = re.search(r'%s' % pattern, string)
|
||||
if m is not None:
|
||||
return True, m.group(1)
|
||||
|
||||
return False, -1
|
||||
|
||||
|
||||
def StringToFloat(value):
|
||||
try:
|
||||
value = float(value)
|
||||
except ValueError:
|
||||
print "Not a float, skipped %s" % value
|
||||
return False, -1
|
||||
|
||||
return True, value
|
||||
|
||||
|
||||
def Plot(y_metric, x_metric, metrics):
|
||||
"""Plots y_metric vs x_metric per key in metrics.
|
||||
|
||||
For example:
|
||||
y_metric = 'PSNR (dB)'
|
||||
x_metric = 'bitrate (kbps)'
|
||||
metrics = {
|
||||
"cores: 1.0": {
|
||||
"PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
|
||||
"bitrate (kbps)": [50, 100, 300, 500, 1000]
|
||||
},
|
||||
"cores: 2.0": {
|
||||
"PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
|
||||
"bitrate (kbps)": [50, 100, 300, 500, 1000]
|
||||
},
|
||||
}
|
||||
"""
|
||||
for key in metrics:
|
||||
data = metrics[key]
|
||||
if y_metric not in data:
|
||||
print "Failed to find metric: %s" % y_metric
|
||||
continue
|
||||
|
||||
y = numpy.array(data[y_metric])
|
||||
x = numpy.array(data[x_metric])
|
||||
if len(y) != len(x):
|
||||
print "Length mismatch for %s, %s" % (y, x)
|
||||
continue
|
||||
|
||||
label = y_metric + ' - ' + str(key)
|
||||
|
||||
plt.plot(x, y, label=label, linewidth=1.5, marker='o', markersize=5,
|
||||
markeredgewidth=0.0)
|
||||
|
||||
|
||||
def PlotFigure(settings, y_metrics, x_metric, metrics, title):
|
||||
"""Plots metrics in y_metrics list. One figure is plotted and each entry
|
||||
in the list is plotted in a subplot (and sorted per settings).
|
||||
|
||||
For example:
|
||||
settings = ['width: 128.0', 'width: 160.0']. Sort subplot per setting.
|
||||
y_metrics = ['PSNR (dB)', 'PSNR (dB)']. Metric to plot per subplot.
|
||||
x_metric = 'bitrate (kbps)'
|
||||
|
||||
"""
|
||||
|
||||
plt.figure()
|
||||
plt.suptitle(title, fontsize='small', fontweight='bold')
|
||||
rows = len(settings)
|
||||
cols = 1
|
||||
pos = 1
|
||||
while pos <= rows:
|
||||
plt.rc('grid', color=GRID_COLOR)
|
||||
ax = plt.subplot(rows, cols, pos)
|
||||
plt.grid()
|
||||
plt.setp(ax.get_xticklabels(), visible=(pos == rows), fontsize='small')
|
||||
plt.setp(ax.get_yticklabels(), fontsize='small')
|
||||
setting = settings[pos - 1]
|
||||
Plot(y_metrics[pos - 1], x_metric, metrics[setting])
|
||||
plt.title(setting, fontsize='x-small')
|
||||
plt.legend(fontsize='xx-small')
|
||||
pos += 1
|
||||
|
||||
plt.xlabel(x_metric, fontsize='small')
|
||||
plt.subplots_adjust(left=0.04, right=0.98, bottom=0.04, top=0.96, hspace=0.1)
|
||||
|
||||
|
||||
def GetTitle(filename):
|
||||
title = ''
|
||||
codec_types = ParseSetting(filename, CODEC_TYPE[1])
|
||||
for i in range(0, len(codec_types)):
|
||||
title += codec_types[i] + ', '
|
||||
|
||||
framerate = ParseSetting(filename, FRAMERATE[1])
|
||||
for i in range(0, len(framerate)):
|
||||
title += framerate[i].split('.')[0] + ', '
|
||||
|
||||
enc_names = ParseSetting(filename, ENCODER_IMPLEMENTATION_NAME[1])
|
||||
for i in range(0, len(enc_names)):
|
||||
title += enc_names[i] + ', '
|
||||
|
||||
dec_names = ParseSetting(filename, DECODER_IMPLEMENTATION_NAME[1])
|
||||
for i in range(0, len(dec_names)):
|
||||
title += dec_names[i] + ', '
|
||||
|
||||
filenames = ParseSetting(filename, FILENAME[1])
|
||||
title += filenames[0].split('_')[0]
|
||||
|
||||
num_frames = ParseSetting(filename, NUM_FRAMES[1])
|
||||
for i in range(0, len(num_frames)):
|
||||
title += ' (' + num_frames[i].split('.')[0] + ')'
|
||||
|
||||
return title
|
||||
|
||||
|
||||
def ToString(input_list):
|
||||
return ToStringWithoutMetric(input_list, ('', ''))
|
||||
|
||||
|
||||
def ToStringWithoutMetric(input_list, metric):
|
||||
i = 1
|
||||
output_str = ""
|
||||
for m in input_list:
|
||||
if m != metric:
|
||||
output_str = output_str + ("%s. %s\n" % (i, m[1]))
|
||||
i += 1
|
||||
return output_str
|
||||
|
||||
|
||||
def GetIdx(text_list):
|
||||
return int(raw_input(text_list)) - 1
|
||||
|
||||
|
||||
def main():
|
||||
filename = sys.argv[1]
|
||||
|
||||
# Setup.
|
||||
idx_metric = GetIdx("Choose metric:\n0. All\n%s" % ToString(RESULTS))
|
||||
if idx_metric == -1:
|
||||
# Plot all metrics. One subplot for each metric.
|
||||
# Per subplot: metric vs bitrate (per resolution).
|
||||
cores = ParseSetting(filename, CORES[1])
|
||||
setting1 = CORES[1]
|
||||
setting2 = WIDTH[1]
|
||||
sub_keys = [cores[0]] * len(Y_METRICS)
|
||||
y_metrics = Y_METRICS
|
||||
x_metric = BITRATE[1]
|
||||
else:
|
||||
resolutions = ParseSetting(filename, WIDTH[1])
|
||||
idx = GetIdx("Select metric for x-axis:\n%s" % ToString(X_SETTINGS))
|
||||
if X_SETTINGS[idx] == BITRATE:
|
||||
idx = GetIdx("Plot per:\n%s" % ToStringWithoutMetric(X_SETTINGS, BITRATE))
|
||||
idx_setting = METRICS_TO_PARSE.index(X_SETTINGS[idx])
|
||||
# Plot one metric. One subplot for each resolution.
|
||||
# Per subplot: metric vs bitrate (per setting).
|
||||
setting1 = WIDTH[1]
|
||||
setting2 = METRICS_TO_PARSE[idx_setting][1]
|
||||
sub_keys = resolutions
|
||||
y_metrics = [RESULTS[idx_metric][1]] * len(sub_keys)
|
||||
x_metric = BITRATE[1]
|
||||
else:
|
||||
# Plot one metric. One subplot for each resolution.
|
||||
# Per subplot: metric vs setting (per bitrate).
|
||||
setting1 = WIDTH[1]
|
||||
setting2 = BITRATE[1]
|
||||
sub_keys = resolutions
|
||||
y_metrics = [RESULTS[idx_metric][1]] * len(sub_keys)
|
||||
x_metric = X_SETTINGS[idx][1]
|
||||
|
||||
metrics = ParseMetrics(filename, setting1, setting2)
|
||||
|
||||
# Stretch fig size.
|
||||
figsize = plt.rcParams["figure.figsize"]
|
||||
figsize[0] *= FIG_SIZE_SCALE_FACTOR_X
|
||||
figsize[1] *= FIG_SIZE_SCALE_FACTOR_Y
|
||||
plt.rcParams["figure.figsize"] = figsize
|
||||
|
||||
PlotFigure(sub_keys, y_metrics, x_metric, metrics, GetTitle(filename))
|
||||
|
||||
plt.show()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -8,578 +8,10 @@
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <math.h>
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "webrtc/modules/video_coding/codecs/h264/include/h264.h"
|
||||
#include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h"
|
||||
#include "webrtc/modules/video_coding/codecs/test/videoprocessor.h"
|
||||
#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
|
||||
#include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h"
|
||||
#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
|
||||
#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h"
|
||||
#include "webrtc/modules/video_coding/include/video_codec_interface.h"
|
||||
#include "webrtc/modules/video_coding/include/video_coding.h"
|
||||
#include "webrtc/test/gtest.h"
|
||||
#include "webrtc/test/testsupport/fileutils.h"
|
||||
#include "webrtc/test/testsupport/frame_reader.h"
|
||||
#include "webrtc/test/testsupport/frame_writer.h"
|
||||
#include "webrtc/test/testsupport/metrics/video_metrics.h"
|
||||
#include "webrtc/test/testsupport/packet_reader.h"
|
||||
#include "webrtc/typedefs.h"
|
||||
#include "webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h"
|
||||
|
||||
namespace webrtc {
|
||||
namespace {
|
||||
// Maximum number of rate updates (i.e., calls to encoder to change bitrate
|
||||
// and/or frame rate) for the current tests.
|
||||
const int kMaxNumRateUpdates = 3;
|
||||
|
||||
const int kPercTargetvsActualMismatch = 20;
|
||||
const int kBaseKeyFrameInterval = 3000;
|
||||
|
||||
// Codec and network settings.
|
||||
struct CodecConfigPars {
|
||||
VideoCodecType codec_type;
|
||||
float packet_loss;
|
||||
int num_temporal_layers;
|
||||
int key_frame_interval;
|
||||
bool error_concealment_on;
|
||||
bool denoising_on;
|
||||
bool frame_dropper_on;
|
||||
bool spatial_resize_on;
|
||||
};
|
||||
|
||||
// Quality metrics.
|
||||
struct QualityMetrics {
|
||||
double minimum_avg_psnr;
|
||||
double minimum_min_psnr;
|
||||
double minimum_avg_ssim;
|
||||
double minimum_min_ssim;
|
||||
};
|
||||
|
||||
// The sequence of bitrate and frame rate changes for the encoder, the frame
|
||||
// number where the changes are made, and the total number of frames for the
|
||||
// test.
|
||||
struct RateProfile {
|
||||
int target_bit_rate[kMaxNumRateUpdates];
|
||||
int input_frame_rate[kMaxNumRateUpdates];
|
||||
int frame_index_rate_update[kMaxNumRateUpdates + 1];
|
||||
int num_frames;
|
||||
};
|
||||
|
||||
// Metrics for the rate control. The rate mismatch metrics are defined as
|
||||
// percentages.|max_time_hit_target| is defined as number of frames, after a
|
||||
// rate update is made to the encoder, for the encoder to reach within
|
||||
// |kPercTargetvsActualMismatch| of new target rate. The metrics are defined for
|
||||
// each rate update sequence.
|
||||
struct RateControlMetrics {
|
||||
int max_num_dropped_frames;
|
||||
int max_key_frame_size_mismatch;
|
||||
int max_delta_frame_size_mismatch;
|
||||
int max_encoding_rate_mismatch;
|
||||
int max_time_hit_target;
|
||||
int num_spatial_resizes;
|
||||
int num_key_frames;
|
||||
};
|
||||
|
||||
// Sequence used is foreman (CIF): may be better to use VGA for resize test.
|
||||
const int kCIFWidth = 352;
|
||||
const int kCIFHeight = 288;
|
||||
#if !defined(WEBRTC_IOS)
|
||||
const int kNbrFramesShort = 100; // Some tests are run for shorter sequence.
|
||||
#endif
|
||||
const int kNbrFramesLong = 299;
|
||||
|
||||
// Parameters from VP8 wrapper, which control target size of key frames.
|
||||
const float kInitialBufferSize = 0.5f;
|
||||
const float kOptimalBufferSize = 0.6f;
|
||||
const float kScaleKeyFrameSize = 0.5f;
|
||||
|
||||
void SetRateProfilePars(RateProfile* rate_profile,
|
||||
int update_index,
|
||||
int bit_rate,
|
||||
int frame_rate,
|
||||
int frame_index_rate_update) {
|
||||
rate_profile->target_bit_rate[update_index] = bit_rate;
|
||||
rate_profile->input_frame_rate[update_index] = frame_rate;
|
||||
rate_profile->frame_index_rate_update[update_index] = frame_index_rate_update;
|
||||
}
|
||||
|
||||
void SetCodecParameters(CodecConfigPars* process_settings,
|
||||
VideoCodecType codec_type,
|
||||
float packet_loss,
|
||||
int key_frame_interval,
|
||||
int num_temporal_layers,
|
||||
bool error_concealment_on,
|
||||
bool denoising_on,
|
||||
bool frame_dropper_on,
|
||||
bool spatial_resize_on) {
|
||||
process_settings->codec_type = codec_type;
|
||||
process_settings->packet_loss = packet_loss;
|
||||
process_settings->key_frame_interval = key_frame_interval;
|
||||
process_settings->num_temporal_layers = num_temporal_layers,
|
||||
process_settings->error_concealment_on = error_concealment_on;
|
||||
process_settings->denoising_on = denoising_on;
|
||||
process_settings->frame_dropper_on = frame_dropper_on;
|
||||
process_settings->spatial_resize_on = spatial_resize_on;
|
||||
}
|
||||
|
||||
void SetQualityMetrics(QualityMetrics* quality_metrics,
|
||||
double minimum_avg_psnr,
|
||||
double minimum_min_psnr,
|
||||
double minimum_avg_ssim,
|
||||
double minimum_min_ssim) {
|
||||
quality_metrics->minimum_avg_psnr = minimum_avg_psnr;
|
||||
quality_metrics->minimum_min_psnr = minimum_min_psnr;
|
||||
quality_metrics->minimum_avg_ssim = minimum_avg_ssim;
|
||||
quality_metrics->minimum_min_ssim = minimum_min_ssim;
|
||||
}
|
||||
|
||||
void SetRateControlMetrics(RateControlMetrics* rc_metrics,
|
||||
int update_index,
|
||||
int max_num_dropped_frames,
|
||||
int max_key_frame_size_mismatch,
|
||||
int max_delta_frame_size_mismatch,
|
||||
int max_encoding_rate_mismatch,
|
||||
int max_time_hit_target,
|
||||
int num_spatial_resizes,
|
||||
int num_key_frames) {
|
||||
rc_metrics[update_index].max_num_dropped_frames = max_num_dropped_frames;
|
||||
rc_metrics[update_index].max_key_frame_size_mismatch =
|
||||
max_key_frame_size_mismatch;
|
||||
rc_metrics[update_index].max_delta_frame_size_mismatch =
|
||||
max_delta_frame_size_mismatch;
|
||||
rc_metrics[update_index].max_encoding_rate_mismatch =
|
||||
max_encoding_rate_mismatch;
|
||||
rc_metrics[update_index].max_time_hit_target = max_time_hit_target;
|
||||
rc_metrics[update_index].num_spatial_resizes = num_spatial_resizes;
|
||||
rc_metrics[update_index].num_key_frames = num_key_frames;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
// Integration test for video processor. Encodes+decodes a clip and
|
||||
// writes it to the output directory. After completion, quality metrics
|
||||
// (PSNR and SSIM) and rate control metrics are computed to verify that the
|
||||
// quality and encoder response is acceptable. The rate control tests allow us
|
||||
// to verify the behavior for changing bitrate, changing frame rate, frame
|
||||
// dropping/spatial resize, and temporal layers. The limits for the rate
|
||||
// control metrics are set to be fairly conservative, so failure should only
|
||||
// happen when some significant regression or breakdown occurs.
|
||||
class VideoProcessorIntegrationTest : public testing::Test {
|
||||
protected:
|
||||
std::unique_ptr<VideoEncoder> encoder_;
|
||||
std::unique_ptr<VideoDecoder> decoder_;
|
||||
std::unique_ptr<test::FrameReader> frame_reader_;
|
||||
std::unique_ptr<test::FrameWriter> frame_writer_;
|
||||
test::PacketReader packet_reader_;
|
||||
std::unique_ptr<test::PacketManipulator> packet_manipulator_;
|
||||
test::Stats stats_;
|
||||
test::TestConfig config_;
|
||||
VideoCodec codec_settings_;
|
||||
std::unique_ptr<test::VideoProcessor> processor_;
|
||||
TemporalLayersFactory tl_factory_;
|
||||
|
||||
// Quantities defined/updated for every encoder rate update.
|
||||
// Some quantities defined per temporal layer (at most 3 layers in this test).
|
||||
int num_frames_per_update_[3];
|
||||
float sum_frame_size_mismatch_[3];
|
||||
float sum_encoded_frame_size_[3];
|
||||
float encoding_bitrate_[3];
|
||||
float per_frame_bandwidth_[3];
|
||||
float bit_rate_layer_[3];
|
||||
float frame_rate_layer_[3];
|
||||
int num_frames_total_;
|
||||
float sum_encoded_frame_size_total_;
|
||||
float encoding_bitrate_total_;
|
||||
float perc_encoding_rate_mismatch_;
|
||||
int num_frames_to_hit_target_;
|
||||
bool encoding_rate_within_target_;
|
||||
int bit_rate_;
|
||||
int frame_rate_;
|
||||
int layer_;
|
||||
float target_size_key_frame_initial_;
|
||||
float target_size_key_frame_;
|
||||
float sum_key_frame_size_mismatch_;
|
||||
int num_key_frames_;
|
||||
float start_bitrate_;
|
||||
|
||||
// Codec and network settings.
|
||||
VideoCodecType codec_type_;
|
||||
float packet_loss_;
|
||||
int num_temporal_layers_;
|
||||
int key_frame_interval_;
|
||||
bool error_concealment_on_;
|
||||
bool denoising_on_;
|
||||
bool frame_dropper_on_;
|
||||
bool spatial_resize_on_;
|
||||
|
||||
VideoProcessorIntegrationTest() {}
|
||||
virtual ~VideoProcessorIntegrationTest() {}
|
||||
|
||||
void SetUpCodecConfig() {
|
||||
if (codec_type_ == kVideoCodecH264) {
|
||||
encoder_.reset(H264Encoder::Create(cricket::VideoCodec("H264")));
|
||||
decoder_.reset(H264Decoder::Create());
|
||||
VideoCodingModule::Codec(kVideoCodecH264, &codec_settings_);
|
||||
} else if (codec_type_ == kVideoCodecVP8) {
|
||||
encoder_.reset(VP8Encoder::Create());
|
||||
decoder_.reset(VP8Decoder::Create());
|
||||
VideoCodingModule::Codec(kVideoCodecVP8, &codec_settings_);
|
||||
} else if (codec_type_ == kVideoCodecVP9) {
|
||||
encoder_.reset(VP9Encoder::Create());
|
||||
decoder_.reset(VP9Decoder::Create());
|
||||
VideoCodingModule::Codec(kVideoCodecVP9, &codec_settings_);
|
||||
}
|
||||
|
||||
// CIF is currently used for all tests below.
|
||||
// Setup the TestConfig struct for processing of a clip in CIF resolution.
|
||||
config_.input_filename = webrtc::test::ResourcePath("foreman_cif", "yuv");
|
||||
|
||||
// Generate an output filename in a safe way.
|
||||
config_.output_filename = webrtc::test::TempFilename(
|
||||
webrtc::test::OutputPath(), "videoprocessor_integrationtest");
|
||||
config_.frame_length_in_bytes =
|
||||
CalcBufferSize(kI420, kCIFWidth, kCIFHeight);
|
||||
config_.verbose = false;
|
||||
// Only allow encoder/decoder to use single core, for predictability.
|
||||
config_.use_single_core = true;
|
||||
// Key frame interval and packet loss are set for each test.
|
||||
config_.keyframe_interval = key_frame_interval_;
|
||||
config_.networking_config.packet_loss_probability = packet_loss_;
|
||||
|
||||
// Configure codec settings.
|
||||
config_.codec_settings = &codec_settings_;
|
||||
config_.codec_settings->startBitrate = start_bitrate_;
|
||||
config_.codec_settings->width = kCIFWidth;
|
||||
config_.codec_settings->height = kCIFHeight;
|
||||
|
||||
// These features may be set depending on the test.
|
||||
switch (config_.codec_settings->codecType) {
|
||||
case kVideoCodecH264:
|
||||
config_.codec_settings->H264()->frameDroppingOn = frame_dropper_on_;
|
||||
config_.codec_settings->H264()->keyFrameInterval =
|
||||
kBaseKeyFrameInterval;
|
||||
break;
|
||||
case kVideoCodecVP8:
|
||||
config_.codec_settings->VP8()->errorConcealmentOn =
|
||||
error_concealment_on_;
|
||||
config_.codec_settings->VP8()->denoisingOn = denoising_on_;
|
||||
config_.codec_settings->VP8()->numberOfTemporalLayers =
|
||||
num_temporal_layers_;
|
||||
config_.codec_settings->VP8()->frameDroppingOn = frame_dropper_on_;
|
||||
config_.codec_settings->VP8()->automaticResizeOn = spatial_resize_on_;
|
||||
config_.codec_settings->VP8()->keyFrameInterval = kBaseKeyFrameInterval;
|
||||
break;
|
||||
case kVideoCodecVP9:
|
||||
config_.codec_settings->VP9()->denoisingOn = denoising_on_;
|
||||
config_.codec_settings->VP9()->numberOfTemporalLayers =
|
||||
num_temporal_layers_;
|
||||
config_.codec_settings->VP9()->frameDroppingOn = frame_dropper_on_;
|
||||
config_.codec_settings->VP9()->automaticResizeOn = spatial_resize_on_;
|
||||
config_.codec_settings->VP9()->keyFrameInterval = kBaseKeyFrameInterval;
|
||||
break;
|
||||
default:
|
||||
assert(false);
|
||||
break;
|
||||
}
|
||||
frame_reader_.reset(new test::FrameReaderImpl(
|
||||
config_.input_filename, config_.codec_settings->width,
|
||||
config_.codec_settings->height));
|
||||
frame_writer_.reset(new test::FrameWriterImpl(
|
||||
config_.output_filename, config_.frame_length_in_bytes));
|
||||
ASSERT_TRUE(frame_reader_->Init());
|
||||
ASSERT_TRUE(frame_writer_->Init());
|
||||
|
||||
packet_manipulator_.reset(new test::PacketManipulatorImpl(
|
||||
&packet_reader_, config_.networking_config, config_.verbose));
|
||||
processor_.reset(new test::VideoProcessorImpl(
|
||||
encoder_.get(), decoder_.get(), frame_reader_.get(),
|
||||
frame_writer_.get(), packet_manipulator_.get(), config_, &stats_));
|
||||
ASSERT_TRUE(processor_->Init());
|
||||
}
|
||||
|
||||
// Reset quantities after each encoder update, update the target
|
||||
// per-frame bandwidth.
|
||||
void ResetRateControlMetrics(int num_frames) {
|
||||
for (int i = 0; i < num_temporal_layers_; i++) {
|
||||
num_frames_per_update_[i] = 0;
|
||||
sum_frame_size_mismatch_[i] = 0.0f;
|
||||
sum_encoded_frame_size_[i] = 0.0f;
|
||||
encoding_bitrate_[i] = 0.0f;
|
||||
// Update layer per-frame-bandwidth.
|
||||
per_frame_bandwidth_[i] = static_cast<float>(bit_rate_layer_[i]) /
|
||||
static_cast<float>(frame_rate_layer_[i]);
|
||||
}
|
||||
// Set maximum size of key frames, following setting in the VP8 wrapper.
|
||||
float max_key_size = kScaleKeyFrameSize * kOptimalBufferSize * frame_rate_;
|
||||
// We don't know exact target size of the key frames (except for first one),
|
||||
// but the minimum in libvpx is ~|3 * per_frame_bandwidth| and maximum is
|
||||
// set by |max_key_size_ * per_frame_bandwidth|. Take middle point/average
|
||||
// as reference for mismatch. Note key frames always correspond to base
|
||||
// layer frame in this test.
|
||||
target_size_key_frame_ = 0.5 * (3 + max_key_size) * per_frame_bandwidth_[0];
|
||||
num_frames_total_ = 0;
|
||||
sum_encoded_frame_size_total_ = 0.0f;
|
||||
encoding_bitrate_total_ = 0.0f;
|
||||
perc_encoding_rate_mismatch_ = 0.0f;
|
||||
num_frames_to_hit_target_ = num_frames;
|
||||
encoding_rate_within_target_ = false;
|
||||
sum_key_frame_size_mismatch_ = 0.0;
|
||||
num_key_frames_ = 0;
|
||||
}
|
||||
|
||||
// For every encoded frame, update the rate control metrics.
|
||||
void UpdateRateControlMetrics(int frame_num, FrameType frame_type) {
|
||||
float encoded_size_kbits = processor_->EncodedFrameSize() * 8.0f / 1000.0f;
|
||||
// Update layer data.
|
||||
// Update rate mismatch relative to per-frame bandwidth for delta frames.
|
||||
if (frame_type == kVideoFrameDelta) {
|
||||
// TODO(marpan): Should we count dropped (zero size) frames in mismatch?
|
||||
sum_frame_size_mismatch_[layer_] +=
|
||||
fabs(encoded_size_kbits - per_frame_bandwidth_[layer_]) /
|
||||
per_frame_bandwidth_[layer_];
|
||||
} else {
|
||||
float target_size = (frame_num == 1) ? target_size_key_frame_initial_
|
||||
: target_size_key_frame_;
|
||||
sum_key_frame_size_mismatch_ +=
|
||||
fabs(encoded_size_kbits - target_size) / target_size;
|
||||
num_key_frames_ += 1;
|
||||
}
|
||||
sum_encoded_frame_size_[layer_] += encoded_size_kbits;
|
||||
// Encoding bitrate per layer: from the start of the update/run to the
|
||||
// current frame.
|
||||
encoding_bitrate_[layer_] = sum_encoded_frame_size_[layer_] *
|
||||
frame_rate_layer_[layer_] /
|
||||
num_frames_per_update_[layer_];
|
||||
// Total encoding rate: from the start of the update/run to current frame.
|
||||
sum_encoded_frame_size_total_ += encoded_size_kbits;
|
||||
encoding_bitrate_total_ =
|
||||
sum_encoded_frame_size_total_ * frame_rate_ / num_frames_total_;
|
||||
perc_encoding_rate_mismatch_ =
|
||||
100 * fabs(encoding_bitrate_total_ - bit_rate_) / bit_rate_;
|
||||
if (perc_encoding_rate_mismatch_ < kPercTargetvsActualMismatch &&
|
||||
!encoding_rate_within_target_) {
|
||||
num_frames_to_hit_target_ = num_frames_total_;
|
||||
encoding_rate_within_target_ = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Verify expected behavior of rate control and print out data.
|
||||
void VerifyRateControl(int update_index,
|
||||
int max_key_frame_size_mismatch,
|
||||
int max_delta_frame_size_mismatch,
|
||||
int max_encoding_rate_mismatch,
|
||||
int max_time_hit_target,
|
||||
int max_num_dropped_frames,
|
||||
int num_spatial_resizes,
|
||||
int num_key_frames) {
|
||||
int num_dropped_frames = processor_->NumberDroppedFrames();
|
||||
int num_resize_actions = processor_->NumberSpatialResizes();
|
||||
printf(
|
||||
"For update #: %d,\n "
|
||||
" Target Bitrate: %d,\n"
|
||||
" Encoding bitrate: %f,\n"
|
||||
" Frame rate: %d \n",
|
||||
update_index, bit_rate_, encoding_bitrate_total_, frame_rate_);
|
||||
printf(
|
||||
" Number of frames to approach target rate: %d, \n"
|
||||
" Number of dropped frames: %d, \n"
|
||||
" Number of spatial resizes: %d, \n",
|
||||
num_frames_to_hit_target_, num_dropped_frames, num_resize_actions);
|
||||
EXPECT_LE(perc_encoding_rate_mismatch_, max_encoding_rate_mismatch);
|
||||
if (num_key_frames_ > 0) {
|
||||
int perc_key_frame_size_mismatch =
|
||||
100 * sum_key_frame_size_mismatch_ / num_key_frames_;
|
||||
printf(
|
||||
" Number of Key frames: %d \n"
|
||||
" Key frame rate mismatch: %d \n",
|
||||
num_key_frames_, perc_key_frame_size_mismatch);
|
||||
EXPECT_LE(perc_key_frame_size_mismatch, max_key_frame_size_mismatch);
|
||||
}
|
||||
printf("\n");
|
||||
printf("Rates statistics for Layer data \n");
|
||||
for (int i = 0; i < num_temporal_layers_; i++) {
|
||||
printf("Temporal layer #%d \n", i);
|
||||
int perc_frame_size_mismatch =
|
||||
100 * sum_frame_size_mismatch_[i] / num_frames_per_update_[i];
|
||||
int perc_encoding_rate_mismatch =
|
||||
100 * fabs(encoding_bitrate_[i] - bit_rate_layer_[i]) /
|
||||
bit_rate_layer_[i];
|
||||
printf(
|
||||
" Target Layer Bit rate: %f \n"
|
||||
" Layer frame rate: %f, \n"
|
||||
" Layer per frame bandwidth: %f, \n"
|
||||
" Layer Encoding bit rate: %f, \n"
|
||||
" Layer Percent frame size mismatch: %d, \n"
|
||||
" Layer Percent encoding rate mismatch: %d, \n"
|
||||
" Number of frame processed per layer: %d \n",
|
||||
bit_rate_layer_[i], frame_rate_layer_[i], per_frame_bandwidth_[i],
|
||||
encoding_bitrate_[i], perc_frame_size_mismatch,
|
||||
perc_encoding_rate_mismatch, num_frames_per_update_[i]);
|
||||
EXPECT_LE(perc_frame_size_mismatch, max_delta_frame_size_mismatch);
|
||||
EXPECT_LE(perc_encoding_rate_mismatch, max_encoding_rate_mismatch);
|
||||
}
|
||||
printf("\n");
|
||||
EXPECT_LE(num_frames_to_hit_target_, max_time_hit_target);
|
||||
EXPECT_LE(num_dropped_frames, max_num_dropped_frames);
|
||||
EXPECT_EQ(num_resize_actions, num_spatial_resizes);
|
||||
EXPECT_EQ(num_key_frames_, num_key_frames);
|
||||
}
|
||||
|
||||
// Layer index corresponding to frame number, for up to 3 layers.
|
||||
void LayerIndexForFrame(int frame_number) {
|
||||
if (num_temporal_layers_ == 1) {
|
||||
layer_ = 0;
|
||||
} else if (num_temporal_layers_ == 2) {
|
||||
// layer 0: 0 2 4 ...
|
||||
// layer 1: 1 3
|
||||
if (frame_number % 2 == 0) {
|
||||
layer_ = 0;
|
||||
} else {
|
||||
layer_ = 1;
|
||||
}
|
||||
} else if (num_temporal_layers_ == 3) {
|
||||
// layer 0: 0 4 8 ...
|
||||
// layer 1: 2 6
|
||||
// layer 2: 1 3 5 7
|
||||
if (frame_number % 4 == 0) {
|
||||
layer_ = 0;
|
||||
} else if ((frame_number + 2) % 4 == 0) {
|
||||
layer_ = 1;
|
||||
} else if ((frame_number + 1) % 2 == 0) {
|
||||
layer_ = 2;
|
||||
}
|
||||
} else {
|
||||
assert(false); // Only up to 3 layers.
|
||||
}
|
||||
}
|
||||
|
||||
// Set the bitrate and frame rate per layer, for up to 3 layers.
|
||||
void SetLayerRates() {
|
||||
assert(num_temporal_layers_ <= 3);
|
||||
for (int i = 0; i < num_temporal_layers_; i++) {
|
||||
float bit_rate_ratio =
|
||||
kVp8LayerRateAlloction[num_temporal_layers_ - 1][i];
|
||||
if (i > 0) {
|
||||
float bit_rate_delta_ratio =
|
||||
kVp8LayerRateAlloction[num_temporal_layers_ - 1][i] -
|
||||
kVp8LayerRateAlloction[num_temporal_layers_ - 1][i - 1];
|
||||
bit_rate_layer_[i] = bit_rate_ * bit_rate_delta_ratio;
|
||||
} else {
|
||||
bit_rate_layer_[i] = bit_rate_ * bit_rate_ratio;
|
||||
}
|
||||
frame_rate_layer_[i] =
|
||||
frame_rate_ / static_cast<float>(1 << (num_temporal_layers_ - 1));
|
||||
}
|
||||
if (num_temporal_layers_ == 3) {
|
||||
frame_rate_layer_[2] = frame_rate_ / 2.0f;
|
||||
}
|
||||
}
|
||||
|
||||
// Processes all frames in the clip and verifies the result.
|
||||
void ProcessFramesAndVerify(QualityMetrics quality_metrics,
|
||||
RateProfile rate_profile,
|
||||
CodecConfigPars process,
|
||||
RateControlMetrics* rc_metrics) {
|
||||
// Codec/config settings.
|
||||
codec_type_ = process.codec_type;
|
||||
start_bitrate_ = rate_profile.target_bit_rate[0];
|
||||
packet_loss_ = process.packet_loss;
|
||||
key_frame_interval_ = process.key_frame_interval;
|
||||
num_temporal_layers_ = process.num_temporal_layers;
|
||||
error_concealment_on_ = process.error_concealment_on;
|
||||
denoising_on_ = process.denoising_on;
|
||||
frame_dropper_on_ = process.frame_dropper_on;
|
||||
spatial_resize_on_ = process.spatial_resize_on;
|
||||
SetUpCodecConfig();
|
||||
// Update the layers and the codec with the initial rates.
|
||||
bit_rate_ = rate_profile.target_bit_rate[0];
|
||||
frame_rate_ = rate_profile.input_frame_rate[0];
|
||||
SetLayerRates();
|
||||
// Set the initial target size for key frame.
|
||||
target_size_key_frame_initial_ =
|
||||
0.5 * kInitialBufferSize * bit_rate_layer_[0];
|
||||
processor_->SetRates(bit_rate_, frame_rate_);
|
||||
// Process each frame, up to |num_frames|.
|
||||
int num_frames = rate_profile.num_frames;
|
||||
int update_index = 0;
|
||||
ResetRateControlMetrics(
|
||||
rate_profile.frame_index_rate_update[update_index + 1]);
|
||||
int frame_number = 0;
|
||||
FrameType frame_type = kVideoFrameDelta;
|
||||
while (processor_->ProcessFrame(frame_number) &&
|
||||
frame_number < num_frames) {
|
||||
// Get the layer index for the frame |frame_number|.
|
||||
LayerIndexForFrame(frame_number);
|
||||
// Get the frame_type.
|
||||
frame_type = processor_->EncodedFrameType();
|
||||
// Counter for whole sequence run.
|
||||
++frame_number;
|
||||
// Counters for each rate update.
|
||||
++num_frames_per_update_[layer_];
|
||||
++num_frames_total_;
|
||||
UpdateRateControlMetrics(frame_number, frame_type);
|
||||
// If we hit another/next update, verify stats for current state and
|
||||
// update layers and codec with new rates.
|
||||
if (frame_number ==
|
||||
rate_profile.frame_index_rate_update[update_index + 1]) {
|
||||
VerifyRateControl(
|
||||
update_index, rc_metrics[update_index].max_key_frame_size_mismatch,
|
||||
rc_metrics[update_index].max_delta_frame_size_mismatch,
|
||||
rc_metrics[update_index].max_encoding_rate_mismatch,
|
||||
rc_metrics[update_index].max_time_hit_target,
|
||||
rc_metrics[update_index].max_num_dropped_frames,
|
||||
rc_metrics[update_index].num_spatial_resizes,
|
||||
rc_metrics[update_index].num_key_frames);
|
||||
// Update layer rates and the codec with new rates.
|
||||
++update_index;
|
||||
bit_rate_ = rate_profile.target_bit_rate[update_index];
|
||||
frame_rate_ = rate_profile.input_frame_rate[update_index];
|
||||
SetLayerRates();
|
||||
ResetRateControlMetrics(
|
||||
rate_profile.frame_index_rate_update[update_index + 1]);
|
||||
processor_->SetRates(bit_rate_, frame_rate_);
|
||||
}
|
||||
}
|
||||
VerifyRateControl(update_index,
|
||||
rc_metrics[update_index].max_key_frame_size_mismatch,
|
||||
rc_metrics[update_index].max_delta_frame_size_mismatch,
|
||||
rc_metrics[update_index].max_encoding_rate_mismatch,
|
||||
rc_metrics[update_index].max_time_hit_target,
|
||||
rc_metrics[update_index].max_num_dropped_frames,
|
||||
rc_metrics[update_index].num_spatial_resizes,
|
||||
rc_metrics[update_index].num_key_frames);
|
||||
EXPECT_EQ(num_frames, frame_number);
|
||||
EXPECT_EQ(num_frames + 1, static_cast<int>(stats_.stats_.size()));
|
||||
|
||||
// Release encoder and decoder to make sure they have finished processing:
|
||||
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
|
||||
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release());
|
||||
// Close the files before we start using them for SSIM/PSNR calculations.
|
||||
frame_reader_->Close();
|
||||
frame_writer_->Close();
|
||||
|
||||
// TODO(marpan): should compute these quality metrics per SetRates update.
|
||||
test::QualityMetricsResult psnr_result, ssim_result;
|
||||
EXPECT_EQ(0, test::I420MetricsFromFiles(config_.input_filename.c_str(),
|
||||
config_.output_filename.c_str(),
|
||||
config_.codec_settings->width,
|
||||
config_.codec_settings->height,
|
||||
&psnr_result, &ssim_result));
|
||||
printf("PSNR avg: %f, min: %f\nSSIM avg: %f, min: %f\n",
|
||||
psnr_result.average, psnr_result.min, ssim_result.average,
|
||||
ssim_result.min);
|
||||
stats_.PrintSummary();
|
||||
EXPECT_GT(psnr_result.average, quality_metrics.minimum_avg_psnr);
|
||||
EXPECT_GT(psnr_result.min, quality_metrics.minimum_min_psnr);
|
||||
EXPECT_GT(ssim_result.average, quality_metrics.minimum_avg_ssim);
|
||||
EXPECT_GT(ssim_result.min, quality_metrics.minimum_min_ssim);
|
||||
if (remove(config_.output_filename.c_str()) < 0) {
|
||||
fprintf(stderr, "Failed to remove temporary file!\n");
|
||||
}
|
||||
}
|
||||
};
|
||||
namespace test {
|
||||
|
||||
#if defined(WEBRTC_VIDEOPROCESSOR_H264_TESTS)
|
||||
|
||||
@ -961,4 +393,5 @@ TEST_F(VideoProcessorIntegrationTest, MAYBE_ProcessNoLossTemporalLayersVP8) {
|
||||
ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings,
|
||||
rc_metrics);
|
||||
}
|
||||
} // namespace test
|
||||
} // namespace webrtc
|
||||
|
||||
@ -0,0 +1,624 @@
|
||||
/*
|
||||
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_
|
||||
#define WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_
|
||||
|
||||
#include <math.h>
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "webrtc/modules/video_coding/codecs/h264/include/h264.h"
|
||||
#include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h"
|
||||
#include "webrtc/modules/video_coding/codecs/test/videoprocessor.h"
|
||||
#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
|
||||
#include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h"
|
||||
#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
|
||||
#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h"
|
||||
#include "webrtc/modules/video_coding/include/video_codec_interface.h"
|
||||
#include "webrtc/modules/video_coding/include/video_coding.h"
|
||||
#include "webrtc/test/gtest.h"
|
||||
#include "webrtc/test/testsupport/fileutils.h"
|
||||
#include "webrtc/test/testsupport/frame_reader.h"
|
||||
#include "webrtc/test/testsupport/frame_writer.h"
|
||||
#include "webrtc/test/testsupport/metrics/video_metrics.h"
|
||||
#include "webrtc/test/testsupport/packet_reader.h"
|
||||
#include "webrtc/typedefs.h"
|
||||
|
||||
namespace webrtc {
|
||||
namespace test {
|
||||
// Maximum number of rate updates (i.e., calls to encoder to change bitrate
|
||||
// and/or frame rate) for the current tests.
|
||||
const int kMaxNumRateUpdates = 3;
|
||||
|
||||
const int kPercTargetvsActualMismatch = 20;
|
||||
const int kBaseKeyFrameInterval = 3000;
|
||||
|
||||
// Default sequence is foreman (CIF): may be better to use VGA for resize test.
|
||||
const int kCifWidth = 352;
|
||||
const int kCifHeight = 288;
|
||||
const char kFilenameForemanCif[] = "foreman_cif";
|
||||
|
||||
// Codec and network settings.
|
||||
struct CodecConfigPars {
|
||||
VideoCodecType codec_type;
|
||||
float packet_loss;
|
||||
int num_temporal_layers;
|
||||
int key_frame_interval;
|
||||
bool error_concealment_on;
|
||||
bool denoising_on;
|
||||
bool frame_dropper_on;
|
||||
bool spatial_resize_on;
|
||||
int width;
|
||||
int height;
|
||||
std::string filename;
|
||||
bool verbose_logging;
|
||||
};
|
||||
|
||||
// Quality metrics.
|
||||
struct QualityMetrics {
|
||||
double minimum_avg_psnr;
|
||||
double minimum_min_psnr;
|
||||
double minimum_avg_ssim;
|
||||
double minimum_min_ssim;
|
||||
};
|
||||
|
||||
// The sequence of bitrate and frame rate changes for the encoder, the frame
|
||||
// number where the changes are made, and the total number of frames for the
|
||||
// test.
|
||||
struct RateProfile {
|
||||
int target_bit_rate[kMaxNumRateUpdates];
|
||||
int input_frame_rate[kMaxNumRateUpdates];
|
||||
int frame_index_rate_update[kMaxNumRateUpdates + 1];
|
||||
int num_frames;
|
||||
};
|
||||
|
||||
// Metrics for the rate control. The rate mismatch metrics are defined as
|
||||
// percentages.|max_time_hit_target| is defined as number of frames, after a
|
||||
// rate update is made to the encoder, for the encoder to reach within
|
||||
// |kPercTargetvsActualMismatch| of new target rate. The metrics are defined for
|
||||
// each rate update sequence.
|
||||
struct RateControlMetrics {
|
||||
int max_num_dropped_frames;
|
||||
int max_key_frame_size_mismatch;
|
||||
int max_delta_frame_size_mismatch;
|
||||
int max_encoding_rate_mismatch;
|
||||
int max_time_hit_target;
|
||||
int num_spatial_resizes;
|
||||
int num_key_frames;
|
||||
};
|
||||
|
||||
#if !defined(WEBRTC_IOS)
|
||||
const int kNbrFramesShort = 100; // Some tests are run for shorter sequence.
|
||||
#endif
|
||||
const int kNbrFramesLong = 299;
|
||||
|
||||
// Parameters from VP8 wrapper, which control target size of key frames.
|
||||
const float kInitialBufferSize = 0.5f;
|
||||
const float kOptimalBufferSize = 0.6f;
|
||||
const float kScaleKeyFrameSize = 0.5f;
|
||||
|
||||
// Integration test for video processor. Encodes+decodes a clip and
|
||||
// writes it to the output directory. After completion, quality metrics
|
||||
// (PSNR and SSIM) and rate control metrics are computed to verify that the
|
||||
// quality and encoder response is acceptable. The rate control tests allow us
|
||||
// to verify the behavior for changing bitrate, changing frame rate, frame
|
||||
// dropping/spatial resize, and temporal layers. The limits for the rate
|
||||
// control metrics are set to be fairly conservative, so failure should only
|
||||
// happen when some significant regression or breakdown occurs.
|
||||
class VideoProcessorIntegrationTest : public testing::Test {
|
||||
protected:
|
||||
std::unique_ptr<VideoEncoder> encoder_;
|
||||
std::unique_ptr<VideoDecoder> decoder_;
|
||||
std::unique_ptr<test::FrameReader> frame_reader_;
|
||||
std::unique_ptr<test::FrameWriter> frame_writer_;
|
||||
test::PacketReader packet_reader_;
|
||||
std::unique_ptr<test::PacketManipulator> packet_manipulator_;
|
||||
test::Stats stats_;
|
||||
test::TestConfig config_;
|
||||
VideoCodec codec_settings_;
|
||||
std::unique_ptr<test::VideoProcessor> processor_;
|
||||
TemporalLayersFactory tl_factory_;
|
||||
|
||||
// Quantities defined/updated for every encoder rate update.
|
||||
// Some quantities defined per temporal layer (at most 3 layers in this test).
|
||||
int num_frames_per_update_[3];
|
||||
float sum_frame_size_mismatch_[3];
|
||||
float sum_encoded_frame_size_[3];
|
||||
float encoding_bitrate_[3];
|
||||
float per_frame_bandwidth_[3];
|
||||
float bit_rate_layer_[3];
|
||||
float frame_rate_layer_[3];
|
||||
int num_frames_total_;
|
||||
float sum_encoded_frame_size_total_;
|
||||
float encoding_bitrate_total_;
|
||||
float perc_encoding_rate_mismatch_;
|
||||
int num_frames_to_hit_target_;
|
||||
bool encoding_rate_within_target_;
|
||||
int bit_rate_;
|
||||
int frame_rate_;
|
||||
int layer_;
|
||||
float target_size_key_frame_initial_;
|
||||
float target_size_key_frame_;
|
||||
float sum_key_frame_size_mismatch_;
|
||||
int num_key_frames_;
|
||||
float start_bitrate_;
|
||||
|
||||
// Codec and network settings.
|
||||
VideoCodecType codec_type_;
|
||||
float packet_loss_;
|
||||
int num_temporal_layers_;
|
||||
int key_frame_interval_;
|
||||
bool error_concealment_on_;
|
||||
bool denoising_on_;
|
||||
bool frame_dropper_on_;
|
||||
bool spatial_resize_on_;
|
||||
|
||||
VideoProcessorIntegrationTest() {}
|
||||
virtual ~VideoProcessorIntegrationTest() {}
|
||||
|
||||
void SetUpCodecConfig(const std::string& filename,
|
||||
int width,
|
||||
int height,
|
||||
bool verbose_logging) {
|
||||
if (codec_type_ == kVideoCodecH264) {
|
||||
encoder_.reset(H264Encoder::Create(cricket::VideoCodec("H264")));
|
||||
decoder_.reset(H264Decoder::Create());
|
||||
VideoCodingModule::Codec(kVideoCodecH264, &codec_settings_);
|
||||
} else if (codec_type_ == kVideoCodecVP8) {
|
||||
encoder_.reset(VP8Encoder::Create());
|
||||
decoder_.reset(VP8Decoder::Create());
|
||||
VideoCodingModule::Codec(kVideoCodecVP8, &codec_settings_);
|
||||
} else if (codec_type_ == kVideoCodecVP9) {
|
||||
encoder_.reset(VP9Encoder::Create());
|
||||
decoder_.reset(VP9Decoder::Create());
|
||||
VideoCodingModule::Codec(kVideoCodecVP9, &codec_settings_);
|
||||
}
|
||||
|
||||
// Configure input filename.
|
||||
config_.input_filename = test::ResourcePath(filename, "yuv");
|
||||
if (verbose_logging)
|
||||
printf("Filename: %s\n", filename.c_str());
|
||||
// Generate an output filename in a safe way.
|
||||
config_.output_filename = test::TempFilename(
|
||||
test::OutputPath(), "videoprocessor_integrationtest");
|
||||
config_.frame_length_in_bytes = CalcBufferSize(kI420, width, height);
|
||||
config_.verbose = verbose_logging;
|
||||
// Only allow encoder/decoder to use single core, for predictability.
|
||||
config_.use_single_core = true;
|
||||
// Key frame interval and packet loss are set for each test.
|
||||
config_.keyframe_interval = key_frame_interval_;
|
||||
config_.networking_config.packet_loss_probability = packet_loss_;
|
||||
|
||||
// Configure codec settings.
|
||||
config_.codec_settings = &codec_settings_;
|
||||
config_.codec_settings->startBitrate = start_bitrate_;
|
||||
config_.codec_settings->width = width;
|
||||
config_.codec_settings->height = height;
|
||||
|
||||
// These features may be set depending on the test.
|
||||
switch (config_.codec_settings->codecType) {
|
||||
case kVideoCodecH264:
|
||||
config_.codec_settings->H264()->frameDroppingOn = frame_dropper_on_;
|
||||
config_.codec_settings->H264()->keyFrameInterval =
|
||||
kBaseKeyFrameInterval;
|
||||
break;
|
||||
case kVideoCodecVP8:
|
||||
config_.codec_settings->VP8()->errorConcealmentOn =
|
||||
error_concealment_on_;
|
||||
config_.codec_settings->VP8()->denoisingOn = denoising_on_;
|
||||
config_.codec_settings->VP8()->numberOfTemporalLayers =
|
||||
num_temporal_layers_;
|
||||
config_.codec_settings->VP8()->frameDroppingOn = frame_dropper_on_;
|
||||
config_.codec_settings->VP8()->automaticResizeOn = spatial_resize_on_;
|
||||
config_.codec_settings->VP8()->keyFrameInterval = kBaseKeyFrameInterval;
|
||||
break;
|
||||
case kVideoCodecVP9:
|
||||
config_.codec_settings->VP9()->denoisingOn = denoising_on_;
|
||||
config_.codec_settings->VP9()->numberOfTemporalLayers =
|
||||
num_temporal_layers_;
|
||||
config_.codec_settings->VP9()->frameDroppingOn = frame_dropper_on_;
|
||||
config_.codec_settings->VP9()->automaticResizeOn = spatial_resize_on_;
|
||||
config_.codec_settings->VP9()->keyFrameInterval = kBaseKeyFrameInterval;
|
||||
break;
|
||||
default:
|
||||
assert(false);
|
||||
break;
|
||||
}
|
||||
frame_reader_.reset(new test::FrameReaderImpl(
|
||||
config_.input_filename, config_.codec_settings->width,
|
||||
config_.codec_settings->height));
|
||||
frame_writer_.reset(new test::FrameWriterImpl(
|
||||
config_.output_filename, config_.frame_length_in_bytes));
|
||||
ASSERT_TRUE(frame_reader_->Init());
|
||||
ASSERT_TRUE(frame_writer_->Init());
|
||||
|
||||
packet_manipulator_.reset(new test::PacketManipulatorImpl(
|
||||
&packet_reader_, config_.networking_config, config_.verbose));
|
||||
processor_.reset(new test::VideoProcessorImpl(
|
||||
encoder_.get(), decoder_.get(), frame_reader_.get(),
|
||||
frame_writer_.get(), packet_manipulator_.get(), config_, &stats_));
|
||||
ASSERT_TRUE(processor_->Init());
|
||||
}
|
||||
|
||||
// Reset quantities after each encoder update, update the target
|
||||
// per-frame bandwidth.
|
||||
void ResetRateControlMetrics(int num_frames) {
|
||||
for (int i = 0; i < num_temporal_layers_; i++) {
|
||||
num_frames_per_update_[i] = 0;
|
||||
sum_frame_size_mismatch_[i] = 0.0f;
|
||||
sum_encoded_frame_size_[i] = 0.0f;
|
||||
encoding_bitrate_[i] = 0.0f;
|
||||
// Update layer per-frame-bandwidth.
|
||||
per_frame_bandwidth_[i] = static_cast<float>(bit_rate_layer_[i]) /
|
||||
static_cast<float>(frame_rate_layer_[i]);
|
||||
}
|
||||
// Set maximum size of key frames, following setting in the VP8 wrapper.
|
||||
float max_key_size = kScaleKeyFrameSize * kOptimalBufferSize * frame_rate_;
|
||||
// We don't know exact target size of the key frames (except for first one),
|
||||
// but the minimum in libvpx is ~|3 * per_frame_bandwidth| and maximum is
|
||||
// set by |max_key_size_ * per_frame_bandwidth|. Take middle point/average
|
||||
// as reference for mismatch. Note key frames always correspond to base
|
||||
// layer frame in this test.
|
||||
target_size_key_frame_ = 0.5 * (3 + max_key_size) * per_frame_bandwidth_[0];
|
||||
num_frames_total_ = 0;
|
||||
sum_encoded_frame_size_total_ = 0.0f;
|
||||
encoding_bitrate_total_ = 0.0f;
|
||||
perc_encoding_rate_mismatch_ = 0.0f;
|
||||
num_frames_to_hit_target_ = num_frames;
|
||||
encoding_rate_within_target_ = false;
|
||||
sum_key_frame_size_mismatch_ = 0.0;
|
||||
num_key_frames_ = 0;
|
||||
}
|
||||
|
||||
// For every encoded frame, update the rate control metrics.
|
||||
void UpdateRateControlMetrics(int frame_num, FrameType frame_type) {
|
||||
float encoded_size_kbits = processor_->EncodedFrameSize() * 8.0f / 1000.0f;
|
||||
// Update layer data.
|
||||
// Update rate mismatch relative to per-frame bandwidth for delta frames.
|
||||
if (frame_type == kVideoFrameDelta) {
|
||||
// TODO(marpan): Should we count dropped (zero size) frames in mismatch?
|
||||
sum_frame_size_mismatch_[layer_] +=
|
||||
fabs(encoded_size_kbits - per_frame_bandwidth_[layer_]) /
|
||||
per_frame_bandwidth_[layer_];
|
||||
} else {
|
||||
float target_size = (frame_num == 1) ? target_size_key_frame_initial_
|
||||
: target_size_key_frame_;
|
||||
sum_key_frame_size_mismatch_ +=
|
||||
fabs(encoded_size_kbits - target_size) / target_size;
|
||||
num_key_frames_ += 1;
|
||||
}
|
||||
sum_encoded_frame_size_[layer_] += encoded_size_kbits;
|
||||
// Encoding bitrate per layer: from the start of the update/run to the
|
||||
// current frame.
|
||||
encoding_bitrate_[layer_] = sum_encoded_frame_size_[layer_] *
|
||||
frame_rate_layer_[layer_] /
|
||||
num_frames_per_update_[layer_];
|
||||
// Total encoding rate: from the start of the update/run to current frame.
|
||||
sum_encoded_frame_size_total_ += encoded_size_kbits;
|
||||
encoding_bitrate_total_ =
|
||||
sum_encoded_frame_size_total_ * frame_rate_ / num_frames_total_;
|
||||
perc_encoding_rate_mismatch_ =
|
||||
100 * fabs(encoding_bitrate_total_ - bit_rate_) / bit_rate_;
|
||||
if (perc_encoding_rate_mismatch_ < kPercTargetvsActualMismatch &&
|
||||
!encoding_rate_within_target_) {
|
||||
num_frames_to_hit_target_ = num_frames_total_;
|
||||
encoding_rate_within_target_ = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Verify expected behavior of rate control and print out data.
|
||||
void VerifyRateControl(int update_index,
|
||||
int max_key_frame_size_mismatch,
|
||||
int max_delta_frame_size_mismatch,
|
||||
int max_encoding_rate_mismatch,
|
||||
int max_time_hit_target,
|
||||
int max_num_dropped_frames,
|
||||
int num_spatial_resizes,
|
||||
int num_key_frames) {
|
||||
int num_dropped_frames = processor_->NumberDroppedFrames();
|
||||
int num_resize_actions = processor_->NumberSpatialResizes();
|
||||
printf(
|
||||
"For update #: %d,\n"
|
||||
" Target Bitrate: %d,\n"
|
||||
" Encoding bitrate: %f,\n"
|
||||
" Frame rate: %d \n",
|
||||
update_index, bit_rate_, encoding_bitrate_total_, frame_rate_);
|
||||
printf(
|
||||
" Number of frames to approach target rate: %d, \n"
|
||||
" Number of dropped frames: %d, \n"
|
||||
" Number of spatial resizes: %d, \n",
|
||||
num_frames_to_hit_target_, num_dropped_frames, num_resize_actions);
|
||||
EXPECT_LE(perc_encoding_rate_mismatch_, max_encoding_rate_mismatch);
|
||||
if (num_key_frames_ > 0) {
|
||||
int perc_key_frame_size_mismatch =
|
||||
100 * sum_key_frame_size_mismatch_ / num_key_frames_;
|
||||
printf(
|
||||
" Number of Key frames: %d \n"
|
||||
" Key frame rate mismatch: %d \n",
|
||||
num_key_frames_, perc_key_frame_size_mismatch);
|
||||
EXPECT_LE(perc_key_frame_size_mismatch, max_key_frame_size_mismatch);
|
||||
}
|
||||
printf("\n");
|
||||
printf("Rates statistics for Layer data \n");
|
||||
for (int i = 0; i < num_temporal_layers_; i++) {
|
||||
printf("Temporal layer #%d \n", i);
|
||||
int perc_frame_size_mismatch =
|
||||
100 * sum_frame_size_mismatch_[i] / num_frames_per_update_[i];
|
||||
int perc_encoding_rate_mismatch =
|
||||
100 * fabs(encoding_bitrate_[i] - bit_rate_layer_[i]) /
|
||||
bit_rate_layer_[i];
|
||||
printf(
|
||||
" Target Layer Bit rate: %f \n"
|
||||
" Layer frame rate: %f, \n"
|
||||
" Layer per frame bandwidth: %f, \n"
|
||||
" Layer Encoding bit rate: %f, \n"
|
||||
" Layer Percent frame size mismatch: %d, \n"
|
||||
" Layer Percent encoding rate mismatch: %d, \n"
|
||||
" Number of frame processed per layer: %d \n",
|
||||
bit_rate_layer_[i], frame_rate_layer_[i], per_frame_bandwidth_[i],
|
||||
encoding_bitrate_[i], perc_frame_size_mismatch,
|
||||
perc_encoding_rate_mismatch, num_frames_per_update_[i]);
|
||||
EXPECT_LE(perc_frame_size_mismatch, max_delta_frame_size_mismatch);
|
||||
EXPECT_LE(perc_encoding_rate_mismatch, max_encoding_rate_mismatch);
|
||||
}
|
||||
printf("\n");
|
||||
EXPECT_LE(num_frames_to_hit_target_, max_time_hit_target);
|
||||
EXPECT_LE(num_dropped_frames, max_num_dropped_frames);
|
||||
EXPECT_EQ(num_resize_actions, num_spatial_resizes);
|
||||
EXPECT_EQ(num_key_frames_, num_key_frames);
|
||||
}
|
||||
|
||||
// Layer index corresponding to frame number, for up to 3 layers.
|
||||
void LayerIndexForFrame(int frame_number) {
|
||||
if (num_temporal_layers_ == 1) {
|
||||
layer_ = 0;
|
||||
} else if (num_temporal_layers_ == 2) {
|
||||
// layer 0: 0 2 4 ...
|
||||
// layer 1: 1 3
|
||||
if (frame_number % 2 == 0) {
|
||||
layer_ = 0;
|
||||
} else {
|
||||
layer_ = 1;
|
||||
}
|
||||
} else if (num_temporal_layers_ == 3) {
|
||||
// layer 0: 0 4 8 ...
|
||||
// layer 1: 2 6
|
||||
// layer 2: 1 3 5 7
|
||||
if (frame_number % 4 == 0) {
|
||||
layer_ = 0;
|
||||
} else if ((frame_number + 2) % 4 == 0) {
|
||||
layer_ = 1;
|
||||
} else if ((frame_number + 1) % 2 == 0) {
|
||||
layer_ = 2;
|
||||
}
|
||||
} else {
|
||||
assert(false); // Only up to 3 layers.
|
||||
}
|
||||
}
|
||||
|
||||
// Set the bitrate and frame rate per layer, for up to 3 layers.
|
||||
void SetLayerRates() {
|
||||
assert(num_temporal_layers_ <= 3);
|
||||
for (int i = 0; i < num_temporal_layers_; i++) {
|
||||
float bit_rate_ratio =
|
||||
kVp8LayerRateAlloction[num_temporal_layers_ - 1][i];
|
||||
if (i > 0) {
|
||||
float bit_rate_delta_ratio =
|
||||
kVp8LayerRateAlloction[num_temporal_layers_ - 1][i] -
|
||||
kVp8LayerRateAlloction[num_temporal_layers_ - 1][i - 1];
|
||||
bit_rate_layer_[i] = bit_rate_ * bit_rate_delta_ratio;
|
||||
} else {
|
||||
bit_rate_layer_[i] = bit_rate_ * bit_rate_ratio;
|
||||
}
|
||||
frame_rate_layer_[i] =
|
||||
frame_rate_ / static_cast<float>(1 << (num_temporal_layers_ - 1));
|
||||
}
|
||||
if (num_temporal_layers_ == 3) {
|
||||
frame_rate_layer_[2] = frame_rate_ / 2.0f;
|
||||
}
|
||||
}
|
||||
|
||||
// Processes all frames in the clip and verifies the result.
|
||||
void ProcessFramesAndVerify(QualityMetrics quality_metrics,
|
||||
RateProfile rate_profile,
|
||||
CodecConfigPars process,
|
||||
RateControlMetrics* rc_metrics) {
|
||||
// Codec/config settings.
|
||||
codec_type_ = process.codec_type;
|
||||
start_bitrate_ = rate_profile.target_bit_rate[0];
|
||||
packet_loss_ = process.packet_loss;
|
||||
key_frame_interval_ = process.key_frame_interval;
|
||||
num_temporal_layers_ = process.num_temporal_layers;
|
||||
error_concealment_on_ = process.error_concealment_on;
|
||||
denoising_on_ = process.denoising_on;
|
||||
frame_dropper_on_ = process.frame_dropper_on;
|
||||
spatial_resize_on_ = process.spatial_resize_on;
|
||||
SetUpCodecConfig(process.filename, process.width, process.height,
|
||||
process.verbose_logging);
|
||||
// Update the layers and the codec with the initial rates.
|
||||
bit_rate_ = rate_profile.target_bit_rate[0];
|
||||
frame_rate_ = rate_profile.input_frame_rate[0];
|
||||
SetLayerRates();
|
||||
// Set the initial target size for key frame.
|
||||
target_size_key_frame_initial_ =
|
||||
0.5 * kInitialBufferSize * bit_rate_layer_[0];
|
||||
processor_->SetRates(bit_rate_, frame_rate_);
|
||||
// Process each frame, up to |num_frames|.
|
||||
int num_frames = rate_profile.num_frames;
|
||||
int update_index = 0;
|
||||
ResetRateControlMetrics(
|
||||
rate_profile.frame_index_rate_update[update_index + 1]);
|
||||
int frame_number = 0;
|
||||
FrameType frame_type = kVideoFrameDelta;
|
||||
while (processor_->ProcessFrame(frame_number) &&
|
||||
frame_number < num_frames) {
|
||||
// Get the layer index for the frame |frame_number|.
|
||||
LayerIndexForFrame(frame_number);
|
||||
// Get the frame_type.
|
||||
frame_type = processor_->EncodedFrameType();
|
||||
// Counter for whole sequence run.
|
||||
++frame_number;
|
||||
// Counters for each rate update.
|
||||
++num_frames_per_update_[layer_];
|
||||
++num_frames_total_;
|
||||
UpdateRateControlMetrics(frame_number, frame_type);
|
||||
// If we hit another/next update, verify stats for current state and
|
||||
// update layers and codec with new rates.
|
||||
if (frame_number ==
|
||||
rate_profile.frame_index_rate_update[update_index + 1]) {
|
||||
VerifyRateControl(
|
||||
update_index, rc_metrics[update_index].max_key_frame_size_mismatch,
|
||||
rc_metrics[update_index].max_delta_frame_size_mismatch,
|
||||
rc_metrics[update_index].max_encoding_rate_mismatch,
|
||||
rc_metrics[update_index].max_time_hit_target,
|
||||
rc_metrics[update_index].max_num_dropped_frames,
|
||||
rc_metrics[update_index].num_spatial_resizes,
|
||||
rc_metrics[update_index].num_key_frames);
|
||||
// Update layer rates and the codec with new rates.
|
||||
++update_index;
|
||||
bit_rate_ = rate_profile.target_bit_rate[update_index];
|
||||
frame_rate_ = rate_profile.input_frame_rate[update_index];
|
||||
SetLayerRates();
|
||||
ResetRateControlMetrics(
|
||||
rate_profile.frame_index_rate_update[update_index + 1]);
|
||||
processor_->SetRates(bit_rate_, frame_rate_);
|
||||
}
|
||||
}
|
||||
VerifyRateControl(update_index,
|
||||
rc_metrics[update_index].max_key_frame_size_mismatch,
|
||||
rc_metrics[update_index].max_delta_frame_size_mismatch,
|
||||
rc_metrics[update_index].max_encoding_rate_mismatch,
|
||||
rc_metrics[update_index].max_time_hit_target,
|
||||
rc_metrics[update_index].max_num_dropped_frames,
|
||||
rc_metrics[update_index].num_spatial_resizes,
|
||||
rc_metrics[update_index].num_key_frames);
|
||||
EXPECT_EQ(num_frames, frame_number);
|
||||
EXPECT_EQ(num_frames + 1, static_cast<int>(stats_.stats_.size()));
|
||||
|
||||
// Release encoder and decoder to make sure they have finished processing:
|
||||
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
|
||||
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release());
|
||||
// Close the files before we start using them for SSIM/PSNR calculations.
|
||||
frame_reader_->Close();
|
||||
frame_writer_->Close();
|
||||
|
||||
// TODO(marpan): should compute these quality metrics per SetRates update.
|
||||
test::QualityMetricsResult psnr_result, ssim_result;
|
||||
EXPECT_EQ(0, test::I420MetricsFromFiles(config_.input_filename.c_str(),
|
||||
config_.output_filename.c_str(),
|
||||
config_.codec_settings->width,
|
||||
config_.codec_settings->height,
|
||||
&psnr_result, &ssim_result));
|
||||
printf("PSNR avg: %f, min: %f\nSSIM avg: %f, min: %f\n",
|
||||
psnr_result.average, psnr_result.min, ssim_result.average,
|
||||
ssim_result.min);
|
||||
stats_.PrintSummary();
|
||||
EXPECT_GT(psnr_result.average, quality_metrics.minimum_avg_psnr);
|
||||
EXPECT_GT(psnr_result.min, quality_metrics.minimum_min_psnr);
|
||||
EXPECT_GT(ssim_result.average, quality_metrics.minimum_avg_ssim);
|
||||
EXPECT_GT(ssim_result.min, quality_metrics.minimum_min_ssim);
|
||||
if (remove(config_.output_filename.c_str()) < 0) {
|
||||
fprintf(stderr, "Failed to remove temporary file!\n");
|
||||
}
|
||||
}
|
||||
|
||||
static void SetRateProfilePars(RateProfile* rate_profile,
|
||||
int update_index,
|
||||
int bit_rate,
|
||||
int frame_rate,
|
||||
int frame_index_rate_update) {
|
||||
rate_profile->target_bit_rate[update_index] = bit_rate;
|
||||
rate_profile->input_frame_rate[update_index] = frame_rate;
|
||||
rate_profile->frame_index_rate_update[update_index] =
|
||||
frame_index_rate_update;
|
||||
}
|
||||
|
||||
static void SetCodecParameters(CodecConfigPars* process_settings,
|
||||
VideoCodecType codec_type,
|
||||
float packet_loss,
|
||||
int key_frame_interval,
|
||||
int num_temporal_layers,
|
||||
bool error_concealment_on,
|
||||
bool denoising_on,
|
||||
bool frame_dropper_on,
|
||||
bool spatial_resize_on,
|
||||
int width,
|
||||
int height,
|
||||
const std::string& filename,
|
||||
bool verbose_logging) {
|
||||
process_settings->codec_type = codec_type;
|
||||
process_settings->packet_loss = packet_loss;
|
||||
process_settings->key_frame_interval = key_frame_interval;
|
||||
process_settings->num_temporal_layers = num_temporal_layers,
|
||||
process_settings->error_concealment_on = error_concealment_on;
|
||||
process_settings->denoising_on = denoising_on;
|
||||
process_settings->frame_dropper_on = frame_dropper_on;
|
||||
process_settings->spatial_resize_on = spatial_resize_on;
|
||||
process_settings->width = width;
|
||||
process_settings->height = height;
|
||||
process_settings->filename = filename;
|
||||
process_settings->verbose_logging = verbose_logging;
|
||||
}
|
||||
|
||||
static void SetCodecParameters(CodecConfigPars* process_settings,
|
||||
VideoCodecType codec_type,
|
||||
float packet_loss,
|
||||
int key_frame_interval,
|
||||
int num_temporal_layers,
|
||||
bool error_concealment_on,
|
||||
bool denoising_on,
|
||||
bool frame_dropper_on,
|
||||
bool spatial_resize_on) {
|
||||
SetCodecParameters(process_settings, codec_type, packet_loss,
|
||||
key_frame_interval, num_temporal_layers,
|
||||
error_concealment_on, denoising_on, frame_dropper_on,
|
||||
spatial_resize_on, kCifWidth, kCifHeight,
|
||||
kFilenameForemanCif, false /* verbose_logging */);
|
||||
}
|
||||
|
||||
static void SetQualityMetrics(QualityMetrics* quality_metrics,
|
||||
double minimum_avg_psnr,
|
||||
double minimum_min_psnr,
|
||||
double minimum_avg_ssim,
|
||||
double minimum_min_ssim) {
|
||||
quality_metrics->minimum_avg_psnr = minimum_avg_psnr;
|
||||
quality_metrics->minimum_min_psnr = minimum_min_psnr;
|
||||
quality_metrics->minimum_avg_ssim = minimum_avg_ssim;
|
||||
quality_metrics->minimum_min_ssim = minimum_min_ssim;
|
||||
}
|
||||
|
||||
static void SetRateControlMetrics(RateControlMetrics* rc_metrics,
|
||||
int update_index,
|
||||
int max_num_dropped_frames,
|
||||
int max_key_frame_size_mismatch,
|
||||
int max_delta_frame_size_mismatch,
|
||||
int max_encoding_rate_mismatch,
|
||||
int max_time_hit_target,
|
||||
int num_spatial_resizes,
|
||||
int num_key_frames) {
|
||||
rc_metrics[update_index].max_num_dropped_frames = max_num_dropped_frames;
|
||||
rc_metrics[update_index].max_key_frame_size_mismatch =
|
||||
max_key_frame_size_mismatch;
|
||||
rc_metrics[update_index].max_delta_frame_size_mismatch =
|
||||
max_delta_frame_size_mismatch;
|
||||
rc_metrics[update_index].max_encoding_rate_mismatch =
|
||||
max_encoding_rate_mismatch;
|
||||
rc_metrics[update_index].max_time_hit_target = max_time_hit_target;
|
||||
rc_metrics[update_index].num_spatial_resizes = num_spatial_resizes;
|
||||
rc_metrics[update_index].num_key_frames = num_key_frames;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace test
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_
|
||||
Reference in New Issue
Block a user