zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Better comparison of videos with barcode errors

Browse Source 
The frame_analyzer which is used by compare_videos.py needs to handle
barcode errors. As before the reference and the test video can contain
repeated frames. When there are barcode decode errors in the test video,
then we should not let that contribute to the skipped frames score. When
there are barcode decode errors in the reference video, then we need to
take proper care to still calculate skipped barcodes when the
corresponding frames are not present in the test video and the test
video does not have a frame with a barcode decode error that could have
been the same frame as the one in the reference.

A new metric total number of skipped frames and for number of decode
errors is introduced. Barcodes that appears in the test video, but not
in the reference video are also listed.

BUG=webrtc:6967

Review-Url: https://codereview.webrtc.org/2666333003
Cr-Commit-Position: refs/heads/master@{#16638}
This commit is contained in:
mandermo
2017-02-16 01:36:43 -08:00
committed by Commit bot
parent 12b3e03bde
commit 7cebe78332
3 changed files with 317 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
webrtc/tools/frame_analyzer/video_quality_analysis.cc
View File

@ -325,23 +325,32 @@ void PrintMaxRepeatedAndSkippedFrames(const std::string& label,
stats_file_test_name);
}
namespace {
// Clusters the frames in the file. First in the pair is the frame number and
// second is the number
// of frames in that cluster. So if first frame in video has number 100 and it
// is repeated 3 after
// each other, then the first entry in the returned vector has first set to 100
// and second set
// to 3.
std::vector<std::pair<int, int> > CalculateFrameClusters(FILE* file) {
std::vector<std::pair<int, int> > CalculateFrameClusters(
FILE* file,
int* num_decode_errors) {
if (num_decode_errors) {
*num_decode_errors = 0;
}
std::vector<std::pair<int, int> > frame_cnt;
char line[STATS_LINE_LENGTH];
while (GetNextStatsLine(file, line)) {
int decoded_frame_number = ExtractDecodedFrameNumber(line);
if (decoded_frame_number == -1) {
continue;
int decoded_frame_number;
if (IsThereBarcodeError(line)) {
decoded_frame_number = DECODE_ERROR;
if (num_decode_errors) {
++*num_decode_errors;
}
} else {
decoded_frame_number = ExtractDecodedFrameNumber(line);
}
if (frame_cnt.empty() || frame_cnt.back().first != decoded_frame_number) {
if (frame_cnt.size() >= 2 && decoded_frame_number != DECODE_ERROR &&
frame_cnt.back().first == DECODE_ERROR &&
frame_cnt[frame_cnt.size() - 2].first == decoded_frame_number) {
// Handle when there is a decoding error inside a cluster of frames.
frame_cnt[frame_cnt.size() - 2].second += frame_cnt.back().second + 1;
frame_cnt.pop_back();
} else if (frame_cnt.empty() ||
frame_cnt.back().first != decoded_frame_number) {
frame_cnt.push_back(std::make_pair(decoded_frame_number, 1));
} else {
++frame_cnt.back().second;
@ -349,7 +358,6 @@ std::vector<std::pair<int, int> > CalculateFrameClusters(FILE* file) {
}
return frame_cnt;
}
} // namespace
void PrintMaxRepeatedAndSkippedFrames(FILE* output,
const std::string& label,
@ -370,13 +378,16 @@ void PrintMaxRepeatedAndSkippedFrames(FILE* output,
}
int max_repeated_frames = 1;
int max_skipped_frames = 1;
int max_skipped_frames = 0;
int decode_errors_ref = 0;
int decode_errors_test = 0;
std::vector<std::pair<int, int> > frame_cnt_ref =
CalculateFrameClusters(stats_file_ref);
CalculateFrameClusters(stats_file_ref, &decode_errors_ref);
std::vector<std::pair<int, int> > frame_cnt_test =
CalculateFrameClusters(stats_file_test);
CalculateFrameClusters(stats_file_test, &decode_errors_test);
fclose(stats_file_ref);
fclose(stats_file_test);
@ -391,9 +402,19 @@ void PrintMaxRepeatedAndSkippedFrames(FILE* output,
return;
}
while (it_test != end_test && it_test->first == DECODE_ERROR) {
++it_test;
}
if (it_test == end_test) {
fprintf(stderr, "Test video only has barcode decode errors\n");
return;
}
// Find the first frame in the reference video that match the first frame in
// the test video.
while (it_ref != end_ref && it_ref->first != it_test->first) {
while (it_ref != end_ref &&
(it_ref->first == DECODE_ERROR || it_ref->first != it_test->first)) {
++it_ref;
}
if (it_ref == end_ref) {
@ -403,33 +424,58 @@ void PrintMaxRepeatedAndSkippedFrames(FILE* output,
return;
}
int total_skipped_frames = 0;
for (;;) {
max_repeated_frames =
std::max(max_repeated_frames, it_test->second - it_ref->second + 1);
bool passed_error = false;
++it_test;
while (it_test != end_test && it_test->first == DECODE_ERROR) {
++it_test;
passed_error = true;
}
if (it_test == end_test) {
break;
}
int skipped_frames = 0;
++it_ref;
while (it_ref != end_ref && it_ref->first != it_test->first) {
skipped_frames += it_ref->second;
++it_ref;
for (; it_ref != end_ref; ++it_ref) {
if (it_ref->first != DECODE_ERROR && it_ref->first >= it_test->first) {
break;
}
++skipped_frames;
}
if (it_ref == end_ref) {
fprintf(stderr,
"The barcode in the test video is not in the reference video.\n");
return;
if (passed_error) {
// If we pass an error in the test video, then we are conservative
// and will not calculate skipped frames for that part.
skipped_frames = 0;
}
if (skipped_frames > max_skipped_frames) {
max_skipped_frames = skipped_frames;
if (it_ref != end_ref && it_ref->first == it_test->first) {
total_skipped_frames += skipped_frames;
if (skipped_frames > max_skipped_frames) {
max_skipped_frames = skipped_frames;
}
continue;
}
fprintf(stderr,
"Found barcode %d in test video, which is not in reference video",
it_test->first);
return;
}
fprintf(output, "RESULT Max_repeated: %s= %d\n", label.c_str(),
max_repeated_frames);
fprintf(output, "RESULT Max_skipped: %s= %d\n", label.c_str(),
max_skipped_frames);
fprintf(output, "RESULT Total_skipped: %s= %d\n", label.c_str(),
total_skipped_frames);
fprintf(output, "RESULT Decode_errors_reference: %s= %d\n", label.c_str(),
decode_errors_ref);
fprintf(output, "RESULT Decode_errors_test: %s= %d\n", label.c_str(),
decode_errors_test);
}
void PrintAnalysisResults(const std::string& label, ResultsContainer* results) {

18
webrtc/tools/frame_analyzer/video_quality_analysis.h
View File

@ -13,6 +13,7 @@
#include <string>
#include <vector>
#include <utility>
#include "libyuv/compare.h" // NOLINT
#include "libyuv/convert.h" // NOLINT
@ -82,6 +83,23 @@ void PrintAnalysisResults(const std::string& label, ResultsContainer* results);
void PrintAnalysisResults(FILE* output, const std::string& label,
ResultsContainer* results);
// The barcode number that means that the barcode could not be decoded.
const int DECODE_ERROR = -1;
// Clusters the frames in the file. First in the pair is the frame number and
// second is the number of frames in that cluster. So if first frame in video
// has number 100 and it is repeated 3 after each other, then the first entry
// in the returned vector has first set to 100 and second set to 3.
// Decode errors between two frames with same barcode, then it interprets
// the frame with the decode error as having the same id as the two frames
// around it. Eg. [400, DECODE_ERROR, DECODE_ERROR, 400] is becomes an entry
// in return vector with first==400 and second==4. In other cases with decode
// errors like [400, DECODE_ERROR, 401] becomes three entries, each with
// second==1 and the middle has first==DECODE_ERROR.
std::vector<std::pair<int, int> > CalculateFrameClusters(
FILE* file,
int* num_decode_errors);
// Calculates max repeated and skipped frames and prints them to stdout in a
// format that is compatible with Chromium performance numbers.
void PrintMaxRepeatedAndSkippedFrames(const std::string& label,

249
webrtc/tools/frame_analyzer/video_quality_analysis_unittest.cc
View File

@ -26,18 +26,20 @@ namespace test {
// want those numbers to be picked up as perf numbers.
class VideoQualityAnalysisTest : public ::testing::Test {
protected:
static void SetUpTestCase() {
std::string log_filename = webrtc::test::OutputPath() +
"VideoQualityAnalysisTest.log";
void SetUp() {
std::string log_filename = TempFilename(webrtc::test::OutputPath(),
"VideoQualityAnalysisTest.log");
logfile_ = fopen(log_filename.c_str(), "w");
ASSERT_TRUE(logfile_ != NULL);
stats_filename_ref_ = TempFilename(OutputPath(), "stats-1.txt");
stats_filename_ = TempFilename(OutputPath(), "stats-2.txt");
}
static void TearDownTestCase() {
ASSERT_EQ(0, fclose(logfile_));
}
static FILE* logfile_;
void TearDown() { ASSERT_EQ(0, fclose(logfile_)); }
FILE* logfile_;
std::string stats_filename_ref_;
std::string stats_filename_;
};
FILE* VideoQualityAnalysisTest::logfile_ = NULL;
TEST_F(VideoQualityAnalysisTest, MatchExtractedY4mFrame) {
std::string video_file =
@ -85,33 +87,26 @@ TEST_F(VideoQualityAnalysisTest, PrintAnalysisResultsThreeFrames) {
}
TEST_F(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesInvalidFile) {
std::string stats_filename_ref =
OutputPath() + "non-existing-stats-file-1.txt";
std::string stats_filename = OutputPath() + "non-existing-stats-file-2.txt";
remove(stats_filename.c_str());
remove(stats_filename_.c_str());
PrintMaxRepeatedAndSkippedFrames(logfile_, "NonExistingStatsFile",
stats_filename_ref, stats_filename);
stats_filename_ref_, stats_filename_);
}
TEST_F(VideoQualityAnalysisTest,
PrintMaxRepeatedAndSkippedFramesEmptyStatsFile) {
std::string stats_filename_ref = OutputPath() + "empty-stats-1.txt";
std::string stats_filename = OutputPath() + "empty-stats-2.txt";
std::ofstream stats_file;
stats_file.open(stats_filename_ref.c_str());
stats_file.open(stats_filename_ref_.c_str());
stats_file.close();
stats_file.open(stats_filename.c_str());
stats_file.open(stats_filename_.c_str());
stats_file.close();
PrintMaxRepeatedAndSkippedFrames(logfile_, "EmptyStatsFile",
stats_filename_ref, stats_filename);
stats_filename_ref_, stats_filename_);
}
TEST_F(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesNormalFile) {
std::string stats_filename_ref = OutputPath() + "stats-1.txt";
std::string stats_filename = OutputPath() + "stats-2.txt";
std::ofstream stats_file;
stats_file.open(stats_filename_ref.c_str());
stats_file.open(stats_filename_ref_.c_str());
stats_file << "frame_0001 0100\n";
stats_file << "frame_0002 0101\n";
stats_file << "frame_0003 0102\n";
@ -121,7 +116,7 @@ TEST_F(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesNormalFile) {
stats_file << "frame_0007 0108\n";
stats_file.close();
stats_file.open(stats_filename.c_str());
stats_file.open(stats_filename_.c_str());
stats_file << "frame_0001 0100\n";
stats_file << "frame_0002 0101\n";
stats_file << "frame_0003 0101\n";
@ -129,9 +124,217 @@ TEST_F(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesNormalFile) {
stats_file.close();
PrintMaxRepeatedAndSkippedFrames(logfile_, "NormalStatsFile",
stats_filename_ref, stats_filename);
stats_filename_ref_, stats_filename_);
}
namespace {
void VerifyLogOutput(const std::string& log_filename,
const std::vector<std::string>& expected_out) {
std::ifstream logf(log_filename);
std::string line;
std::size_t i;
for (i = 0; i < expected_out.size() && getline(logf, line); ++i) {
ASSERT_EQ(expected_out.at(i), line);
}
ASSERT_TRUE(i == expected_out.size()) << "Not enough input data";
}
} // unnamed namespace
TEST_F(VideoQualityAnalysisTest,
PrintMaxRepeatedAndSkippedFramesSkippedFrames) {
std::ofstream stats_file;
std::string log_filename =
TempFilename(webrtc::test::OutputPath(), "log.log");
FILE* logfile = fopen(log_filename.c_str(), "w");
ASSERT_TRUE(logfile != NULL);
stats_file.open(stats_filename_ref_.c_str());
stats_file << "frame_0001 0100\n";
stats_file << "frame_0002 0101\n";
stats_file << "frame_0002 0101\n";
stats_file << "frame_0003 0103\n";
stats_file << "frame_0004 0103\n";
stats_file << "frame_0005 0106\n";
stats_file << "frame_0006 0106\n";
stats_file << "frame_0007 0108\n";
stats_file << "frame_0008 0110\n";
stats_file << "frame_0009 0112\n";
stats_file.close();
stats_file.open(stats_filename_.c_str());
stats_file << "frame_0001 0101\n";
stats_file << "frame_0002 0101\n";
stats_file << "frame_0003 0101\n";
stats_file << "frame_0004 0108\n";
stats_file << "frame_0005 0108\n";
stats_file << "frame_0006 0112\n";
stats_file.close();
PrintMaxRepeatedAndSkippedFrames(logfile, "NormalStatsFile",
stats_filename_ref_, stats_filename_);
ASSERT_EQ(0, fclose(logfile));
std::vector<std::string> expected_out = {
"RESULT Max_repeated: NormalStatsFile= 2",
"RESULT Max_skipped: NormalStatsFile= 2",
"RESULT Total_skipped: NormalStatsFile= 3",
"RESULT Decode_errors_reference: NormalStatsFile= 0",
"RESULT Decode_errors_test: NormalStatsFile= 0"};
VerifyLogOutput(log_filename, expected_out);
}
TEST_F(VideoQualityAnalysisTest,
PrintMaxRepeatedAndSkippedFramesDecodeErrorInTest) {
std::ofstream stats_file;
std::string log_filename =
TempFilename(webrtc::test::OutputPath(), "log.log");
FILE* logfile = fopen(log_filename.c_str(), "w");
ASSERT_TRUE(logfile != NULL);
stats_file.open(stats_filename_ref_.c_str());
stats_file << "frame_0001 0100\n";
stats_file << "frame_0002 0100\n";
stats_file << "frame_0002 0101\n";
stats_file << "frame_0003 0103\n";
stats_file << "frame_0004 0103\n";
stats_file << "frame_0005 0106\n";
stats_file << "frame_0006 0107\n";
stats_file << "frame_0007 0107\n";
stats_file << "frame_0008 0110\n";
stats_file << "frame_0009 0112\n";
stats_file.close();
stats_file.open(stats_filename_.c_str());
stats_file << "frame_0001 0101\n";
stats_file << "frame_0002 Barcode error\n";
stats_file << "frame_0003 Barcode error\n";
stats_file << "frame_0004 Barcode error\n";
stats_file << "frame_0005 0107\n";
stats_file << "frame_0006 0110\n";
stats_file.close();
PrintMaxRepeatedAndSkippedFrames(logfile, "NormalStatsFile",
stats_filename_ref_, stats_filename_);
ASSERT_EQ(0, fclose(logfile));
std::vector<std::string> expected_out = {
"RESULT Max_repeated: NormalStatsFile= 1",
"RESULT Max_skipped: NormalStatsFile= 0",
"RESULT Total_skipped: NormalStatsFile= 0",
"RESULT Decode_errors_reference: NormalStatsFile= 0",
"RESULT Decode_errors_test: NormalStatsFile= 3"};
VerifyLogOutput(log_filename, expected_out);
}
TEST_F(VideoQualityAnalysisTest, CalculateFrameClustersOneValue) {
std::ofstream stats_file;
stats_file.open(stats_filename_.c_str());
stats_file << "frame_0001 0101\n";
stats_file.close();
FILE* stats_filef = fopen(stats_filename_.c_str(), "r");
ASSERT_TRUE(stats_filef != NULL);
auto clusters = CalculateFrameClusters(stats_filef, nullptr);
ASSERT_EQ(0, fclose(stats_filef));
decltype(clusters) expected = {std::make_pair(101, 1)};
ASSERT_EQ(expected, clusters);
}
TEST_F(VideoQualityAnalysisTest, CalculateFrameClustersOneOneTwo) {
std::ofstream stats_file;
stats_file.open(stats_filename_.c_str());
stats_file << "frame_0001 0101\n";
stats_file << "frame_0002 0101\n";
stats_file << "frame_0003 0102\n";
stats_file.close();
FILE* stats_filef = fopen(stats_filename_.c_str(), "r");
ASSERT_TRUE(stats_filef != NULL);
auto clusters = CalculateFrameClusters(stats_filef, nullptr);
ASSERT_EQ(0, fclose(stats_filef));
decltype(clusters) expected = {std::make_pair(101, 2),
std::make_pair(102, 1)};
ASSERT_EQ(expected, clusters);
}
TEST_F(VideoQualityAnalysisTest, CalculateFrameClustersOneOneErrErrThree) {
std::ofstream stats_file;
stats_file.open(stats_filename_.c_str());
stats_file << "frame_0001 0101\n";
stats_file << "frame_0002 0101\n";
stats_file << "frame_0003 Barcode error\n";
stats_file << "frame_0004 Barcode error\n";
stats_file << "frame_0005 0103\n";
stats_file.close();
FILE* stats_filef = fopen(stats_filename_.c_str(), "r");
ASSERT_TRUE(stats_filef != NULL);
auto clusters = CalculateFrameClusters(stats_filef, nullptr);
ASSERT_EQ(0, fclose(stats_filef));
decltype(clusters) expected = {std::make_pair(101, 2),
std::make_pair(DECODE_ERROR, 2),
std::make_pair(103, 1)};
ASSERT_EQ(expected, clusters);
}
TEST_F(VideoQualityAnalysisTest, CalculateFrameClustersErrErr) {
std::ofstream stats_file;
stats_file.open(stats_filename_.c_str());
stats_file << "frame_0001 Barcode error\n";
stats_file << "frame_0002 Barcode error\n";
stats_file.close();
FILE* stats_filef = fopen(stats_filename_.c_str(), "r");
ASSERT_TRUE(stats_filef != NULL);
auto clusters = CalculateFrameClusters(stats_filef, nullptr);
ASSERT_EQ(0, fclose(stats_filef));
decltype(clusters) expected = {std::make_pair(DECODE_ERROR, 2)};
ASSERT_EQ(expected, clusters);
}
TEST_F(VideoQualityAnalysisTest, CalculateFrameClustersOneOneErrErrOneOne) {
std::ofstream stats_file;
stats_file.open(stats_filename_.c_str());
stats_file << "frame_0001 0101\n";
stats_file << "frame_0002 0101\n";
stats_file << "frame_0003 Barcode error\n";
stats_file << "frame_0004 Barcode error\n";
stats_file << "frame_0005 0101\n";
stats_file << "frame_0006 0101\n";
stats_file.close();
FILE* stats_filef = fopen(stats_filename_.c_str(), "r");
ASSERT_TRUE(stats_filef != NULL);
auto clusters = CalculateFrameClusters(stats_filef, nullptr);
ASSERT_EQ(0, fclose(stats_filef));
decltype(clusters) expected = {std::make_pair(101, 6)};
ASSERT_EQ(expected, clusters);
}
TEST_F(VideoQualityAnalysisTest, CalculateFrameClustersEmpty) {
std::ofstream stats_file;
stats_file.open(stats_filename_.c_str());
stats_file.close();
FILE* stats_filef = fopen(stats_filename_.c_str(), "r");
ASSERT_TRUE(stats_filef != NULL);
auto clusters = CalculateFrameClusters(stats_filef, nullptr);
ASSERT_EQ(0, fclose(stats_filef));
decltype(clusters) expected;
ASSERT_EQ(expected, clusters);
}
} // namespace test
} // namespace webrtc