Introduce possibility to poll stats and notify analyzers.

This CL introduces the possibility to poll the 2 peer connections
at constant intervals.

It also introduces a dummy AudioQualityAnalyzer that will have to
be implemented in a follow-up CL and it moves every type of the
test framework inside the webrtc::test namespace.

Bug: webrtc:10138
Change-Id: I40acf7894bd67ea5229baba2d2cf18cd8ef65e67
Reviewed-on: https://webrtc-review.googlesource.com/c/123441
Commit-Queue: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26854}
This commit is contained in:
Mirko Bonadei
2019-02-26 15:19:07 +01:00
committed by Commit Bot
parent 2684ab3db0
commit 12ae4f4d50
17 changed files with 339 additions and 12 deletions

View File

@ -151,8 +151,10 @@ if (rtc_include_tests) {
"../../../api/video_codecs:video_codecs_api",
"../../../test:video_test_common",
"../../../test:video_test_support",
"api:stats_observer_interface",
"api:video_quality_analyzer_api",
"//third_party/abseil-cpp/absl/memory:memory",
"//third_party/abseil-cpp/absl/strings",
]
}
@ -205,8 +207,10 @@ if (rtc_include_tests) {
"peer_connection_quality_test.h",
]
deps = [
":default_audio_quality_analyzer",
":example_video_quality_analyzer",
":single_process_encoded_image_data_injector",
":stats_poller",
":test_peer",
":video_quality_analyzer_injection_helper",
"../../../api:libjingle_peerconnection_api",
@ -220,6 +224,7 @@ if (rtc_include_tests) {
"../../../rtc_base:rtc_base_approved",
"../../../rtc_base:rtc_task_queue",
"../../../rtc_base:safe_conversions",
"../../../rtc_base/task_utils:repeating_task",
"../../../system_wrappers:system_wrappers",
"../../../test:fileutils",
"../../../test:video_test_support",
@ -265,6 +270,7 @@ if (rtc_include_tests) {
"peer_connection_e2e_smoke_test.cc",
]
deps = [
":default_audio_quality_analyzer",
":default_video_quality_analyzer",
"../../../api:callfactory_api",
"../../../api:libjingle_peerconnection_api",
@ -299,6 +305,41 @@ if (rtc_include_tests) {
suppressed_configs += [ "//build/config/clang:find_bad_constructs" ]
}
}
rtc_source_set("stats_poller") {
testonly = true
sources = [
"stats_poller.cc",
"stats_poller.h",
]
deps = [
":test_peer",
"../../../api:libjingle_peerconnection_api",
"../../../rtc_base:logging",
"api:stats_observer_interface",
]
if (!build_with_chromium && is_clang) {
# Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163).
suppressed_configs += [ "//build/config/clang:find_bad_constructs" ]
}
}
}
rtc_source_set("default_audio_quality_analyzer") {
visibility = [ "*" ]
testonly = true
sources = [
"analyzer/audio/default_audio_quality_analyzer.cc",
"analyzer/audio/default_audio_quality_analyzer.h",
]
deps = [
"../../../api:libjingle_peerconnection_api",
"../../../rtc_base:logging",
"api:audio_quality_analyzer_api",
"api:stats_observer_interface",
"//third_party/abseil-cpp/absl/strings",
]
}
rtc_source_set("example_video_quality_analyzer") {

View File

@ -0,0 +1,25 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h"
#include "rtc_base/logging.h"
namespace webrtc {
namespace test {
void DefaultAudioQualityAnalyzer::OnStatsReports(
absl::string_view pc_label,
const StatsReports& stats_reports) {
// TODO(bugs.webrtc.org/10138): Implement audio stats collection.
}
} // namespace test
} // namespace webrtc

View File

@ -0,0 +1,30 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef TEST_PC_E2E_ANALYZER_AUDIO_DEFAULT_AUDIO_QUALITY_ANALYZER_H_
#define TEST_PC_E2E_ANALYZER_AUDIO_DEFAULT_AUDIO_QUALITY_ANALYZER_H_
#include "absl/strings/string_view.h"
#include "api/stats_types.h"
#include "test/pc/e2e/api/audio_quality_analyzer_interface.h"
namespace webrtc {
namespace test {
class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface {
public:
void OnStatsReports(absl::string_view pc_label,
const StatsReports& stats_reports) override;
};
} // namespace test
} // namespace webrtc
#endif // TEST_PC_E2E_ANALYZER_AUDIO_DEFAULT_AUDIO_QUALITY_ANALYZER_H_

View File

@ -149,6 +149,9 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
std::map<std::string, StreamStats> GetStats() const;
AnalyzerStats GetAnalyzerStats() const;
// TODO(bugs.webrtc.org/10138): Provide a real implementation for
// OnStatsReport.
private:
struct FrameStats {
FrameStats(std::string stream_label, Timestamp captured_time);

View File

@ -137,6 +137,12 @@ void VideoQualityAnalyzerInjectionHelper::Start(int max_threads_count) {
analyzer_->Start(max_threads_count);
}
void VideoQualityAnalyzerInjectionHelper::OnStatsReports(
absl::string_view pc_label,
const StatsReports& stats_reports) {
analyzer_->OnStatsReports(pc_label, stats_reports);
}
void VideoQualityAnalyzerInjectionHelper::Stop() {
analyzer_->Stop();
}

View File

@ -15,6 +15,7 @@
#include <memory>
#include <string>
#include "absl/strings/string_view.h"
#include "api/video/video_frame.h"
#include "api/video/video_sink_interface.h"
#include "api/video_codecs/video_decoder_factory.h"
@ -22,6 +23,7 @@
#include "test/frame_generator.h"
#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h"
#include "test/pc/e2e/analyzer/video/id_generator.h"
#include "test/pc/e2e/api/stats_observer_interface.h"
#include "test/pc/e2e/api/video_quality_analyzer_interface.h"
#include "test/testsupport/video_frame_writer.h"
@ -30,13 +32,13 @@ namespace test {
// Provides factory methods for components, that will be used to inject
// VideoQualityAnalyzerInterface into PeerConnection pipeline.
class VideoQualityAnalyzerInjectionHelper {
class VideoQualityAnalyzerInjectionHelper : public StatsObserverInterface {
public:
VideoQualityAnalyzerInjectionHelper(
std::unique_ptr<VideoQualityAnalyzerInterface> analyzer,
EncodedImageDataInjector* injector,
EncodedImageDataExtractor* extractor);
~VideoQualityAnalyzerInjectionHelper();
~VideoQualityAnalyzerInjectionHelper() override;
// Wraps video encoder factory to give video quality analyzer access to frames
// before encoding and encoded images after.
@ -64,6 +66,11 @@ class VideoQualityAnalyzerInjectionHelper {
void Start(int max_threads_count);
// Forwards |stats_reports| for Peer Connection |pc_label| to
// |analyzer_|.
void OnStatsReports(absl::string_view pc_label,
const StatsReports& stats_reports) override;
// Stops VideoQualityAnalyzerInterface to populate final data and metrics.
void Stop();

View File

@ -15,9 +15,11 @@ rtc_source_set("video_quality_analyzer_api") {
]
deps = [
":stats_observer_interface",
"../../../../api/video:encoded_image",
"../../../../api/video:video_frame",
"../../../../api/video_codecs:video_codecs_api",
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
]
}
@ -28,7 +30,21 @@ rtc_source_set("audio_quality_analyzer_api") {
"audio_quality_analyzer_interface.h",
]
deps = []
deps = [
":stats_observer_interface",
]
}
rtc_source_set("stats_observer_interface") {
visibility = [ "*" ]
sources = [
"stats_observer_interface.h",
]
deps = [
"../../../../api:libjingle_peerconnection_api",
"//third_party/abseil-cpp/absl/strings",
]
}
rtc_source_set("peer_connection_quality_test_fixture_api") {

View File

@ -11,13 +11,17 @@
#ifndef TEST_PC_E2E_API_AUDIO_QUALITY_ANALYZER_INTERFACE_H_
#define TEST_PC_E2E_API_AUDIO_QUALITY_ANALYZER_INTERFACE_H_
namespace webrtc {
#include "test/pc/e2e/api/stats_observer_interface.h"
class AudioQualityAnalyzerInterface {
namespace webrtc {
namespace test {
class AudioQualityAnalyzerInterface : public StatsObserverInterface {
public:
virtual ~AudioQualityAnalyzerInterface() = default;
~AudioQualityAnalyzerInterface() override = default;
};
} // namespace test
} // namespace webrtc
#endif // TEST_PC_E2E_API_AUDIO_QUALITY_ANALYZER_INTERFACE_H_

View File

@ -16,6 +16,7 @@
#include "test/pc/e2e/peer_connection_quality_test.h"
namespace webrtc {
namespace test {
std::unique_ptr<PeerConnectionE2EQualityTestFixture>
CreatePeerConnectionE2EQualityTestFixture(
@ -25,4 +26,5 @@ CreatePeerConnectionE2EQualityTestFixture(
std::move(audio_quality_analyzer), std::move(video_quality_analyzer));
}
} // namespace test
} // namespace webrtc

View File

@ -17,6 +17,7 @@
#include "test/pc/e2e/api/video_quality_analyzer_interface.h"
namespace webrtc {
namespace test {
// API is in development. Can be changed/removed without notice.
// Create test fixture to establish test call between Alice and Bob.
@ -26,6 +27,7 @@ CreatePeerConnectionE2EQualityTestFixture(
std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer,
std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer);
} // namespace test
} // namespace webrtc
#endif // TEST_PC_E2E_API_CREATE_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_

View File

@ -0,0 +1,33 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef TEST_PC_E2E_API_STATS_OBSERVER_INTERFACE_H_
#define TEST_PC_E2E_API_STATS_OBSERVER_INTERFACE_H_
#include "absl/strings/string_view.h"
#include "api/stats_types.h"
namespace webrtc {
namespace test {
class StatsObserverInterface {
public:
virtual ~StatsObserverInterface() = default;
// Method called when stats reports are available for the PeerConnection
// identified by |pc_label|.
virtual void OnStatsReports(absl::string_view pc_label,
const StatsReports& reports) = 0;
};
} // namespace test
} // namespace webrtc
#endif // TEST_PC_E2E_API_STATS_OBSERVER_INTERFACE_H_

View File

@ -14,12 +14,15 @@
#include <memory>
#include <string>
#include "absl/strings/string_view.h"
#include "absl/types/optional.h"
#include "api/video/encoded_image.h"
#include "api/video/video_frame.h"
#include "api/video_codecs/video_encoder.h"
#include "test/pc/e2e/api/stats_observer_interface.h"
namespace webrtc {
namespace test {
// Base interface for video quality analyzer for peer connection level end-2-end
// tests. Interface has only one abstract method, which have to return frame id.
@ -47,9 +50,9 @@ namespace webrtc {
// | Sink | | Stack | | Decoder |
// ¯¯¯¯¯¯¯ ¯¯¯¯¯¯¯¯ ¯¯¯¯¯¯¯¯¯
// The analyzer will be injected in all points from A to F.
class VideoQualityAnalyzerInterface {
class VideoQualityAnalyzerInterface : public StatsObserverInterface {
public:
virtual ~VideoQualityAnalyzerInterface() = default;
~VideoQualityAnalyzerInterface() override = default;
// Will be called by framework before test. |threads_count| is number of
// threads that analyzer can use for heavy calculations. Analyzer can perform
@ -89,6 +92,10 @@ class VideoQualityAnalyzerInterface {
// All available codes are listed in
// modules/video_coding/include/video_error_codes.h
virtual void OnDecoderError(uint16_t frame_id, int32_t error_code) {}
// Will be called everytime new stats reports are available for the
// Peer Connection identified by |pc_label|.
void OnStatsReports(absl::string_view pc_label,
const StatsReports& stats_reports) override {}
// Tells analyzer that analysis complete and it should calculate final
// statistics.
@ -97,6 +104,7 @@ class VideoQualityAnalyzerInterface {
virtual std::string GetStreamLabel(uint16_t frame_id) = 0;
};
} // namespace test
} // namespace webrtc
#endif // TEST_PC_E2E_API_VIDEO_QUALITY_ANALYZER_INTERFACE_H_

View File

@ -16,6 +16,7 @@
#include "rtc_base/async_invoker.h"
#include "rtc_base/fake_network.h"
#include "test/gtest.h"
#include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
#include "test/pc/e2e/api/create_peerconnection_quality_test_fixture.h"
#include "test/pc/e2e/api/peerconnection_quality_test_fixture.h"
@ -105,8 +106,11 @@ TEST(PeerConnectionE2EQualityTestSmokeTest, RunWithEmulatedNetwork) {
auto* video_analyzer_ptr =
static_cast<DefaultVideoQualityAnalyzer*>(video_quality_analyzer.get());
std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer =
absl::make_unique<DefaultAudioQualityAnalyzer>();
auto fixture = CreatePeerConnectionE2EQualityTestFixture(
nullptr, std::move(video_quality_analyzer));
std::move(audio_quality_analyzer), std::move(video_quality_analyzer));
fixture->Run(std::move(alice_components), std::move(alice_params),
std::move(bob_components), absl::make_unique<Params>(),
RunParams{TimeDelta::seconds(5)});

View File

@ -27,6 +27,7 @@
#include "system_wrappers/include/cpu_info.h"
#include "test/pc/e2e/analyzer/video/example_video_quality_analyzer.h"
#include "test/pc/e2e/api/video_quality_analyzer_interface.h"
#include "test/pc/e2e/stats_poller.h"
#include "test/testsupport/file_utils.h"
namespace webrtc {
@ -44,6 +45,9 @@ constexpr int kPeerConnectionUsedThreads = 7;
constexpr int kFrameworkUsedThreads = 2;
constexpr int kMaxVideoAnalyzerThreads = 8;
constexpr TimeDelta kStatsUpdateInterval = TimeDelta::Seconds<1>();
constexpr TimeDelta kStatsPollingStopTimeout = TimeDelta::Seconds<1>();
std::string VideoConfigSourcePresenceToString(const VideoConfig& video_config) {
char buf[1024];
rtc::SimpleStringBuilder builder(buf);
@ -108,6 +112,11 @@ PeerConnectionE2EQualityTest::PeerConnectionE2EQualityTest(
absl::make_unique<VideoQualityAnalyzerInjectionHelper>(
std::move(video_quality_analyzer), encoded_image_id_controller_.get(),
encoded_image_id_controller_.get());
if (audio_quality_analyzer == nullptr) {
audio_quality_analyzer = absl::make_unique<DefaultAudioQualityAnalyzer>();
}
audio_quality_analyzer_.swap(audio_quality_analyzer);
}
void PeerConnectionE2EQualityTest::Run(
@ -206,10 +215,31 @@ void PeerConnectionE2EQualityTest::Run(
rtc::Bind(&PeerConnectionE2EQualityTest::SetupCallOnSignalingThread,
this));
// TODO(bugs.webrtc.org/10138): Implement stats collection and send stats
// reports to analyzers every 1 second.
StatsPoller stats_poller({audio_quality_analyzer_.get(),
video_quality_analyzer_injection_helper_.get()},
{{"alice", alice_.get()}, {"bob", bob_.get()}});
task_queue_.PostTask([&stats_poller, this]() {
RTC_DCHECK_RUN_ON(&task_queue_);
stats_polling_task_ = RepeatingTaskHandle::Start([this, &stats_poller]() {
RTC_DCHECK_RUN_ON(&task_queue_);
stats_poller.PollStatsAndNotifyObservers();
return kStatsUpdateInterval;
});
});
rtc::Event done;
done.Wait(rtc::checked_cast<int>(run_params.run_duration.ms()));
done.Wait(run_params.run_duration.ms());
rtc::Event stats_polling_stopped;
task_queue_.PostTask([&stats_polling_stopped, this]() {
RTC_DCHECK_RUN_ON(&task_queue_);
stats_polling_task_.Stop();
stats_polling_stopped.Set();
});
bool no_timeout = stats_polling_stopped.Wait(kStatsPollingStopTimeout.ms());
RTC_CHECK(no_timeout) << "Failed to stop Stats polling after "
<< kStatsPollingStopTimeout.seconds() << " seconds.";
signaling_thread->Invoke<void>(
RTC_FROM_HERE,

View File

@ -16,8 +16,11 @@
#include "pc/test/frame_generator_capturer_video_track_source.h"
#include "rtc_base/task_queue.h"
#include "rtc_base/task_utils/repeating_task.h"
#include "rtc_base/thread.h"
#include "rtc_base/thread_annotations.h"
#include "system_wrappers/include/clock.h"
#include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h"
#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h"
#include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h"
#include "test/pc/e2e/api/peerconnection_quality_test_fixture.h"
@ -83,6 +86,7 @@ class PeerConnectionE2EQualityTest
video_quality_analyzer_injection_helper_;
std::unique_ptr<SingleProcessEncodedImageDataInjector>
encoded_image_id_controller_;
std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer_;
std::unique_ptr<TestPeer> alice_;
std::unique_ptr<TestPeer> bob_;
@ -95,6 +99,7 @@ class PeerConnectionE2EQualityTest
std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>
output_video_sinks_;
RepeatingTaskHandle stats_polling_task_ RTC_GUARDED_BY(&task_queue_);
// Must be the last field, so it will be deleted first, because tasks
// in the TaskQueue can access other fields of the instance of this class.
rtc::TaskQueue task_queue_;

View File

@ -0,0 +1,47 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "test/pc/e2e/stats_poller.h"
#include <utility>
#include "rtc_base/logging.h"
namespace webrtc {
namespace test {
void InternalStatsObserver::PollStats() {
peer_->pc()->GetStats(this, nullptr,
webrtc::PeerConnectionInterface::StatsOutputLevel::
kStatsOutputLevelStandard);
}
void InternalStatsObserver::OnComplete(const StatsReports& reports) {
for (auto* observer : observers_) {
observer->OnStatsReports(pc_label_, reports);
}
}
StatsPoller::StatsPoller(std::vector<StatsObserverInterface*> observers,
std::map<std::string, TestPeer*> peers) {
for (auto& peer : peers) {
pollers_.push_back(new rtc::RefCountedObject<InternalStatsObserver>(
peer.first, peer.second, observers));
}
}
void StatsPoller::PollStatsAndNotifyObservers() {
for (auto& poller : pollers_) {
poller->PollStats();
}
}
} // namespace test
} // namespace webrtc

View File

@ -0,0 +1,64 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef TEST_PC_E2E_STATS_POLLER_H_
#define TEST_PC_E2E_STATS_POLLER_H_
#include <map>
#include <string>
#include <utility>
#include <vector>
#include "api/peer_connection_interface.h"
#include "test/pc/e2e/api/stats_observer_interface.h"
#include "test/pc/e2e/test_peer.h"
namespace webrtc {
namespace test {
// Helper class that will notify all the webrtc::test::StatsObserverInterface
// objects subscribed.
class InternalStatsObserver : public StatsObserver {
public:
InternalStatsObserver(std::string pc_label,
TestPeer* peer,
std::vector<StatsObserverInterface*> observers)
: pc_label_(std::move(pc_label)),
peer_(peer),
observers_(std::move(observers)) {}
void PollStats();
void OnComplete(const StatsReports& reports) override;
private:
std::string pc_label_;
TestPeer* peer_;
std::vector<StatsObserverInterface*> observers_;
};
// Helper class to invoke GetStats on a PeerConnection by passing a
// webrtc::StatsObserver that will notify all the
// webrtc::test::StatsObserverInterface subscribed.
class StatsPoller {
public:
StatsPoller(std::vector<StatsObserverInterface*> observers,
std::map<std::string, TestPeer*> peers_to_observe);
void PollStatsAndNotifyObservers();
private:
std::vector<rtc::scoped_refptr<InternalStatsObserver>> pollers_;
};
} // namespace test
} // namespace webrtc
#endif // TEST_PC_E2E_STATS_POLLER_H_