zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[DVQA] Remove sender from DVQA StatsKey

Browse Source 
Sender isn't actually require to identify the stream, so specifying it
every time is useless. This CL removes sender from StatsKey object and
introduces StreamsInfo object which contains all required metadata about
streams that are seen by DVQA.

Bug: b/205824594
Change-Id: I5b6be3865a30fd5980ff6e7e50906abe70a632ee
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/238562
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Per Kjellander <perkj@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35399}
This commit is contained in:
Artem Titov
2021-11-22 12:49:23 +01:00
committed by WebRTC LUCI CQ
parent f002e2fc90
commit 8ef7da78b3
6 changed files with 181 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
View File

@ -676,6 +676,40 @@ std::set<StatsKey> DefaultVideoQualityAnalyzer::GetKnownVideoStreams() const {
return out;
}
VideoStreamsInfo DefaultVideoQualityAnalyzer::GetKnownStreams() const {
MutexLock lock(&mutex_);
std::map<std::string, std::string> stream_to_sender;
std::map<std::string, std::set<std::string>> sender_to_streams;
std::map<std::string, std::set<std::string>> stream_to_receivers;
for (auto& item : frames_comparator_.stream_stats()) {
const std::string& stream_label = streams_.name(item.first.stream);
const std::string& sender = peers_->name(item.first.sender);
const std::string& receiver = peers_->name(item.first.receiver);
RTC_LOG(LS_INFO) << item.first.ToString() << " ==> "
<< "stream=" << stream_label << "; sender=" << sender
<< "; receiver=" << receiver;
stream_to_sender.emplace(stream_label, sender);
auto streams_it = sender_to_streams.find(sender);
if (streams_it != sender_to_streams.end()) {
streams_it->second.emplace(stream_label);
} else {
sender_to_streams.emplace(sender, std::set<std::string>{stream_label});
}
auto receivers_it = stream_to_receivers.find(stream_label);
if (receivers_it != stream_to_receivers.end()) {
receivers_it->second.emplace(receiver);
} else {
stream_to_receivers.emplace(stream_label,
std::set<std::string>{receiver});
}
}
return VideoStreamsInfo(std::move(stream_to_sender),
std::move(sender_to_streams),
std::move(stream_to_receivers));
}
const FrameCounters& DefaultVideoQualityAnalyzer::GetGlobalCounters() const {
MutexLock lock(&mutex_);
return frame_counters_;
@ -710,8 +744,8 @@ void DefaultVideoQualityAnalyzer::ReportResults() {
MutexLock lock(&mutex_);
for (auto& item : frames_comparator_.stream_stats()) {
ReportResults(GetTestCaseName(StatsKeyToMetricName(ToStatsKey(item.first))),
item.second, stream_frame_counters_.at(item.first));
ReportResults(GetTestCaseName(ToMetricName(item.first)), item.second,
stream_frame_counters_.at(item.first));
}
test::PrintResult("cpu_usage", "", test_label_.c_str(), GetCpuUsagePercent(),
"%", false, ImproveDirection::kSmallerIsBetter);
@ -855,16 +889,19 @@ Timestamp DefaultVideoQualityAnalyzer::Now() {
StatsKey DefaultVideoQualityAnalyzer::ToStatsKey(
const InternalStatsKey& key) const {
return StatsKey(streams_.name(key.stream), peers_->name(key.sender),
peers_->name(key.receiver));
return StatsKey(streams_.name(key.stream), peers_->name(key.receiver));
}
std::string DefaultVideoQualityAnalyzer::StatsKeyToMetricName(
const StatsKey& key) const {
std::string DefaultVideoQualityAnalyzer::ToMetricName(
const InternalStatsKey& key) const {
const std::string& stream_label = streams_.name(key.stream);
if (peers_->size() <= 2 && key.sender != key.receiver) {
return key.stream_label;
return stream_label;
}
return key.ToString();
rtc::StringBuilder out;
out << stream_label << "_" << peers_->name(key.sender) << "_"
<< peers_->name(key.receiver);
return out.str();
}
double DefaultVideoQualityAnalyzer::GetCpuUsagePercent() {

3
test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
View File

@ -82,6 +82,7 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
// Returns set of stream labels, that were met during test call.
std::set<StatsKey> GetKnownVideoStreams() const;
VideoStreamsInfo GetKnownStreams() const;
const FrameCounters& GetGlobalCounters() const;
// Returns frame counter per stream label. Valid stream labels can be obtained
// by calling GetKnownVideoStreams()
@ -322,7 +323,7 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
// Returns string representation of stats key for metrics naming. Used for
// backward compatibility by metrics naming for 2 peers cases.
std::string StatsKeyToMetricName(const StatsKey& key) const
std::string ToMetricName(const InternalStatsKey& key) const
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
const DefaultVideoQualityAnalyzerOptions options_;

64
test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc
View File

@ -9,6 +9,9 @@
*/
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
#include <algorithm>
#include <iterator>
#include "api/units/timestamp.h"
#include "rtc_base/checks.h"
#include "rtc_base/strings/string_builder.h"
@ -47,7 +50,7 @@ StreamStats::StreamStats(Timestamp stream_started_time)
std::string StatsKey::ToString() const {
rtc::StringBuilder out;
out << stream_label << "_" << sender << "_" << receiver;
out << stream_label << "_" << receiver;
return out.str();
}
@ -55,15 +58,64 @@ bool operator<(const StatsKey& a, const StatsKey& b) {
if (a.stream_label != b.stream_label) {
return a.stream_label < b.stream_label;
}
if (a.sender != b.sender) {
return a.sender < b.sender;
}
return a.receiver < b.receiver;
}
bool operator==(const StatsKey& a, const StatsKey& b) {
return a.stream_label == b.stream_label && a.sender == b.sender &&
a.receiver == b.receiver;
return a.stream_label == b.stream_label && a.receiver == b.receiver;
}
VideoStreamsInfo::VideoStreamsInfo(
std::map<std::string, std::string> stream_to_sender,
std::map<std::string, std::set<std::string>> sender_to_streams,
std::map<std::string, std::set<std::string>> stream_to_receivers)
: stream_to_sender_(std::move(stream_to_sender)),
sender_to_streams_(std::move(sender_to_streams)),
stream_to_receivers_(std::move(stream_to_receivers)) {}
std::set<StatsKey> VideoStreamsInfo::GetStatsKeys() const {
std::set<StatsKey> out;
for (const std::string& stream_label : GetStreams()) {
for (const std::string& receiver : GetReceivers(stream_label)) {
out.insert(StatsKey(stream_label, receiver));
}
}
return out;
}
std::set<std::string> VideoStreamsInfo::GetStreams() const {
std::set<std::string> out;
std::transform(stream_to_sender_.begin(), stream_to_sender_.end(),
std::inserter(out, out.end()),
[](auto map_entry) { return map_entry.first; });
return out;
}
std::set<std::string> VideoStreamsInfo::GetStreams(
absl::string_view sender_name) const {
auto it = sender_to_streams_.find(std::string(sender_name));
if (it == sender_to_streams_.end()) {
return {};
}
return it->second;
}
absl::optional<std::string> VideoStreamsInfo::GetSender(
absl::string_view stream_label) const {
auto it = stream_to_sender_.find(std::string(stream_label));
if (it == stream_to_sender_.end()) {
return absl::nullopt;
}
return it->second;
}
std::set<std::string> VideoStreamsInfo::GetReceivers(
absl::string_view stream_label) const {
auto it = stream_to_receivers_.find(std::string(stream_label));
if (it == stream_to_receivers_.end()) {
return {};
}
return it->second;
}
} // namespace webrtc

51
test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h
View File

@ -13,6 +13,7 @@
#include <map>
#include <memory>
#include <set>
#include <string>
#include <utility>
#include <vector>
@ -156,17 +157,19 @@ struct AnalyzerStats {
};
struct StatsKey {
StatsKey(std::string stream_label, std::string sender, std::string receiver)
: stream_label(std::move(stream_label)),
sender(std::move(sender)),
receiver(std::move(receiver)) {}
// Keep this constructor for temporary backward compatibility.
StatsKey(std::string stream_label,
std::string /*sender*/,
std::string receiver)
: stream_label(std::move(stream_label)), receiver(std::move(receiver)) {}
StatsKey(std::string stream_label, std::string receiver)
: stream_label(std::move(stream_label)), receiver(std::move(receiver)) {}
std::string ToString() const;
// Label of video stream to which stats belongs to.
std::string stream_label;
// Name of the peer which send this stream.
std::string sender;
// Name of the peer on which stream was received.
std::string receiver;
};
@ -175,6 +178,42 @@ struct StatsKey {
bool operator<(const StatsKey& a, const StatsKey& b);
bool operator==(const StatsKey& a, const StatsKey& b);
// Contains all metadata related to the video streams that were seen by the
// video analyzer.
class VideoStreamsInfo {
public:
std::set<StatsKey> GetStatsKeys() const;
// Returns all stream labels that are known to the video analyzer.
std::set<std::string> GetStreams() const;
// Returns set of the stream for specified `sender_name`. If sender didn't
// send any streams or `sender_name` isn't known to the video analyzer
// empty set will be returned.
std::set<std::string> GetStreams(absl::string_view sender_name) const;
// Returns sender name for specified `stream_label`. Returns `absl::nullopt`
// if provided `stream_label` isn't known to the video analyzer.
absl::optional<std::string> GetSender(absl::string_view stream_label) const;
// Returns set of the receivers for specified `stream_label`. If stream wasn't
// received by any peer or `stream_label` isn't known to the video analyzer
// empty set will be returned.
std::set<std::string> GetReceivers(absl::string_view stream_label) const;
protected:
friend class DefaultVideoQualityAnalyzer;
VideoStreamsInfo(
std::map<std::string, std::string> stream_to_sender,
std::map<std::string, std::set<std::string>> sender_to_streams,
std::map<std::string, std::set<std::string>> stream_to_receivers);
private:
std::map<std::string, std::string> stream_to_sender_;
std::map<std::string, std::set<std::string>> sender_to_streams_;
std::map<std::string, std::set<std::string>> stream_to_receivers_;
};
struct DefaultVideoQualityAnalyzerOptions {
// Tells DefaultVideoQualityAnalyzer if heavy metrics like PSNR and SSIM have
// to be computed or not.

49
test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc
View File

@ -472,8 +472,17 @@ TEST(DefaultVideoQualityAnalyzerTest, NormalScenario2Receivers) {
EXPECT_EQ(frame_counters.decoded, kMaxFramesInFlightPerStream);
EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream);
EXPECT_EQ(frame_counters.dropped, kMaxFramesInFlightPerStream);
EXPECT_EQ(analyzer.GetKnownVideoStreams().size(), 2lu);
for (auto stream_key : analyzer.GetKnownVideoStreams()) {
VideoStreamsInfo streams_info = analyzer.GetKnownStreams();
EXPECT_EQ(streams_info.GetStreams(), std::set<std::string>{kStreamLabel});
EXPECT_EQ(streams_info.GetStreams(kAlice),
std::set<std::string>{kStreamLabel});
EXPECT_EQ(streams_info.GetSender(kStreamLabel), kAlice);
EXPECT_EQ(streams_info.GetReceivers(kStreamLabel),
(std::set<std::string>{kBob, kCharlie}));
EXPECT_EQ(streams_info.GetStatsKeys().size(), 2lu);
for (auto stream_key : streams_info.GetStatsKeys()) {
FrameCounters stream_conters =
analyzer.GetPerStreamCounters().at(stream_key);
// On some devices the pipeline can be too slow, so we actually can't
@ -489,8 +498,8 @@ TEST(DefaultVideoQualityAnalyzerTest, NormalScenario2Receivers) {
}
std::map<StatsKey, StreamStats> stats = analyzer.GetStats();
const StatsKey kAliceBobStats(kStreamLabel, kAlice, kBob);
const StatsKey kAliceCharlieStats(kStreamLabel, kAlice, kCharlie);
const StatsKey kAliceBobStats(kStreamLabel, kBob);
const StatsKey kAliceCharlieStats(kStreamLabel, kCharlie);
EXPECT_EQ(stats.size(), 2lu);
{
auto it = stats.find(kAliceBobStats);
@ -629,8 +638,7 @@ TEST(DefaultVideoQualityAnalyzerTest, HeavyQualityMetricsFromEqualFrames) {
<< ToString(frames_in_flight_sizes);
std::map<StatsKey, StreamStats> stream_stats = analyzer.GetStats();
const StatsKey kAliceBobStats(kStreamLabel, kSenderPeerName,
kReceiverPeerName);
const StatsKey kAliceBobStats(kStreamLabel, kReceiverPeerName);
EXPECT_EQ(stream_stats.size(), 1lu);
auto it = stream_stats.find(kAliceBobStats);
@ -694,8 +702,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
<< ToString(frames_in_flight_sizes);
std::map<StatsKey, StreamStats> stream_stats = analyzer.GetStats();
const StatsKey kAliceBobStats(kStreamLabel, kSenderPeerName,
kReceiverPeerName);
const StatsKey kAliceBobStats(kStreamLabel, kReceiverPeerName);
EXPECT_EQ(stream_stats.size(), 1lu);
auto it = stream_stats.find(kAliceBobStats);
@ -872,10 +879,12 @@ TEST(DefaultVideoQualityAnalyzerTest, RuntimeParticipantsAdding) {
EXPECT_EQ(frame_counters.rendered, 2 * kFramesCount);
EXPECT_EQ(frame_counters.dropped, kOneThirdFrames);
EXPECT_EQ(analyzer.GetKnownVideoStreams().size(), 3lu);
const StatsKey kAliceBobStats(kStreamLabel, kAlice, kBob);
const StatsKey kAliceCharlieStats(kStreamLabel, kAlice, kCharlie);
const StatsKey kAliceKatieStats(kStreamLabel, kAlice, kKatie);
const StatsKey kAliceBobStats(kStreamLabel, kBob);
const StatsKey kAliceCharlieStats(kStreamLabel, kCharlie);
const StatsKey kAliceKatieStats(kStreamLabel, kKatie);
EXPECT_EQ(analyzer.GetKnownStreams().GetStatsKeys(),
(std::set<StatsKey>{kAliceBobStats, kAliceCharlieStats,
kAliceKatieStats}));
{
FrameCounters stream_conters =
analyzer.GetPerStreamCounters().at(kAliceBobStats);
@ -1044,7 +1053,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
EXPECT_EQ(analyzer.GetStats().size(), 2lu);
{
FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
StatsKey(kStreamLabel, kSenderPeerName, kReceiverPeerName));
StatsKey(kStreamLabel, kReceiverPeerName));
EXPECT_EQ(stream_conters.captured, 3);
EXPECT_EQ(stream_conters.pre_encoded, 3);
EXPECT_EQ(stream_conters.encoded, 3);
@ -1054,7 +1063,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
}
{
FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
StatsKey(kStreamLabel, kSenderPeerName, kSenderPeerName));
StatsKey(kStreamLabel, kSenderPeerName));
EXPECT_EQ(stream_conters.captured, 3);
EXPECT_EQ(stream_conters.pre_encoded, 3);
EXPECT_EQ(stream_conters.encoded, 3);
@ -1139,7 +1148,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
EXPECT_EQ(analyzer.GetStats().size(), 2lu);
{
FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
StatsKey(kStreamLabel, kSenderPeerName, kReceiverPeerName));
StatsKey(kStreamLabel, kReceiverPeerName));
EXPECT_EQ(stream_conters.captured, 3);
EXPECT_EQ(stream_conters.pre_encoded, 3);
EXPECT_EQ(stream_conters.encoded, 3);
@ -1149,7 +1158,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
}
{
FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
StatsKey(kStreamLabel, kSenderPeerName, kSenderPeerName));
StatsKey(kStreamLabel, kSenderPeerName));
EXPECT_EQ(stream_conters.captured, 3);
EXPECT_EQ(stream_conters.pre_encoded, 3);
EXPECT_EQ(stream_conters.encoded, 3);
@ -1210,7 +1219,7 @@ TEST(DefaultVideoQualityAnalyzerTest, CodecTrackedCorrectly) {
std::map<StatsKey, StreamStats> stats = analyzer.GetStats();
ASSERT_EQ(stats.size(), 1lu);
const StreamStats& stream_stats =
stats.at(StatsKey(kStreamLabel, kSenderPeerName, kReceiverPeerName));
stats.at(StatsKey(kStreamLabel, kReceiverPeerName));
ASSERT_EQ(stream_stats.encoders.size(), 2lu);
EXPECT_EQ(stream_stats.encoders[0].codec_name, codec_names[0]);
EXPECT_EQ(stream_stats.encoders[0].first_frame_id, frames[0].id());
@ -1308,7 +1317,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
EXPECT_EQ(analyzer.GetStats().size(), 1lu);
{
FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
StatsKey(kStreamLabel, kSenderPeerName, kReceiverPeerName));
StatsKey(kStreamLabel, kReceiverPeerName));
EXPECT_EQ(stream_conters.captured, 6);
EXPECT_EQ(stream_conters.pre_encoded, 5);
EXPECT_EQ(stream_conters.encoded, 4);
@ -1405,7 +1414,7 @@ TEST(
EXPECT_EQ(analyzer.GetStats().size(), 2lu);
{
FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
StatsKey(kStreamLabel, kSenderPeerName, kReceiverPeerName));
StatsKey(kStreamLabel, kReceiverPeerName));
EXPECT_EQ(stream_conters.captured, 6);
EXPECT_EQ(stream_conters.pre_encoded, 5);
EXPECT_EQ(stream_conters.encoded, 4);
@ -1415,7 +1424,7 @@ TEST(
}
{
FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
StatsKey(kStreamLabel, kSenderPeerName, kSenderPeerName));
StatsKey(kStreamLabel, kSenderPeerName));
EXPECT_EQ(stream_conters.captured, 6);
EXPECT_EQ(stream_conters.pre_encoded, 5);
EXPECT_EQ(stream_conters.encoded, 4);

3
test/pc/e2e/peer_connection_e2e_smoke_test.cc
View File

@ -93,7 +93,8 @@ class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test {
fixture_->Run(run_params);
EXPECT_GE(fixture_->GetRealTestDuration(), run_params.run_duration);
for (auto stream_key : video_quality_analyzer_->GetKnownVideoStreams()) {
VideoStreamsInfo known_streams = video_quality_analyzer_->GetKnownStreams();
for (const StatsKey& stream_key : known_streams.GetStatsKeys()) {
FrameCounters stream_conters =
video_quality_analyzer_->GetPerStreamCounters().at(stream_key);
// On some devices the pipeline can be too slow, so we actually can't