The purpose of this library is to create a utility library that is not

dependent on maxscale for use in both maxscale and system test, and
possibly other apps. As time permits general purpose utilities from
maxscale-common can be moved to the new library.

Here are answers to questions you may have:
- Headers and sources are separated to allow public/private headers.
- A top level directory "util" contains the libraries. The current
  structure is simply util/maxbase. The name of the top level
  directory is not important.
- Code is in a namespace with the same name as the directory.
- Headers are included like this: \`#include <maxbase/stopwatch.hh>\`
- In case the library is published on its own, the include directives stay
  the same (headers would be in /usr/include/maxbase, for example).
- I am not advocating many small libraries. But if some larger library
  is written, say a general purpose statemachine, it would not pollute
  util/maxutil but go to util/maxsm.
  Another example: Worker. It is a larger concept, but used so widely in
  code that it could very well live in maxutil.

NOTE: this was previously Review Request #6245.
This commit is contained in:
Niclas Antti 2018-06-14 16:10:22 +03:00
parent de37f1a5c4
commit 68c0c59d92
7 changed files with 616 additions and 0 deletions

View File

@ -166,6 +166,7 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${DEBUG_FLAGS} -DSS_DEBUG -D
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -Wno-deprecated-declarations -Wno-uninitialized -std=c++11")
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -ggdb -Wno-deprecated-declarations -Wno-uninitialized -std=c++11")
include_directories(util)
include_directories(include)
include_directories(server/inih)
include_directories(server/modules/include)

1
util/CMakeLists.txt Normal file
View File

@ -0,0 +1 @@
include_directories(maxbase)

View File

@ -0,0 +1,3 @@
add_library(base SHARED eventcount.cc stopwatch.cc)
set_target_properties(base PROPERTIES VERSION "1.0.0")
install_module(base core)

290
util/maxbase/eventcount.cc Normal file
View File

@ -0,0 +1,290 @@
/*
* Copyright (c) 2018 MariaDB Corporation Ab
*
* Use of this software is governed by the Business Source License included
* in the LICENSE.TXT file and at www.mariadb.com/bsl11.
*
* Change Date: 2020-01-01
*
* On the date above, in accordance with the Business Source License, use
* of this software will be governed by version 2 or later of the General
* Public License.
*/
#include "eventcount.hh"
#include <algorithm>
#include <iostream>
#include <iomanip>
#include <sstream>
#include <map>
namespace base
{
EventCount::EventCount(const std::string& event_id, Duration time_window, Duration granularity) :
m_event_id(event_id),
m_time_window(time_window),
m_granularity(granularity.count())
{
increment();
}
void EventCount::increment()
{
using namespace std::chrono;
auto ticks = time_point_cast<nanoseconds>(Clock::now()).time_since_epoch().count();
if (m_granularity)
{
ticks = ticks / m_granularity * m_granularity;
}
if (m_timestamps.empty()
|| m_timestamps.back().time_point.time_since_epoch().count() != ticks)
{
m_timestamps.emplace_back(TimePoint(ticks), 1);
}
else
{
++m_timestamps.back().count;
}
}
namespace
{
struct TimePointLessEqual
{
TimePoint lhs;
TimePointLessEqual(TimePoint tp) : lhs(tp) {}
bool operator()(const EventCount::Timestamp& rhs) const
{
return lhs <= rhs.time_point;
}
bool operator()(TimePoint rhs) const
{
return lhs <= rhs;
}
};
}
void EventCount::purge() const
{
StopWatch sw;
auto windowBegin = Clock::now() - m_time_window;
auto ite = std::find_if(m_timestamps.begin(), m_timestamps.end(),
TimePointLessEqual(windowBegin));
m_timestamps.erase(m_timestamps.begin(), ite);
}
int EventCount::count() const
{
purge();
int count {0};
for (auto ite = m_timestamps.begin(); ite != m_timestamps.end(); ++ite)
{
count += ite->count;
}
return count;
}
void EventCount::dump(std::ostream &os) const
{
os << m_event_id << ": " << count() << " " << m_timestamps.size();
}
std::ostream& operator<<(std::ostream& os, const EventCount& EventCount)
{
EventCount.dump(os);
return os;
}
// Force a purge once in awhile, could be configurable. This is needed if
// a client generates lots of events but rarely reads them back (purges).
const int CleanupCountdown = 10000;
SessionCount::SessionCount(const std::string& sess_id, Duration time_window,
Duration granularity) :
m_sess_id(sess_id), m_time_window(time_window), m_granularity(granularity),
m_cleanup_countdown(CleanupCountdown)
{
}
const std::vector<EventCount> &SessionCount::event_counts() const
{
purge();
return m_event_counts;
}
bool SessionCount::empty() const
{
purge();
return m_event_counts.empty();
}
namespace
{
struct MatchEventId
{
std::string event_id;
MatchEventId(const std::string& id) : event_id(id) {};
bool operator()(const EventCount& stats) const
{
return event_id == stats.event_id();
}
};
}
void SessionCount::increment(const std::string& event_id)
{
// Always put the incremented entry (latest timestamp) last in the vector (using
// rotate). This means the vector is ordered so that expired entries are always first.
// Find in reverse, the entry is more likely to be towards the end. Actually no,
// for some reason the normal search is slightly faster when measured.
auto ite = find_if(m_event_counts.begin(), m_event_counts.end(),
MatchEventId(event_id));
if (ite == m_event_counts.end())
{
m_event_counts.emplace_back(event_id, m_time_window, m_granularity);
}
else
{
ite->increment();
// rotate so that the entry becomes the last one
auto next = std::next(ite);
std::rotate(ite, next, m_event_counts.end());
}
if (!--m_cleanup_countdown)
{
purge();
}
}
namespace
{
struct NonZeroEntry
{
bool operator()(const EventCount& stats)
{
return stats.count() != 0;
}
};
}
void SessionCount::purge() const
{
StopWatch sw;
m_cleanup_countdown = CleanupCountdown;
// erase entries up to the first non-zero one
auto ite = find_if(m_event_counts.begin(), m_event_counts.end(), NonZeroEntry());
// The gcc 4.4 vector::erase bug only happens if iterators are the same.
if (ite != m_event_counts.begin())
{
m_event_counts.erase(m_event_counts.begin(), ite);
}
}
void SessionCount::dump(std::ostream& os) const
{
purge();
if (!m_event_counts.empty())
{
os << " Session: " << m_sess_id << '\n';
for (auto ite = m_event_counts.begin(); ite != m_event_counts.end(); ++ite)
{
os << " " << *ite << '\n';
}
}
}
void dumpHeader(std::ostream& os, const SessionCount& stats, const std::string& type)
{
TimePoint tp = Clock::now();
os << type << ": Time:" << tp
<< " Time Window: " << stats.time_window() << '\n';
}
void dump(std::ostream& os, const std::vector<SessionCount>& sessions)
{
if (sessions.empty())
{
return;
}
dumpHeader(os, sessions[0], "Count");
for (auto session = sessions.begin(); session != sessions.end(); ++session)
{
session->dump(os);
}
}
void dumpTotals(std::ostream& os, const std::vector<SessionCount> &sessions)
{
if (sessions.empty())
{
return;
}
std::map<std::string, int> counts;
for (auto session = sessions.begin(); session != sessions.end(); ++session)
{
const auto& events = session->event_counts();
for (auto event = events.begin(); event != events.end(); ++event)
{
counts[event->event_id()] += event->count();
}
}
if (!counts.empty())
{
dumpHeader(os, sessions[0], "Count Totals");
for (auto ite = counts.begin(); ite != counts.end(); ++ite)
{
os << " " << ite->first << ": " << ite->second << '\n';
}
}
}
// EXTRA
// This section needed for gcc 4.4, to use move semantics and variadics.
EventCount::EventCount(EventCount && ss) :
m_event_id(std::move(ss.m_event_id)),
m_time_window(std::move(ss.m_time_window)),
m_granularity(std::move(ss.m_granularity)),
m_timestamps(std::move(ss.m_timestamps))
{
}
EventCount &EventCount::operator=(EventCount && ss)
{
m_event_id = std::move(ss.m_event_id);
m_time_window = std::move(ss.m_time_window);
m_granularity = std::move(ss.m_granularity);
m_timestamps = std::move(ss.m_timestamps);
return *this;
}
SessionCount::SessionCount(SessionCount&& ss) :
m_sess_id(std::move(ss.m_sess_id)),
m_time_window(std::move(ss.m_time_window)),
m_granularity(std::move(ss.m_granularity)),
m_cleanup_countdown(std::move(ss.m_cleanup_countdown)),
m_event_counts(std::move(ss.m_event_counts))
{
}
SessionCount & SessionCount::operator=(SessionCount&& ss)
{
m_sess_id = std::move(ss.m_sess_id);
m_time_window = std::move(ss.m_time_window);
m_granularity = std::move(ss.m_granularity);
m_cleanup_countdown = std::move(ss.m_cleanup_countdown);
m_event_counts = std::move(ss.m_event_counts);
return *this;
}
} // base

111
util/maxbase/eventcount.hh Normal file
View File

@ -0,0 +1,111 @@
/*
* Copyright (c) 2018 MariaDB Corporation Ab
*
* Use of this software is governed by the Business Source License included
* in the LICENSE.TXT file and at www.mariadb.com/bsl11.
*
* Change Date: 2020-01-01
*
* On the date above, in accordance with the Business Source License, use
* of this software will be governed by version 2 or later of the General
* Public License.
*/
#pragma once
#include <base/stopwatch.hh>
#include <iosfwd>
#include <vector>
namespace base
{
/**
* @brief Keep a count of an events for a time period from "now" into the past.
*
* Events are stored, or distinguished, with timestamps of a given granularity.
* For example, if the granularity is set to 1s, each time an event is reported the current
* time is rounded down to the nearest whole second. All events that round down to the same
* second share a single entry in the EventCount. Granularity==0 causes all events (timestamps)
* to be stored in their own entry, which could use large amounts of memory.
*
*/
class EventCount
{
public:
explicit EventCount(const std::string& event_id, Duration time_window,
Duration granularity = Duration(std::chrono::milliseconds(10)));
EventCount(const EventCount&) = delete;
EventCount& operator=(const EventCount&) = delete;
EventCount(EventCount&&); // can't be defaulted in gcc 4.4
EventCount& operator=(EventCount&&); // can't be defaulted in gcc 4.4
const std::string& event_id() const
{
return m_event_id;
}
Duration time_window() const
{
return m_time_window;
}
void dump(std::ostream& os) const;
int count() const;
void increment();
// these defs need not be public once lambdas are available
struct Timestamp
{
TimePoint time_point;
int count;
Timestamp(TimePoint p, int c) : time_point(p), count(c) {}
};
private:
void purge() const; // remove out of window stats
std::string m_event_id;
Duration m_time_window;
Duration::rep m_granularity;
mutable std::vector<Timestamp> m_timestamps;
};
std::ostream& operator<<(std::ostream& os, const EventCount& stats);
// Time series statistics for a Session (a collection of related EventCount).
class SessionCount
{
public:
SessionCount(const std::string& sess_id, Duration time_window,
Duration granularity = Duration(std::chrono::milliseconds(10)));
SessionCount(const SessionCount&) = delete;
SessionCount& operator=(const SessionCount&) = delete;
SessionCount(SessionCount &&); // can't be defaulted in gcc 4.4
SessionCount& operator=(SessionCount&&); // can't be defaulted in gcc 4.4
const std::string& session_id() const
{
return m_sess_id;
}
Duration time_window() const
{
return m_time_window;
}
const std::vector<EventCount>& event_counts() const;
void dump(std::ostream& os) const;
bool empty() const; // no stats
void increment(const std::string& event_id);
private:
void purge() const; // remove out of window stats
std::string m_sess_id;
Duration m_time_window;
Duration m_granularity;
mutable int m_cleanup_countdown;
mutable std::vector<EventCount> m_event_counts;
};
// conveniece. Any real formatted output should go elsewhere.
std::ostream& operator<<(std::ostream& os, const SessionCount& stats);
void dump(std::ostream& os, const std::vector<SessionCount>& sessions);
void dumpTotals(std::ostream& os, const std::vector<SessionCount> &sessions);
} // base

149
util/maxbase/stopwatch.cc Normal file
View File

@ -0,0 +1,149 @@
/*
* Copyright (c) 2018 MariaDB Corporation Ab
*
* Use of this software is governed by the Business Source License included
* in the LICENSE.TXT file and at www.mariadb.com/bsl11.
*
* Change Date: 2020-01-01
*
* On the date above, in accordance with the Business Source License, use
* of this software will be governed by version 2 or later of the General
* Public License.
*/
#include "stopwatch.hh"
#include <iomanip>
#include <iostream>
#include <sstream>
#include <ctime>
namespace base
{
StopWatch::StopWatch()
{
restart();
}
Duration StopWatch::lap() const
{
return {Clock::now() - m_start};
}
Duration StopWatch::restart()
{
TimePoint now = Clock::now();
Duration lap = now - m_start;
m_start = now;
return lap;
}
} // base
/********** OUTPUT ***********/
namespace
{
using namespace base;
struct TimeConvert
{
double div; // divide the value of the previous unit by this
std::string suffix; // milliseconds, hours etc.
double max_visual; // threashold to switch to the next unit
};
// Will never get to centuries because the duration is a long carrying nanoseconds
TimeConvert convert[]
{
{1, "ns", 1000}, {1000, "us", 1000}, {1000, "ms", 1000},
{1000, "s", 60}, {60, "min", 60}, {60, "hours", 24},
{24, "days", 365.25}, {365.25, "years", 10000},
{100, "centuries", std::numeric_limits<double>::max()}
};
int convert_size = sizeof(convert) / sizeof(convert[0]);
}
namespace base
{
std::pair<double, std::string> dur_to_human_readable(Duration dur)
{
using namespace std::chrono;
double time = duration_cast<nanoseconds>(dur).count();
bool negative = (time < 0) ? time = -time, true : false;
for (int i = 0; i <= convert_size; ++i)
{
if (i == convert_size)
{
return std::make_pair(negative ? -time : time,
convert[convert_size - 1].suffix);
}
time /= convert[i].div;
if (time < convert[i].max_visual)
{
return std::make_pair(negative ? -time : time, convert[i].suffix);
}
}
abort(); // should never get here
}
std::ostream& operator<<(std::ostream& os, Duration dur)
{
auto p = dur_to_human_readable(dur);
os << p.first << p.second;
return os;
}
// TODO: this will require some thought. time_point_to_string() for a system_clock is
// obvious, but not so for a steady_clock. Maybe TimePoint belongs to a system clock
// and sould be called something else here, and live in a time_measuring namespace.
std::string time_point_to_string(TimePoint tp, const std::string &fmt)
{
using namespace std::chrono;
std::time_t timet = system_clock::to_time_t(system_clock::now()
+ (tp - Clock::now()));
struct tm * ptm;
ptm = gmtime (&timet);
const int sz = 1024;
char buf[sz];
strftime(buf, sz, fmt.c_str(), ptm);
return buf;
}
std::ostream & operator<<(std::ostream & os, TimePoint tp)
{
os << time_point_to_string(tp);
return os;
}
void test_stopwatch_output(std::ostream & os)
{
long long dur[] =
{
400, // 400ns
5 * 1000, // 5us
500 * 1000, // 500us
1 * 1000000, // 1ms
700 * 1000000LL, // 700ms
5 * 1000000000LL, // 5s
200 * 1000000000LL, // 200s
5 * 60 * 1000000000LL, // 5m
45 * 60 * 1000000000LL, // 45m
130 * 60 * 1000000000LL, // 130m
24 * 60 * 60 * 1000000000LL, // 24 hours
3 * 24 * 60 * 60 * 1000000000LL, // 72 hours
180 * 24 * 60 * 60 * 1000000000LL, // 180 days
1000 * 24 * 60 * 60 * 1000000000LL // 1000 days
};
for (unsigned i = 0; i < sizeof(dur) / sizeof(dur[0]); ++i)
{
os << Duration(dur[i]) << std::endl;
}
}
} // base

61
util/maxbase/stopwatch.hh Normal file
View File

@ -0,0 +1,61 @@
/*
* Copyright (c) 2018 MariaDB Corporation Ab
*
* Use of this software is governed by the Business Source License included
* in the LICENSE.TXT file and at www.mariadb.com/bsl11.
*
* Change Date: 2020-01-01
*
* On the date above, in accordance with the Business Source License, use
* of this software will be governed by version 2 or later of the General
* Public License.
*/
#pragma once
#include <chrono>
#include <iosfwd>
#include <string>
namespace base
{
#if __cplusplus >= 201103
typedef std::chrono::steady_clock Clock;
#else
typedef std::chrono::system_clock Clock;
#endif
struct Duration : public Clock::duration // for ADL
{
// gcc 4.4 does not inherit constructors, so this is a bit limited.
Duration() = default;
Duration(long long l) : Clock::duration(l) {}
Duration(Clock::duration d) : Clock::duration(d) {}
};
typedef std::chrono::time_point<Clock, Duration> TimePoint;
class StopWatch
{
public:
// Starts the stopwatch, which is always running.
StopWatch();
// Get elapsed time.
Duration lap() const;
// Get elapsed time, restart StopWatch.
Duration restart();
private:
TimePoint m_start;
};
// Returns the value as a double and string adjusted to a suffix like ms for milliseconds.
std::pair<double, std::string> dur_to_human_readable(Duration dur);
// Human readable output. No standard library for it yet.
std::ostream& operator<<(std::ostream&, Duration dur);
// TimePoint
std::string time_point_to_string(TimePoint tp, const std::string& fmt = "%F %T");
std::ostream& operator<<(std::ostream&, TimePoint tp);
} // base