From 8d77ddec63b88eec1690220a87d6520d3f42cb2f Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 5 Dec 2018 10:58:31 +0200 Subject: [PATCH] MXS-2208 Add simple Http library - Built on top of Curl - Currently only GET --- BUILD/install_build_deps.sh | 6 +- CMakeLists.txt | 1 + .../Building-MaxScale-from-Source-Code.md | 1 + maxutils/maxbase/include/maxbase/http.hh | 72 +++++++++ maxutils/maxbase/src/CMakeLists.txt | 4 + maxutils/maxbase/src/http.cc | 152 ++++++++++++++++++ maxutils/maxbase/src/test/CMakeLists.txt | 4 + maxutils/maxbase/src/test/test_http.cc | 55 +++++++ 8 files changed, 292 insertions(+), 3 deletions(-) create mode 100644 maxutils/maxbase/include/maxbase/http.hh create mode 100644 maxutils/maxbase/src/http.cc create mode 100644 maxutils/maxbase/src/test/test_http.cc diff --git a/BUILD/install_build_deps.sh b/BUILD/install_build_deps.sh index ec347e853..aceaa702e 100755 --- a/BUILD/install_build_deps.sh +++ b/BUILD/install_build_deps.sh @@ -20,7 +20,7 @@ then build-essential libssl-dev ncurses-dev bison flex \ perl libtool libpcre3-dev tcl tcl-dev uuid \ uuid-dev libsqlite3-dev liblzma-dev libpam0g-dev pkg-config \ - libedit-dev + libedit-dev libcurl4-openssl-dev # One of these will work, older systems use libsystemd-daemon-dev sudo apt-get install -y libsystemd-dev || \ @@ -57,7 +57,7 @@ else make libtool libopenssl-devel libaio libaio-devel flex \ pcre-devel git wget tcl libuuid-devel \ xz-devel sqlite3 sqlite3-devel pkg-config lua lua-devel \ - gnutls-devel libgcrypt-devel pam-devel systemd-devel + gnutls-devel libgcrypt-devel pam-devel systemd-devel libcurl-devel sudo zypper -n install rpm-build cat /etc/*-release | grep "SUSE Linux Enterprise Server 11" @@ -74,7 +74,7 @@ else libedit-devel systemtap-sdt-devel rpm-sign wget \ gnupg pcre-devel flex rpmdevtools git wget tcl openssl libuuid-devel xz-devel \ sqlite sqlite-devel pkgconfig lua lua-devel rpm-build createrepo yum-utils \ - gnutls-devel libgcrypt-devel pam-devel + gnutls-devel libgcrypt-devel pam-devel libcurl-devel # Attempt to install libasan, it'll only work on CentOS 7 sudo yum install -y --nogpgcheck libasan diff --git a/CMakeLists.txt b/CMakeLists.txt index eb3d471e0..23a80299a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,6 +44,7 @@ find_package(GSSAPI) find_package(SQLite) find_package(ASAN) find_package(TSAN) +find_package(CURL) # Build PCRE2 so we always know the version # Read BuildPCRE2 for details about how to add pcre2 as a dependency to a target diff --git a/Documentation/Getting-Started/Building-MaxScale-from-Source-Code.md b/Documentation/Getting-Started/Building-MaxScale-from-Source-Code.md index be7621fab..c40021e94 100644 --- a/Documentation/Getting-Started/Building-MaxScale-from-Source-Code.md +++ b/Documentation/Getting-Started/Building-MaxScale-from-Source-Code.md @@ -11,6 +11,7 @@ requirements are as follows: * Flex 2.5.35 or later * libuuid * GNUTLS +* libcurl This is the minimum set of requirements that must be met to build the MaxScale core package. diff --git a/maxutils/maxbase/include/maxbase/http.hh b/maxutils/maxbase/include/maxbase/http.hh new file mode 100644 index 000000000..2d2264cb2 --- /dev/null +++ b/maxutils/maxbase/include/maxbase/http.hh @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2018 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl11. + * + * Change Date: 2022-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ +#pragma once + +#include +#include +#include +#include + +namespace maxbase +{ + +namespace http +{ + +enum +{ + DEFAULT_CONNECT_TIMEOUT = 10, // @see https://curl.haxx.se/libcurl/c/CURLOPT_CONNECTTIMEOUT.html + DEFAULT_TIMEOUT = 10 // @see https://curl.haxx.se/libcurl/c/CURLOPT_TIMEOUT.html +}; + +struct Config +{ + int connect_timeout = DEFAULT_CONNECT_TIMEOUT; + int timeout = DEFAULT_TIMEOUT; +}; + +struct Result +{ + int code = 0; // HTTP response code + std::string body; // Response body + std::map headers; // Headers attached to the response +}; + +/** + * Do a HTTP GET, when no user/password is required. + * + * @param url URL to use. + * @param config The config to use. + * + * @return A @c Result. + */ +Result get(const std::string& url, const Config& config = Config()); + +/** + * Do a HTTP GET + * + * @param url URL to use. + * @param user Username to use, optional. + * @param password Password for the user, optional. + * @param config The config to use. + * + * @return A @c Result. + */ +Result get(const std::string& url, + const std::string& user, const std::string& password, + const Config& config = Config()); + +} + +} + diff --git a/maxutils/maxbase/src/CMakeLists.txt b/maxutils/maxbase/src/CMakeLists.txt index bb135123c..49f54e00d 100644 --- a/maxutils/maxbase/src/CMakeLists.txt +++ b/maxutils/maxbase/src/CMakeLists.txt @@ -3,6 +3,7 @@ add_library(maxbase STATIC atomic.cc eventcount.cc format.cc + http.cc log.cc logger.cc maxbase.cc @@ -21,4 +22,7 @@ target_link_libraries(maxbase systemd) endif() set_target_properties(maxbase PROPERTIES VERSION "1.0.0" LINK_FLAGS -Wl,-z,defs) +target_link_libraries(maxbase + ${CURL_LIBRARIES} +) add_subdirectory(test) diff --git a/maxutils/maxbase/src/http.cc b/maxutils/maxbase/src/http.cc new file mode 100644 index 000000000..b8783a15e --- /dev/null +++ b/maxutils/maxbase/src/http.cc @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2018 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl11. + * + * Change Date: 2022-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include +#include +#include +#include + +using std::map; +using std::string; + +namespace +{ + +// TODO: Remove once trim is in maxbase. +// trim from beginning (in place) +inline void ltrim(std::string &s) { + s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) { + return !std::isspace(ch); + })); +} + +// trim from end (in place) +inline void rtrim(std::string &s) { + s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { + return !std::isspace(ch); + }).base(), s.end()); +} + +// trim from both ends (in place) +inline void trim(std::string &s) { + ltrim(s); + rtrim(s); +} + +template +inline void checked_curl_setopt(CURL* pCurl, CURLoption option, T value) +{ + MXB_AT_DEBUG(CURLcode rv =) curl_easy_setopt(pCurl, option, value); + mxb_assert(rv == CURLE_OK); +} + +// https://curl.haxx.se/libcurl/c/CURLOPT_WRITEFUNCTION.html +size_t write_callback(char* ptr, size_t size, size_t nmemb, void* userdata) +{ + // ptr points to the delivered data, and the size of that data is nmemb; + // size is always 1. + mxb_assert(size == 1); + + string* pString = static_cast(userdata); + + if (nmemb > 0) + { + pString->append(ptr, nmemb); + } + + return nmemb; +} + +// https://curl.haxx.se/libcurl/c/CURLOPT_HEADERFUNCTION +size_t header_callback(char* ptr, size_t size, size_t nmemb, void* userdata) +{ + size_t len = size * nmemb; + + if (len > 0) + { + map* pHeaders = static_cast*>(userdata); + + char* end = ptr + len; + char* i = std::find(ptr, end, ':'); + + if (i != end) + { + string key(ptr, i - ptr); + ++i; + string value(i, end - i); + trim(key); + trim(value); + pHeaders->insert(std::make_pair(key, value)); + } + } + + return len; +} + +} + + +namespace maxbase +{ + +namespace http +{ + +Result get(const std::string& url, const Config& config) +{ + return std::move(get(url, "", "", config)); +} + +Result get(const std::string& url, const std::string& user, const std::string& password, const Config& config) +{ + Result res; + char errbuf[CURL_ERROR_SIZE + 1] = ""; + CURL* pCurl = curl_easy_init(); + + checked_curl_setopt(pCurl, CURLOPT_NOSIGNAL, 1); + checked_curl_setopt(pCurl, CURLOPT_CONNECTTIMEOUT, config.connect_timeout); // For connection phase + checked_curl_setopt(pCurl, CURLOPT_TIMEOUT, config.timeout); // For data transfer phase + checked_curl_setopt(pCurl, CURLOPT_ERRORBUFFER, errbuf); + checked_curl_setopt(pCurl, CURLOPT_WRITEFUNCTION, write_callback); + checked_curl_setopt(pCurl, CURLOPT_WRITEDATA, &res.body); + checked_curl_setopt(pCurl, CURLOPT_URL, url.c_str()); + checked_curl_setopt(pCurl, CURLOPT_HEADERFUNCTION, header_callback); + checked_curl_setopt(pCurl, CURLOPT_HEADERDATA, &res.headers); + + if (!user.empty() && !password.empty()) + { + checked_curl_setopt(pCurl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); + checked_curl_setopt(pCurl, CURLOPT_USERPWD, (user + ":" + password).c_str()); + } + + long code = 0; // needs to be a long + + if (curl_easy_perform(pCurl) == CURLE_OK) + { + curl_easy_getinfo(pCurl, CURLINFO_RESPONSE_CODE, &code); + res.code = code; + } + else + { + res.code = -1; + res.body = errbuf; + } + + curl_easy_cleanup(pCurl); + + return std::move(res); +} + +} + +} diff --git a/maxutils/maxbase/src/test/CMakeLists.txt b/maxutils/maxbase/src/test/CMakeLists.txt index f1a4ccd25..e60d9a208 100644 --- a/maxutils/maxbase/src/test/CMakeLists.txt +++ b/maxutils/maxbase/src/test/CMakeLists.txt @@ -1,3 +1,7 @@ +add_executable(test_mxb_http test_http.cc) +target_link_libraries(test_mxb_http maxbase) +add_test(test_mxb_http test_mxb_http) + add_executable(test_mxb_log test_log.cc) target_link_libraries(test_mxb_log maxbase rt) add_test(test_mxb_log test_mxb_log) diff --git a/maxutils/maxbase/src/test/test_http.cc b/maxutils/maxbase/src/test/test_http.cc new file mode 100644 index 000000000..b5bf497de --- /dev/null +++ b/maxutils/maxbase/src/test/test_http.cc @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl11. + * + * Change Date: 2022-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include +#include +#include + +using namespace std; + +namespace +{ + +int test_http() +{ + int rv = EXIT_FAILURE; + + auto res = mxb::http::get("http://www.example.com/"); + cout << "http://www.example.com/ responded with: " << res.code << endl; + if (res.code == 200) + { + if (res.headers.count("Date")) + { + cout << "The date is: " << res.headers["Date"] << endl; + rv = EXIT_SUCCESS; + } + } + else + { + cout << "error: Exit code not 200 but: " << res.code << endl; + } + + return rv; +} + +} + +int main() +{ + int rv = EXIT_SUCCESS; + mxb::Log log; + + rv = test_http(); + + return rv; +}