MXS-2208 Add simple Http library

- Built on top of Curl
- Currently only GET

BUILD/install_build_deps.sh

@@ -20,7 +20,7 @@ then
        build-essential libssl-dev ncurses-dev bison flex \
        perl libtool libpcre3-dev tcl tcl-dev uuid \
        uuid-dev libsqlite3-dev liblzma-dev libpam0g-dev pkg-config \
-       libedit-dev
+       libedit-dev libcurl4-openssl-dev
 
   # One of these will work, older systems use libsystemd-daemon-dev
   sudo apt-get install -y libsystemd-dev || \
@@ -57,7 +57,7 @@ else
          make libtool libopenssl-devel libaio libaio-devel flex \
          pcre-devel git wget tcl libuuid-devel \
          xz-devel sqlite3 sqlite3-devel pkg-config lua lua-devel \
-         gnutls-devel libgcrypt-devel pam-devel systemd-devel
+         gnutls-devel libgcrypt-devel pam-devel systemd-devel libcurl-devel
     sudo zypper -n install rpm-build
     cat /etc/*-release | grep "SUSE Linux Enterprise Server 11"
 
@@ -74,7 +74,7 @@ else
          libedit-devel systemtap-sdt-devel rpm-sign wget \
          gnupg pcre-devel flex rpmdevtools git wget tcl openssl libuuid-devel xz-devel \
          sqlite sqlite-devel pkgconfig lua lua-devel rpm-build createrepo yum-utils \
-         gnutls-devel libgcrypt-devel pam-devel
+         gnutls-devel libgcrypt-devel pam-devel libcurl-devel
 
     # Attempt to install libasan, it'll only work on CentOS 7
     sudo yum install -y --nogpgcheck libasan

CMakeLists.txt

@@ -44,6 +44,7 @@ find_package(GSSAPI)
 find_package(SQLite)
 find_package(ASAN)
 find_package(TSAN)
+find_package(CURL)
 
 # Build PCRE2 so we always know the version
 # Read BuildPCRE2 for details about how to add pcre2 as a dependency to a target

Documentation/Getting-Started/Building-MaxScale-from-Source-Code.md

@@ -11,6 +11,7 @@ requirements are as follows:
 * Flex 2.5.35 or later
 * libuuid
 * GNUTLS
+* libcurl
 
 This is the minimum set of requirements that must be met to build the MaxScale
 core package.

maxutils/maxbase/include/maxbase/http.hh

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2018 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2022-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+#pragma once
+
+#include <maxbase/ccdefs.hh>
+#include <map>
+#include <memory>
+#include <string>
+
+namespace maxbase
+{
+
+namespace http
+{
+
+enum
+{
+    DEFAULT_CONNECT_TIMEOUT = 10, // @see https://curl.haxx.se/libcurl/c/CURLOPT_CONNECTTIMEOUT.html
+    DEFAULT_TIMEOUT = 10          // @see https://curl.haxx.se/libcurl/c/CURLOPT_TIMEOUT.html
+};
+
+struct Config
+{
+    int connect_timeout = DEFAULT_CONNECT_TIMEOUT;
+    int timeout         = DEFAULT_TIMEOUT;
+};
+
+struct Result
+{
+    int                                code = 0; // HTTP response code
+    std::string                        body;     // Response body
+    std::map<std::string, std::string> headers;  // Headers attached to the response
+};
+
+/**
+ * Do a HTTP GET, when no user/password is required.
+ *
+ * @param url     URL to use.
+ * @param config  The config to use.
+ *
+ * @return A @c Result.
+ */
+Result get(const std::string& url, const Config& config = Config());
+
+/**
+ * Do a HTTP GET
+ *
+ * @param url       URL to use.
+ * @param user      Username to use, optional.
+ * @param password  Password for the user, optional.
+ * @param config    The config to use.
+ *
+ * @return A @c Result.
+ */
+Result get(const std::string& url,
+           const std::string& user, const std::string& password,
+           const Config& config = Config());
+
+}
+
+}
+

maxutils/maxbase/src/CMakeLists.txt

@@ -3,6 +3,7 @@ add_library(maxbase STATIC
   atomic.cc
   eventcount.cc
   format.cc
+  http.cc
   log.cc
   logger.cc
   maxbase.cc
@@ -21,4 +22,7 @@ target_link_libraries(maxbase systemd)
 endif()
 
 set_target_properties(maxbase PROPERTIES VERSION "1.0.0" LINK_FLAGS -Wl,-z,defs)
+target_link_libraries(maxbase
+  ${CURL_LIBRARIES}
+)
 add_subdirectory(test)

maxutils/maxbase/src/http.cc

@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) 2018 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2022-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <maxbase/http.hh>
+#include <algorithm>
+#include <curl/curl.h>
+#include <maxbase/assert.h>
+
+using std::map;
+using std::string;
+
+namespace
+{
+
+// TODO: Remove once trim is in maxbase.
+// trim from beginning (in place)
+inline void ltrim(std::string &s) {
+    s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) {
+        return !std::isspace(ch);
+    }));
+}
+
+// trim from end (in place)
+inline void rtrim(std::string &s) {
+    s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) {
+        return !std::isspace(ch);
+    }).base(), s.end());
+}
+
+// trim from both ends (in place)
+inline void trim(std::string &s) {
+    ltrim(s);
+    rtrim(s);
+}
+
+template<class T>
+inline void checked_curl_setopt(CURL* pCurl, CURLoption option, T value)
+{
+    MXB_AT_DEBUG(CURLcode rv =) curl_easy_setopt(pCurl, option, value);
+    mxb_assert(rv == CURLE_OK);
+}
+
+// https://curl.haxx.se/libcurl/c/CURLOPT_WRITEFUNCTION.html
+size_t write_callback(char* ptr, size_t size, size_t nmemb, void* userdata)
+{
+    // ptr points to the delivered data, and the size of that data is nmemb;
+    // size is always 1.
+    mxb_assert(size == 1);
+
+    string* pString = static_cast<string*>(userdata);
+
+    if (nmemb > 0)
+    {
+        pString->append(ptr, nmemb);
+    }
+
+    return nmemb;
+}
+
+// https://curl.haxx.se/libcurl/c/CURLOPT_HEADERFUNCTION
+size_t header_callback(char* ptr, size_t size, size_t nmemb, void* userdata)
+{
+    size_t len = size * nmemb;
+
+    if (len > 0)
+    {
+        map<string, string>* pHeaders = static_cast<std::map<string, string>*>(userdata);
+
+        char* end = ptr + len;
+        char* i = std::find(ptr, end, ':');
+
+        if (i != end)
+        {
+            string key(ptr, i - ptr);
+            ++i;
+            string value(i, end - i);
+            trim(key);
+            trim(value);
+            pHeaders->insert(std::make_pair(key, value));
+        }
+    }
+
+    return len;
+}
+
+}
+
+
+namespace maxbase
+{
+
+namespace http
+{
+
+Result get(const std::string& url, const Config& config)
+{
+    return std::move(get(url, "", "", config));
+}
+
+Result get(const std::string& url, const std::string& user, const std::string& password, const Config& config)
+{
+    Result res;
+    char errbuf[CURL_ERROR_SIZE + 1] = "";
+    CURL* pCurl = curl_easy_init();
+
+    checked_curl_setopt(pCurl, CURLOPT_NOSIGNAL, 1);
+    checked_curl_setopt(pCurl, CURLOPT_CONNECTTIMEOUT, config.connect_timeout); // For connection phase
+    checked_curl_setopt(pCurl, CURLOPT_TIMEOUT, config.timeout);                // For data transfer phase
+    checked_curl_setopt(pCurl, CURLOPT_ERRORBUFFER, errbuf);
+    checked_curl_setopt(pCurl, CURLOPT_WRITEFUNCTION, write_callback);
+    checked_curl_setopt(pCurl, CURLOPT_WRITEDATA, &res.body);
+    checked_curl_setopt(pCurl, CURLOPT_URL, url.c_str());
+    checked_curl_setopt(pCurl, CURLOPT_HEADERFUNCTION, header_callback);
+    checked_curl_setopt(pCurl, CURLOPT_HEADERDATA, &res.headers);
+
+    if (!user.empty() && !password.empty())
+    {
+        checked_curl_setopt(pCurl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
+        checked_curl_setopt(pCurl, CURLOPT_USERPWD, (user + ":" + password).c_str());
+    }
+
+    long code = 0;      // needs to be a long
+
+    if (curl_easy_perform(pCurl) == CURLE_OK)
+    {
+        curl_easy_getinfo(pCurl, CURLINFO_RESPONSE_CODE, &code);
+        res.code = code;
+    }
+    else
+    {
+        res.code = -1;
+        res.body = errbuf;
+    }
+
+    curl_easy_cleanup(pCurl);
+
+    return std::move(res);
+}
+
+}
+
+}

maxutils/maxbase/src/test/CMakeLists.txt

@@ -1,3 +1,7 @@
+add_executable(test_mxb_http test_http.cc)
+target_link_libraries(test_mxb_http maxbase)
+add_test(test_mxb_http test_mxb_http)
+
 add_executable(test_mxb_log test_log.cc)
 target_link_libraries(test_mxb_log maxbase rt)
 add_test(test_mxb_log test_mxb_log)

maxutils/maxbase/src/test/test_http.cc

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2022-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <maxbase/http.hh>
+#include <iostream>
+#include <maxbase/log.hh>
+
+using namespace std;
+
+namespace
+{
+
+int test_http()
+{
+    int rv = EXIT_FAILURE;
+
+    auto res = mxb::http::get("http://www.example.com/");
+    cout << "http://www.example.com/ responded with: " << res.code << endl;
+    if (res.code == 200)
+    {
+        if (res.headers.count("Date"))
+        {
+            cout << "The date is: " << res.headers["Date"] << endl;
+            rv = EXIT_SUCCESS;
+        }
+    }
+    else
+    {
+        cout << "error: Exit code not 200 but: " << res.code << endl;
+    }
+
+    return rv;
+}
+
+}
+
+int main()
+{
+    int rv = EXIT_SUCCESS;
+    mxb::Log log;
+
+    rv = test_http();
+
+    return rv;
+}