From 9e6872621901bb98cb3a1293bb2379b5afa56327 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 8 May 2017 14:41:29 +0300 Subject: [PATCH 01/55] Fix internal test suite Fixed minor problems in DCB and configuration tests. --- server/core/test/testconfig.cc | 2 +- server/core/test/testdcb.cc | 15 +-------------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/server/core/test/testconfig.cc b/server/core/test/testconfig.cc index 69370c4fc..564c9356b 100644 --- a/server/core/test/testconfig.cc +++ b/server/core/test/testconfig.cc @@ -137,7 +137,7 @@ int test_add_parameter() }; - CONFIG_CONTEXT svc1, svc2, ctx; + CONFIG_CONTEXT svc1 = {}, svc2 = {}, ctx = {}; svc1.object = (char*)"my-service"; svc2.object = (char*)"some-service"; svc2.next = &svc1; diff --git a/server/core/test/testdcb.cc b/server/core/test/testdcb.cc index 7edecc46b..d6ece4e9c 100644 --- a/server/core/test/testdcb.cc +++ b/server/core/test/testdcb.cc @@ -45,12 +45,7 @@ static int test1() { - DCB *dcb, *extra, *clone; - int size = 100; - int bite1 = 35; - int bite2 = 60; - int bite3 = 10; - int buflen; + DCB *dcb; SERV_LISTENER dummy; /* Single buffer tests */ ss_dfprintf(stderr, @@ -59,8 +54,6 @@ test1() printDCB(dcb); ss_info_dassert(dcb_isvalid(dcb), "New DCB must be valid"); ss_dfprintf(stderr, "\t..done\nAllocated dcb."); - clone = dcb_clone(dcb); - ss_dfprintf(stderr, "\t..done\nCloned dcb"); printAllDCBs(); ss_info_dassert(true, "Something is true"); ss_dfprintf(stderr, "\t..done\n"); @@ -68,12 +61,6 @@ test1() dcb_close(dcb); ss_dfprintf(stderr, "Freed original dcb"); ss_info_dassert(!dcb_isvalid(dcb), "Closed DCB must not be valid"); - ss_dfprintf(stderr, "\t..done\nMake clone DCB a zombie"); - clone->state = DCB_STATE_NOPOLLING; - dcb_add_to_list(clone); - dcb_close(clone); - ss_dfprintf(stderr, "\t..done\nCheck clone no longer valid"); - ss_info_dassert(!dcb_isvalid(clone), "After closing, clone DCB must not be valid"); ss_dfprintf(stderr, "\t..done\nProcess the zombies list"); dcb_process_zombies(0); ss_dfprintf(stderr, "\t..done\n"); From c27b2a18051040a3c5f5077c08e1ee52df5f84e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 8 May 2017 21:26:00 +0300 Subject: [PATCH 02/55] Fix internal test header The polling and message queue systems weren't initalized. --- server/core/test/test_utils.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/core/test/test_utils.h b/server/core/test/test_utils.h index 464679bae..172461108 100644 --- a/server/core/test/test_utils.h +++ b/server/core/test/test_utils.h @@ -23,6 +23,7 @@ #include "../maxscale/poll.h" #include "../maxscale/statistics.h" +#include "../maxscale/worker.hh" void init_test_env(char *path) @@ -37,6 +38,8 @@ void init_test_env(char *path) mxs_log_init(NULL, logdir, MXS_LOG_TARGET_DEFAULT); dcb_global_init(); poll_init(); + maxscale::MessageQueue::init(); + maxscale::Worker::init(); hkinit(); } From 18ca4189f0b17ac6e274182db9a375fc93938b1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 8 May 2017 21:58:24 +0300 Subject: [PATCH 03/55] Fix testbinlogrouter The test used service credentials directly instead of copying them. --- server/modules/routing/binlogrouter/test/testbinlog.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/modules/routing/binlogrouter/test/testbinlog.c b/server/modules/routing/binlogrouter/test/testbinlog.c index 3a0ae59f0..2125299b3 100644 --- a/server/modules/routing/binlogrouter/test/testbinlog.c +++ b/server/modules/routing/binlogrouter/test/testbinlog.c @@ -129,8 +129,8 @@ int main(int argc, char **argv) } inst->service = service; - inst->user = service->credentials.name; - inst->password = service->credentials.authdata; + inst->user = MXS_STRDUP_A(service->credentials.name); + inst->password = MXS_STRDUP_A(service->credentials.authdata); MXS_NOTICE("testbinlog v1.0"); From c7cffa0722807c4d7af91a03c6d864a10af08492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Sun, 7 May 2017 11:00:48 +0300 Subject: [PATCH 04/55] Add atomic compare-and-swap The atomic compare-and-swap can be used to implement lock-free structures. The planned use for this is to remove some of the locking done in the services when listeners are being manipulated. --- include/maxscale/atomic.h | 15 +++++++++++++++ server/core/atomic.cc | 10 ++++++++++ server/core/test/testatomic.cc | 29 ++++++++++++++++++++++++++--- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/include/maxscale/atomic.h b/include/maxscale/atomic.h index 639faa216..e571f2d11 100644 --- a/include/maxscale/atomic.h +++ b/include/maxscale/atomic.h @@ -102,4 +102,19 @@ static inline void atomic_synchronize() #endif } +/** + * @brief Atomic compare-and-swap of pointers + * + * @param variable Pointer to the variable + * @param old_value Pointer to the expected value of @variable + * @param new_value Stored value if @c variable is equal to @c old_value + * + * @return True if @c variable and @c old_value were equal + * + * @note If GCC __atomic builtins are available, the contents of @c variable are + * written to @c old_value if the two are not equal. Do not rely on this behavior + * and always do a separate read before attempting a compare-and-swap. + */ +bool atomic_cas_ptr(void **variable, void** old_value, void *new_value); + MXS_END_DECLS diff --git a/server/core/atomic.cc b/server/core/atomic.cc index afff9c25d..5ffa1de90 100644 --- a/server/core/atomic.cc +++ b/server/core/atomic.cc @@ -124,3 +124,13 @@ void atomic_store_ptr(void **variable, void *value) (void)__sync_lock_test_and_set(variable, value); #endif } + +bool atomic_cas_ptr(void **variable, void** old_value, void *new_value) +{ +#ifdef MXS_USE_ATOMIC_BUILTINS + return __atomic_compare_exchange_n(variable, old_value, new_value, + false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); +#else + return __sync_val_compare_and_swap(variable, *old_value, new_value); +#endif +} diff --git a/server/core/test/testatomic.cc b/server/core/test/testatomic.cc index da405567a..f50dbb45b 100644 --- a/server/core/test/testatomic.cc +++ b/server/core/test/testatomic.cc @@ -37,7 +37,6 @@ void test_add(void* data) } } - void test_load_store(void* data) { int id = (size_t)data; @@ -51,6 +50,29 @@ void test_load_store(void* data) } } +static void* cas_dest = (void*)1; + +void test_cas(void* data) +{ + int id = (size_t)data; + static int loops = 0; + + while (atomic_load_int32(&running)) + { + intptr_t my_value = (id + 1) % NTHR; + intptr_t my_expected = id; + + while (!atomic_cas_ptr(&cas_dest, (void**)&my_expected, (void*)&my_value)) + { + ; + } + + loops++; + } + + ss_dassert(loops > 0); +} + int run_test(void(*func)(void*)) { THREAD threads[NTHR]; @@ -58,9 +80,9 @@ int run_test(void(*func)(void*)) atomic_store_int32(&expected, 0); atomic_store_int32(&running, 1); - for (int i = 0; i < NTHR; i++) + for (size_t i = 0; i < NTHR; i++) { - if (thread_start(&threads[i], func, NULL) == NULL) + if (thread_start(&threads[i], func, (void*)(i + 1)) == NULL) { ss_dassert(false); } @@ -83,6 +105,7 @@ int main(int argc, char** argv) run_test(test_load_store); run_test(test_add); + run_test(test_cas); return rval; } From 0ea58ea6bcbecdae9bf3adb0f901b175a4f88d2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 9 May 2017 08:08:05 +0300 Subject: [PATCH 05/55] Add dependency on FLEX/BISON parts for dbfwfilter The dbfwfilter didn't depend on the two targets which could cause compilation problems when running a parallel build. --- server/modules/filter/dbfwfilter/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/modules/filter/dbfwfilter/CMakeLists.txt b/server/modules/filter/dbfwfilter/CMakeLists.txt index 8f03ad2b0..93c08c498 100644 --- a/server/modules/filter/dbfwfilter/CMakeLists.txt +++ b/server/modules/filter/dbfwfilter/CMakeLists.txt @@ -10,11 +10,13 @@ if(BISON_FOUND AND FLEX_FOUND) target_link_libraries(dbfwfilter maxscale-common) set_target_properties(dbfwfilter PROPERTIES VERSION "1.0.0") install_module(dbfwfilter core) + add_dependencies(dbfwfilter token ruleparser) # The offline rule check utility add_executable(dbfwchk dbfw_rule_check.c ${BISON_ruleparser_OUTPUTS} ${FLEX_token_OUTPUTS}) target_link_libraries(dbfwchk maxscale-common) install_executable(dbfwchk core) + add_dependencies(dbfwchk token ruleparser) else() message(FATAL_ERROR "Could not find Bison or Flex: ${BISON_EXECUTABLE} ${FLEX_EXECUTABLE}") From b2e94fc73c1c51d1dc8c0f790aba3e9cb355671d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 2 May 2017 16:34:41 +0300 Subject: [PATCH 06/55] MXS-1220: Add versioned URLs Added versioning to URLs. This should allow somewhat safe modification to the API after it has been finished. --- server/core/admin.cc | 7 ++++- server/core/httprequest.cc | 38 ++++++++++++++++++---------- server/core/httpresponse.cc | 5 ++++ server/core/maxscale/httprequest.hh | 13 +++++++++- server/core/maxscale/httpresponse.hh | 1 + 5 files changed, 49 insertions(+), 15 deletions(-) diff --git a/server/core/admin.cc b/server/core/admin.cc index c54072b85..558b8a0fc 100644 --- a/server/core/admin.cc +++ b/server/core/admin.cc @@ -102,7 +102,12 @@ int Client::process(string url, string method, const char* upload_data, size_t * } HttpRequest request(m_connection, url, method, json); - HttpResponse reply = resource_handle_request(request); + HttpResponse reply(MHD_HTTP_NOT_FOUND); + + if (request.validate_api_version()) + { + reply = resource_handle_request(request); + } string data; diff --git a/server/core/httprequest.cc b/server/core/httprequest.cc index 12301c0f8..f8580411b 100644 --- a/server/core/httprequest.cc +++ b/server/core/httprequest.cc @@ -74,20 +74,12 @@ static void process_uri(string& uri, std::deque& uri_parts) my_uri.erase(my_uri.begin()); } - if (my_uri.length() == 0) + while (my_uri.length() > 0) { - /** Special handling for the / resource */ - uri_parts.push_back(""); - } - else - { - while (my_uri.length() > 0) - { - size_t pos = my_uri.find("/"); - string part = pos == string::npos ? my_uri : my_uri.substr(0, pos); - my_uri.erase(0, pos == string::npos ? pos : pos + 1); - uri_parts.push_back(part); - } + size_t pos = my_uri.find("/"); + string part = pos == string::npos ? my_uri : my_uri.substr(0, pos); + my_uri.erase(0, pos == string::npos ? pos : pos + 1); + uri_parts.push_back(part); } } @@ -102,9 +94,29 @@ HttpRequest::HttpRequest(struct MHD_Connection *connection, string url, string m m_hostname = mxs_admin_https_enabled() ? HttpRequest::HTTPS_PREFIX : HttpRequest::HTTP_PREFIX; m_hostname += get_header(HTTP_HOST_HEADER); + + if (m_hostname[m_hostname.size() - 1] != '/') + { + m_hostname += "/"; + } + + m_hostname += MXS_REST_API_VERSION; } HttpRequest::~HttpRequest() { } +bool HttpRequest::validate_api_version() +{ + bool rval = false; + + if (m_resource_parts.size() > 0 && + m_resource_parts[0] == MXS_REST_API_VERSION) + { + m_resource_parts.pop_front(); + rval = true; + } + + return rval; +} diff --git a/server/core/httpresponse.cc b/server/core/httpresponse.cc index e0c6e4982..8708d7c11 100644 --- a/server/core/httpresponse.cc +++ b/server/core/httpresponse.cc @@ -33,6 +33,11 @@ HttpResponse::HttpResponse(int code, json_t* response): add_header(HTTP_RESPONSE_HEADER_LAST_MODIFIED, http_date); // This ETag is the base64 encoding of `not-yet-implemented` add_header(HTTP_RESPONSE_HEADER_ETAG, "bm90LXlldC1pbXBsZW1lbnRlZAo"); + + if (m_body) + { + add_header(HTTP_RESPONSE_HEADER_CONTENT_TYPE, "application/json"); + } } HttpResponse::HttpResponse(const HttpResponse& response): diff --git a/server/core/maxscale/httprequest.hh b/server/core/maxscale/httprequest.hh index 6986d676a..e99875c3d 100644 --- a/server/core/maxscale/httprequest.hh +++ b/server/core/maxscale/httprequest.hh @@ -26,6 +26,9 @@ #include "http.hh" +// The API version part of the URL +#define MXS_REST_API_VERSION "v1" + static int value_iterator(void *cls, enum MHD_ValueKind kind, const char *key, @@ -160,6 +163,14 @@ public: { return m_hostname.c_str(); } + + /** + * @brief Drop the API version prefix + * + * @return True if prefix is present and was successfully removed + */ + bool validate_api_version(); + private: /** Constants */ @@ -173,5 +184,5 @@ private: std::deque m_resource_parts; /**< @c m_resource split into parts */ std::string m_verb; /**< Request method */ std::string m_hostname; /**< The value of the Host header */ - struct MHD_Connection* m_connection; + struct MHD_Connection* m_connection; }; diff --git a/server/core/maxscale/httpresponse.hh b/server/core/maxscale/httpresponse.hh index 3bf96672d..4e5801f45 100644 --- a/server/core/maxscale/httpresponse.hh +++ b/server/core/maxscale/httpresponse.hh @@ -30,6 +30,7 @@ #define HTTP_RESPONSE_HEADER_LAST_MODIFIED "Last-Modified" #define HTTP_RESPONSE_HEADER_ETAG "ETag" #define HTTP_RESPONSE_HEADER_ACCEPT "Accept" +#define HTTP_RESPONSE_HEADER_CONTENT_TYPE "Content-Type" typedef std::map Headers; From dcf9c8dab676629637ac801dbd491d0a071f7667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 2 May 2017 22:13:55 +0300 Subject: [PATCH 07/55] MXS-1220: Reorganize sessions resource The sessions resource now follows the JSON API specification: http://jsonapi.org/ This makes the API relatively easier to use as the specification and the client libraries to consume this data exist. --- include/maxscale/config.h | 2 + server/core/config.cc | 2 + server/core/session.cc | 123 +++++++++++++++++++++++++++++--------- 3 files changed, 100 insertions(+), 27 deletions(-) diff --git a/include/maxscale/config.h b/include/maxscale/config.h index e82cb518b..7a14168c2 100644 --- a/include/maxscale/config.h +++ b/include/maxscale/config.h @@ -66,6 +66,7 @@ extern const char CN_FEEDBACK[]; extern const char CN_FILTERS[]; extern const char CN_FILTER[]; extern const char CN_GATEWAY[]; +extern const char CN_ID[]; extern const char CN_LISTENER[]; extern const char CN_LISTENERS[]; extern const char CN_LOCALHOST_MATCH_WILDCARD_HOST[]; @@ -89,6 +90,7 @@ extern const char CN_PROTOCOL[]; extern const char CN_QUERY_CLASSIFIER[]; extern const char CN_QUERY_CLASSIFIER_ARGS[]; extern const char CN_RELATIONSHIPS[]; +extern const char CN_LINKS[]; extern const char CN_REQUIRED[]; extern const char CN_RETRY_ON_FAILURE[]; extern const char CN_ROUTER[]; diff --git a/server/core/config.cc b/server/core/config.cc index 9326f1f72..2de4c5b7a 100644 --- a/server/core/config.cc +++ b/server/core/config.cc @@ -74,6 +74,7 @@ const char CN_FEEDBACK[] = "feedback"; const char CN_FILTERS[] = "filters"; const char CN_FILTER[] = "filter"; const char CN_GATEWAY[] = "gateway"; +const char CN_ID[] = "id"; const char CN_LISTENER[] = "listener"; const char CN_LISTENERS[] = "listeners"; const char CN_LOCALHOST_MATCH_WILDCARD_HOST[] = "localhost_match_wildcard_host"; @@ -97,6 +98,7 @@ const char CN_PROTOCOL[] = "protocol"; const char CN_QUERY_CLASSIFIER[] = "query_classifier"; const char CN_QUERY_CLASSIFIER_ARGS[] = "query_classifier_args"; const char CN_RELATIONSHIPS[] = "relationships"; +const char CN_LINKS[] = "links"; const char CN_REQUIRED[] = "required"; const char CN_RETRY_ON_FAILURE[] = "retry_on_failure"; const char CN_ROUTER[] = "router"; diff --git a/server/core/session.cc b/server/core/session.cc index 26e803452..dc072c329 100644 --- a/server/core/session.cc +++ b/server/core/session.cc @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -54,6 +55,7 @@ #include "maxscale/workertask.hh" using std::string; +using std::stringstream; /** Global session id counter. Must be updated atomically. Value 0 is reserved for * dummy/unused sessions. @@ -1075,32 +1077,83 @@ void session_broadcast_kill_command(MXS_SESSION* issuer, uint64_t target_id) } } -json_t* session_to_json(const MXS_SESSION *session, const char *host) +json_t* session_json_data(const MXS_SESSION *session, const char *host) { - json_t* rval = json_object(); + json_t* data = json_object(); - json_object_set_new(rval, "id", json_integer(session->ses_id)); - json_object_set_new(rval, "state", json_string(session_state(session->state))); + stringstream ss; + ss << session->ses_id; + /** ID and type */ + json_object_set_new(data, "id", json_string(ss.str().c_str())); + json_object_set_new(data, "type", json_string("sessions")); + + /** Relationships */ json_t* rel = json_object(); - json_t* arr = json_array(); + /** Service relationship (one-to-one) */ string svc = host; svc += "/services/"; - svc += session->service->name; - json_array_append_new(arr, json_string(svc.c_str())); - json_object_set_new(rel, "services", arr); - json_object_set_new(rval, "relationships", rel); + json_t* services = json_object(); + + /** Service self link */ + json_t* services_links = json_object(); + json_object_set_new(services_links, "self", json_string(svc.c_str())); + + /** Only one value for data, the service where this session belongs to */ + json_t* services_data_value = json_object(); + json_object_set_new(services_data_value, "id", json_string(session->service->name)); + json_object_set_new(services_data_value, "type", json_string("services")); + + json_object_set_new(services, "links", services_links); + json_object_set_new(services, "data", services_data_value); + json_object_set_new(rel, "services", services); + + /** Filter relationships (one-to-many) */ + if (session->n_filters) + { + json_t* filters = json_object(); + + /** Filter self link */ + string fil = host; + fil += "/filters/"; + json_t* filters_links = json_object(); + json_object_set_new(filters_links, "self", json_string(fil.c_str())); + + /** Array of data values */ + json_t* filters_data_array = json_array(); + + for (int i = 0; i < session->n_filters; i++) + { + /** Each value is a reference to a filter */ + json_t* filters_data_value = json_object(); + + json_object_set_new(filters_data_value, "id", json_string(session->filters[i].filter->name)); + json_object_set_new(filters_data_value, "type", json_string("filters")); + + json_array_append_new(filters_data_array, filters_data_value); + } + + json_object_set_new(filters, "data", filters_data_array); + json_object_set_new(filters, "links", filters_links); + json_object_set_new(rel, "filters", filters); + } + + json_object_set_new(data, "relationships", rel); + + /** Session attributes */ + json_t* attr = json_object(); + json_object_set_new(attr, "state", json_string(session_state(session->state))); if (session->client_dcb->user) { - json_object_set_new(rval, "user", json_string(session->client_dcb->user)); + json_object_set_new(attr, "user", json_string(session->client_dcb->user)); } if (session->client_dcb->remote) { - json_object_set_new(rval, "remote", json_string(session->client_dcb->remote)); + json_object_set_new(attr, "remote", json_string(session->client_dcb->remote)); } struct tm result; @@ -1109,29 +1162,33 @@ json_t* session_to_json(const MXS_SESSION *session, const char *host) asctime_r(localtime_r(&session->stats.connect, &result), buf); trim(buf); - json_object_set_new(rval, "connected", json_string(buf)); + json_object_set_new(attr, "connected", json_string(buf)); if (session->client_dcb->state == DCB_STATE_POLLING) { double idle = (hkheartbeat - session->client_dcb->last_read); idle = idle > 0 ? idle / 10.f : 0; - json_object_set_new(rval, "idle", json_real(idle)); + json_object_set_new(attr, "idle", json_real(idle)); } - if (session->n_filters) - { - json_t* filters = json_array(); + json_object_set_new(data, "attributes", attr); - for (int i = 0; i < session->n_filters; i++) - { - string fil = host; - fil += "/filters/"; - fil += session->filters[i].filter->name; - json_array_append_new(filters, json_string(fil.c_str())); - } + return data; +} - json_object_set_new(rval, "filters", filters); - } +json_t* session_to_json(const MXS_SESSION *session, const char *host) +{ + json_t* rval = json_object(); + + /** Create the top level self link */ + stringstream self; + self << host << "/sessions/" << session->ses_id; + json_t* links_self = json_object(); + json_object_set_new(links_self, "self", json_string(self.str().c_str())); + json_object_set_new(rval, "links", links_self); + + /** Only one value */ + json_object_set_new(rval, "data", session_json_data(session, host)); return rval; } @@ -1145,7 +1202,7 @@ struct SessionListData bool seslist_cb(DCB* dcb, void* data) { SessionListData* d = (SessionListData*)data; - json_array_append_new(d->json, session_to_json(dcb->session, d->host)); + json_array_append_new(d->json, session_json_data(dcb->session, d->host)); return true; } @@ -1153,5 +1210,17 @@ json_t* session_list_to_json(const char* host) { SessionListData data = {json_array(), host}; dcb_foreach(seslist_cb, &data); - return data.json; + json_t* rval = json_object(); + + /** Create the top level self link */ + stringstream self; + self << host << "/sessions/"; + json_t* links_self = json_object(); + json_object_set_new(links_self, "self", json_string(self.str().c_str())); + json_object_set_new(rval, "links", links_self); + + /** Array of session values */ + json_object_set_new(rval, "data", data.json); + + return rval; } From 49b45acd1330240bc518095e9a23c1b3ed9e196d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Wed, 3 May 2017 07:52:15 +0300 Subject: [PATCH 08/55] MXS-1220: Reorganize server resource The server resource now conforms to the JSON API schema. --- include/maxscale/config.h | 2 + server/core/config.cc | 2 + server/core/monitor.cc | 34 +++++++-- server/core/server.cc | 149 ++++++++++++++++++++------------------ server/core/service.cc | 49 ++++++++++--- 5 files changed, 147 insertions(+), 89 deletions(-) diff --git a/include/maxscale/config.h b/include/maxscale/config.h index 7a14168c2..5282a4042 100644 --- a/include/maxscale/config.h +++ b/include/maxscale/config.h @@ -52,6 +52,7 @@ extern const char CN_ADMIN_USER[]; extern const char CN_ADMIN_SSL_KEY[]; extern const char CN_ADMIN_SSL_CERT[]; extern const char CN_ADMIN_SSL_CA_CERT[]; +extern const char CN_ATTRIBUTES[]; extern const char CN_AUTHENTICATOR[]; extern const char CN_AUTHENTICATOR_OPTIONS[]; extern const char CN_AUTH_ALL_SERVERS[]; @@ -60,6 +61,7 @@ extern const char CN_AUTH_READ_TIMEOUT[]; extern const char CN_AUTH_WRITE_TIMEOUT[]; extern const char CN_AUTO[]; extern const char CN_CONNECTION_TIMEOUT[]; +extern const char CN_DATA[]; extern const char CN_DEFAULT[]; extern const char CN_ENABLE_ROOT_USER[]; extern const char CN_FEEDBACK[]; diff --git a/server/core/config.cc b/server/core/config.cc index 2de4c5b7a..50da51ff5 100644 --- a/server/core/config.cc +++ b/server/core/config.cc @@ -60,6 +60,7 @@ const char CN_ADMIN_USER[] = "admin_user"; const char CN_ADMIN_SSL_KEY[] = "admin_ssl_key"; const char CN_ADMIN_SSL_CERT[] = "admin_ssl_cert"; const char CN_ADMIN_SSL_CA_CERT[] = "admin_ssl_ca_cert"; +const char CN_ATTRIBUTES[] = "attributes"; const char CN_AUTHENTICATOR[] = "authenticator"; const char CN_AUTHENTICATOR_OPTIONS[] = "authenticator_options"; const char CN_AUTH_ALL_SERVERS[] = "auth_all_servers"; @@ -68,6 +69,7 @@ const char CN_AUTH_READ_TIMEOUT[] = "auth_read_timeout"; const char CN_AUTH_WRITE_TIMEOUT[] = "auth_write_timeout"; const char CN_AUTO[] = "auto"; const char CN_CONNECTION_TIMEOUT[] = "connection_timeout"; +const char CN_DATA[] = "data"; const char CN_DEFAULT[] = "default"; const char CN_ENABLE_ROOT_USER[] = "enable_root_user"; const char CN_FEEDBACK[] = "feedback"; diff --git a/server/core/monitor.cc b/server/core/monitor.cc index 9f2299ff1..d2f67bfa7 100644 --- a/server/core/monitor.cc +++ b/server/core/monitor.cc @@ -1566,9 +1566,31 @@ json_t* monitor_list_to_json(const char* host) return rval; } +static json_t* monitor_self_link(const char* host) +{ + json_t* rel_links = json_object(); + + string links = host; + links += "/monitors/"; + json_object_set_new(rel_links, CN_SELF, json_string(links.c_str())); + + return rel_links; +} + +static void add_monitor_relation(json_t* arr, const MXS_MONITOR* monitor) +{ + json_t* obj = json_object(); + json_object_set_new(obj, CN_ID, json_string(monitor->name)); + json_object_set_new(obj, CN_TYPE, json_string(CN_MONITORS)); + json_array_append_new(arr, obj); +} + json_t* monitor_relations_to_server(const SERVER* server, const char* host) { - json_t* arr = json_array(); + json_t* rel = json_object(); + json_t* rel_data = json_array(); + + json_object_set_new(rel, CN_LINKS, monitor_self_link(host)); spinlock_acquire(&monLock); @@ -1580,10 +1602,8 @@ json_t* monitor_relations_to_server(const SERVER* server, const char* host) { if (db->server == server) { - string m = host; - m += "/monitors/"; - m += mon->name; - json_array_append_new(arr, json_string(m.c_str())); + add_monitor_relation(rel_data, mon); + break; } } @@ -1592,5 +1612,7 @@ json_t* monitor_relations_to_server(const SERVER* server, const char* host) spinlock_release(&monLock); - return arr; + json_object_set_new(rel, CN_DATA, rel_data); + + return rel; } diff --git a/server/core/server.cc b/server/core/server.cc index fdb0f2634..7b7df38ec 100644 --- a/server/core/server.cc +++ b/server/core/server.cc @@ -1360,44 +1360,22 @@ bool server_is_mxs_service(const SERVER *server) return rval; } -json_t* server_list_to_json(const char* host) +json_t* server_self_link(const char* host) { - json_t* rval = json_array(); + json_t* links = json_object(); + string self = host; + self += "/servers/"; + json_object_set_new(links, CN_SELF, json_string(self.c_str())); - if (rval) - { - spinlock_acquire(&server_spin); - - for (SERVER* server = allServers; server; server = server->next) - { - if (SERVER_IS_ACTIVE(server)) - { - json_t* srv_json = server_to_json(server, host); - - if (srv_json == NULL) - { - json_decref(rval); - rval = NULL; - break; - } - - json_array_append_new(rval, srv_json); - } - } - - spinlock_release(&server_spin); - } - - return rval; + return links; } -json_t* server_to_json(const SERVER* server, const char* host) +static json_t* server_json_attributes(const SERVER* server) { - json_t* rval = json_object(); + /** Resource attributes */ + json_t* attr = json_object(); - json_object_set_new(rval, CN_NAME, json_string(server->unique_name)); - - /** Store server parameters */ + /** Store server parameters in attributes */ json_t* params = json_object(); json_object_set_new(params, CN_ADDRESS, json_string(server->name)); @@ -1419,21 +1397,22 @@ json_t* server_to_json(const SERVER* server, const char* host) json_object_set_new(params, p->name, json_string(p->value)); } - json_object_set_new(rval, CN_PARAMETERS, params); + json_object_set_new(attr, CN_PARAMETERS, params); + /** Store general information about the server state */ char* stat = server_status(server); - json_object_set_new(rval, "status", json_string(stat)); + json_object_set_new(attr, CN_STATUS, json_string(stat)); MXS_FREE(stat); if (server->server_string) { - json_object_set_new(rval, "version", json_string(server->server_string)); + json_object_set_new(attr, CN_VERSION_STRING, json_string(server->server_string)); } - json_object_set_new(rval, "node_id", json_integer(server->node_id)); - json_object_set_new(rval, "master_id", json_integer(server->master_id)); - json_object_set_new(rval, "replication_depth", json_integer(server->depth)); + json_object_set_new(attr, "node_id", json_integer(server->node_id)); + json_object_set_new(attr, "master_id", json_integer(server->master_id)); + json_object_set_new(attr, "replication_depth", json_integer(server->depth)); if (server->slaves) { @@ -1444,12 +1423,12 @@ json_t* server_to_json(const SERVER* server, const char* host) json_array_append_new(slaves, json_integer(server->slaves[i])); } - json_object_set_new(rval, "slaves", slaves); + json_object_set_new(attr, "slaves", slaves); } if (server->rlag >= 0) { - json_object_set_new(rval, "replication_lag", json_integer(server->rlag)); + json_object_set_new(attr, "replication_lag", json_integer(server->rlag)); } if (server->node_ts > 0) @@ -1460,7 +1439,7 @@ json_t* server_to_json(const SERVER* server, const char* host) asctime_r(localtime_r(&tim, &result), timebuf); trim(timebuf); - json_object_set_new(rval, "last_heartbeat", json_string(timebuf)); + json_object_set_new(attr, "last_heartbeat", json_string(timebuf)); } /** Store statistics */ @@ -1470,39 +1449,67 @@ json_t* server_to_json(const SERVER* server, const char* host) json_object_set_new(stats, "total_connections", json_integer(server->stats.n_connections)); json_object_set_new(stats, "active_operations", json_integer(server->stats.n_current_ops)); - json_object_set_new(rval, "statictics", stats); + json_object_set_new(attr, "statictics", stats); - /** Store relationships to other objects */ + return attr; +} + +static json_t* server_to_json_data(const SERVER* server, const char* host) +{ + json_t* rval = json_object(); + + /** Add resource identifiers */ + json_object_set_new(rval, CN_ID, json_string(server->unique_name)); + json_object_set_new(rval, CN_TYPE, json_string(CN_SERVERS)); + + /** Relationships */ json_t* rel = json_object(); - - string self = host; - self += "/servers/"; - self += server->unique_name; - json_object_set_new(rel, CN_SELF, json_string(self.c_str())); - - json_t* arr = service_relations_to_server(server, host); - - if (json_array_size(arr) > 0) - { - json_object_set_new(rel, CN_SERVICES, arr); - } - else - { - json_decref(arr); - } - - arr = monitor_relations_to_server(server, host); - - if (json_array_size(arr) > 0) - { - json_object_set_new(rel, CN_MONITORS, arr); - } - else - { - json_decref(arr); - } - + json_object_set_new(rel, CN_SERVICES, service_relations_to_server(server, host)); + json_object_set_new(rel, CN_MONITORS, monitor_relations_to_server(server, host)); json_object_set_new(rval, CN_RELATIONSHIPS, rel); + /** Attributes */ + json_object_set_new(rval, CN_ATTRIBUTES, server_json_attributes(server)); + + return rval; +} + +json_t* server_to_json(const SERVER* server, const char* host) +{ + json_t* rval = json_object(); + + /** Top level self link */ + json_object_set_new(rval, CN_LINKS, server_self_link(host)); + + /** Add server data */ + json_object_set_new(rval, CN_DATA, server_to_json_data(server, host)); + + return rval; +} + +json_t* server_list_to_json(const char* host) +{ + json_t* data = json_array(); + + spinlock_acquire(&server_spin); + + for (SERVER* server = allServers; server; server = server->next) + { + if (SERVER_IS_ACTIVE(server)) + { + json_array_append_new(data, server_to_json_data(server, host)); + } + } + + spinlock_release(&server_spin); + + json_t* rval = json_object(); + + /** Top level self link */ + json_object_set_new(rval, CN_LINKS, server_self_link(host)); + + /** Add server data array */ + json_object_set_new(rval, CN_DATA, data); + return rval; } diff --git a/server/core/service.cc b/server/core/service.cc index 130e2a487..b38c2f7d7 100644 --- a/server/core/service.cc +++ b/server/core/service.cc @@ -2240,7 +2240,8 @@ static bool create_service_config(const SERVICE *service, const char *filename) dprintf(file, "%s=%ld\n", CN_CONNECTION_TIMEOUT, service->conn_idle_timeout); dprintf(file, "%s=%s\n", CN_AUTH_ALL_SERVERS, service->users_from_all ? "true" : "false"); dprintf(file, "%s=%s\n", CN_STRIP_DB_ESC, service->strip_db_esc ? "true" : "false"); - dprintf(file, "%s=%s\n", CN_LOCALHOST_MATCH_WILDCARD_HOST, service->localhost_match_wildcard_host ? "true" : "false"); + dprintf(file, "%s=%s\n", CN_LOCALHOST_MATCH_WILDCARD_HOST, + service->localhost_match_wildcard_host ? "true" : "false"); dprintf(file, "%s=%s\n", CN_VERSION_STRING, service->version_string); dprintf(file, "%s=%s\n", CN_WEIGHTBY, service->weightby); dprintf(file, "%s=%s\n", CN_LOG_AUTH_WARNINGS, service->log_auth_warnings ? "true" : "false"); @@ -2571,17 +2572,32 @@ json_t* service_list_to_json(const char* host) return rval; } -static void add_service_relation(json_t* arr, const char* host, const SERVICE* service) +static void add_service_relation(json_t* arr, const SERVICE* service) { - string svc = host; - svc += "/services/"; - svc += service->name; - json_array_append_new(arr, json_string(svc.c_str())); + json_t* obj = json_object(); + json_object_set_new(obj, CN_ID, json_string(service->name)); + json_object_set_new(obj, CN_TYPE, json_string(CN_SERVICES)); + json_array_append_new(arr, obj); +} + +static json_t* service_self_link(const char* host) +{ + json_t* rel_links = json_object(); + + string links = host; + links += "/services/"; + json_object_set_new(rel_links, CN_SELF, json_string(links.c_str())); + + return rel_links; } json_t* service_relations_to_filter(const MXS_FILTER_DEF* filter, const char* host) { - json_t* arr = json_array(); + json_t* rel = json_object(); + json_t* rel_data = json_array(); + + json_object_set_new(rel, CN_LINKS, service_self_link(host)); + spinlock_acquire(&service_spin); for (SERVICE *service = allServices; service; service = service->next) @@ -2592,7 +2608,7 @@ json_t* service_relations_to_filter(const MXS_FILTER_DEF* filter, const char* ho { if (service->filters[i] == filter) { - add_service_relation(arr, host, service); + add_service_relation(rel_data, service); } } @@ -2601,12 +2617,19 @@ json_t* service_relations_to_filter(const MXS_FILTER_DEF* filter, const char* ho spinlock_release(&service_spin); - return arr; + json_object_set_new(rel, CN_DATA, rel_data); + + return rel; } + json_t* service_relations_to_server(const SERVER* server, const char* host) { - json_t* arr = json_array(); + json_t* rel = json_object(); + json_t* rel_data = json_array(); + + json_object_set_new(rel, CN_LINKS, service_self_link(host)); + spinlock_acquire(&service_spin); for (SERVICE *service = allServices; service; service = service->next) @@ -2617,7 +2640,7 @@ json_t* service_relations_to_server(const SERVER* server, const char* host) { if (ref->server == server && SERVER_REF_IS_ACTIVE(ref)) { - add_service_relation(arr, host, service); + add_service_relation(rel_data, service); } } @@ -2626,5 +2649,7 @@ json_t* service_relations_to_server(const SERVER* server, const char* host) spinlock_release(&service_spin); - return arr; + json_object_set_new(rel, CN_DATA, rel_data); + + return rel; } From 624434a6d4b0c95ff195eb800a9a2b866e3dabda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Wed, 3 May 2017 11:26:43 +0300 Subject: [PATCH 09/55] MXS-1220: Compare headers case-insensitively The header names are not case sensitive. --- server/core/admin.cc | 2 +- server/core/maxscale/httprequest.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/core/admin.cc b/server/core/admin.cc index 558b8a0fc..26ae490ea 100644 --- a/server/core/admin.cc +++ b/server/core/admin.cc @@ -55,7 +55,7 @@ int kv_iter(void *cls, { size_t* rval = (size_t*)cls; - if (strcmp(key, "Content-Length") == 0) + if (strcasecmp(key, "Content-Length") == 0) { *rval = atoi(value); return MHD_NO; diff --git a/server/core/maxscale/httprequest.hh b/server/core/maxscale/httprequest.hh index e99875c3d..c1cabe4fb 100644 --- a/server/core/maxscale/httprequest.hh +++ b/server/core/maxscale/httprequest.hh @@ -36,7 +36,7 @@ static int value_iterator(void *cls, { std::pair* cmp = (std::pair*)cls; - if (cmp->first == key) + if (strcasecmp(cmp->first.c_str(), key) == 0) { cmp->second = value; return MHD_NO; From 5ae9ff9663c514ff17663185c1fc9f7ae5edf5c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Wed, 3 May 2017 11:56:56 +0300 Subject: [PATCH 10/55] MXS-1220: Add simple REST API validator The validator uses Node.js to check that the REST API endpoints conform to the specified schema. --- server/core/test/rest-api/package.json | 19 + .../test/rest-api/test/schema_validation.js | 45 ++ server/core/test/rest-api/utils.js | 422 ++++++++++++++++++ 3 files changed, 486 insertions(+) create mode 100644 server/core/test/rest-api/package.json create mode 100644 server/core/test/rest-api/test/schema_validation.js create mode 100644 server/core/test/rest-api/utils.js diff --git a/server/core/test/rest-api/package.json b/server/core/test/rest-api/package.json new file mode 100644 index 000000000..6e24487e1 --- /dev/null +++ b/server/core/test/rest-api/package.json @@ -0,0 +1,19 @@ +{ + "name": "rest-api-tests", + "version": "1.0.0", + "repository": "https://github.com/mariadb-corporation/MaxScale", + "description": "MaxScale REST API tests", + "scripts": { + "test": "mocha" + }, + "author": "", + "license": "SEE LICENSE IN ../../../../LICENSE.txt", + "dependencies": { + "ajv": "^5.0.1", + "chai": "^3.5.0", + "chai-as-promised": "^6.0.0", + "mocha": "^3.3.0", + "request": "^2.81.0", + "request-promise-native": "^1.0.3" + } +} diff --git a/server/core/test/rest-api/test/schema_validation.js b/server/core/test/rest-api/test/schema_validation.js new file mode 100644 index 000000000..f7ff12a31 --- /dev/null +++ b/server/core/test/rest-api/test/schema_validation.js @@ -0,0 +1,45 @@ +require("../utils.js")() + +describe("Resource Collections", function(){ + + var tests = [ + "/servers/", + "/sessions/", + "/services/", + "/monitors/", + "/filters/", + ] + + tests.forEach(function(endpoint){ + it(endpoint + ': resource should be found', function() { + return request(base_url + endpoint) + .should.be.fulfilled + }); + + it(endpoint + ': resource schema should be valid', function() { + return request(base_url + endpoint) + .should.eventually.satisfy(validate) + }); + }) +}); + +describe("Individual Resources", function(){ + + var tests = [ + "/servers/server1", + "/servers/server2", + "/sessions/1", + ] + + tests.forEach(function(endpoint){ + it(endpoint + ': resource should be found', function() { + return request(base_url + endpoint) + .should.be.fulfilled + }); + + it(endpoint + ': resource schema should be valid', function() { + return request(base_url + endpoint) + .should.eventually.satisfy(validate) + }); + }) +}); diff --git a/server/core/test/rest-api/utils.js b/server/core/test/rest-api/utils.js new file mode 100644 index 000000000..dadb26586 --- /dev/null +++ b/server/core/test/rest-api/utils.js @@ -0,0 +1,422 @@ +var json_api_schema = { + "title": "JSON API Schema", + "description": "This is a schema for responses in the JSON API format. For more, see http://jsonapi.org", + "oneOf": [ + { + "$ref": "#/definitions/success" + }, + { + "$ref": "#/definitions/failure" + }, + { + "$ref": "#/definitions/info" + } + ], + "definitions": { + "success": { + "type": "object", + "required": [ + "data" + ], + "properties": { + "data": { + "$ref": "#/definitions/data" + }, + "included": { + "description": "To reduce the number of HTTP requests, servers **MAY** allow responses that include related resources along with the requested primary resources. Such responses are called \"compound documents\".", + "type": "array", + "items": { + "$ref": "#/definitions/resource" + }, + "uniqueItems": true + }, + "meta": { + "$ref": "#/definitions/meta" + }, + "links": { + "description": "Link members related to the primary data.", + "allOf": [ + { + "$ref": "#/definitions/links" + }, + { + "$ref": "#/definitions/pagination" + } + ] + }, + "jsonapi": { + "$ref": "#/definitions/jsonapi" + } + }, + "additionalProperties": false + }, + "failure": { + "type": "object", + "required": [ + "errors" + ], + "properties": { + "errors": { + "type": "array", + "items": { + "$ref": "#/definitions/error" + }, + "uniqueItems": true + }, + "meta": { + "$ref": "#/definitions/meta" + }, + "jsonapi": { + "$ref": "#/definitions/jsonapi" + }, + "links": { + "$ref": "#/definitions/links" + } + }, + "additionalProperties": false + }, + "info": { + "type": "object", + "required": [ + "meta" + ], + "properties": { + "meta": { + "$ref": "#/definitions/meta" + }, + "links": { + "$ref": "#/definitions/links" + }, + "jsonapi": { + "$ref": "#/definitions/jsonapi" + } + }, + "additionalProperties": false + }, + "meta": { + "description": "Non-standard meta-information that can not be represented as an attribute or relationship.", + "type": "object", + "additionalProperties": true + }, + "data": { + "description": "The document's \"primary data\" is a representation of the resource or collection of resources targeted by a request.", + "oneOf": [ + { + "$ref": "#/definitions/resource" + }, + { + "description": "An array of resource objects, an array of resource identifier objects, or an empty array ([]), for requests that target resource collections.", + "type": "array", + "items": { + "$ref": "#/definitions/resource" + }, + "uniqueItems": true + }, + { + "description": "null if the request is one that might correspond to a single resource, but doesn't currently.", + "type": "null" + } + ] + }, + "resource": { + "description": "\"Resource objects\" appear in a JSON API document to represent resources.", + "type": "object", + "required": [ + "type", + "id" + ], + "properties": { + "type": { + "type": "string" + }, + "id": { + "type": "string" + }, + "attributes": { + "$ref": "#/definitions/attributes" + }, + "relationships": { + "$ref": "#/definitions/relationships" + }, + "links": { + "$ref": "#/definitions/links" + }, + "meta": { + "$ref": "#/definitions/meta" + } + }, + "additionalProperties": false + }, + "links": { + "description": "A resource object **MAY** contain references to other resource objects (\"relationships\"). Relationships may be to-one or to-many. Relationships can be specified by including a member in a resource's links object.", + "type": "object", + "properties": { + "self": { + "description": "A `self` member, whose value is a URL for the relationship itself (a \"relationship URL\"). This URL allows the client to directly manipulate the relationship. For example, it would allow a client to remove an `author` from an `article` without deleting the people resource itself.", + "type": "string", + "format": "uri" + }, + "related": { + "$ref": "#/definitions/link" + } + }, + "additionalProperties": true + }, + "link": { + "description": "A link **MUST** be represented as either: a string containing the link's URL or a link object.", + "oneOf": [ + { + "description": "A string containing the link's URL.", + "type": "string", + "format": "uri" + }, + { + "type": "object", + "required": [ + "href" + ], + "properties": { + "href": { + "description": "A string containing the link's URL.", + "type": "string", + "format": "uri" + }, + "meta": { + "$ref": "#/definitions/meta" + } + } + } + ] + }, + "attributes": { + "description": "Members of the attributes object (\"attributes\") represent information about the resource object in which it's defined.", + "type": "object", + "patternProperties": { + "^(?!relationships$|links$)\\w[-\\w_]*$": { + "description": "Attributes may contain any valid JSON value." + } + }, + "additionalProperties": false + }, + "relationships": { + "description": "Members of the relationships object (\"relationships\") represent references from the resource object in which it's defined to other resource objects.", + "type": "object", + "patternProperties": { + "^\\w[-\\w_]*$": { + "properties": { + "links": { + "$ref": "#/definitions/links" + }, + "data": { + "description": "Member, whose value represents \"resource linkage\".", + "oneOf": [ + { + "$ref": "#/definitions/relationshipToOne" + }, + { + "$ref": "#/definitions/relationshipToMany" + } + ] + }, + "meta": { + "$ref": "#/definitions/meta" + } + }, + "anyOf": [ + { + "required": [ + "data" + ] + }, + { + "required": [ + "meta" + ] + }, + { + "required": [ + "links" + ] + } + ], + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "relationshipToOne": { + "description": "References to other resource objects in a to-one (\"relationship\"). Relationships can be specified by including a member in a resource's links object.", + "anyOf": [ + { + "$ref": "#/definitions/empty" + }, + { + "$ref": "#/definitions/linkage" + } + ] + }, + "relationshipToMany": { + "description": "An array of objects each containing \"type\" and \"id\" members for to-many relationships.", + "type": "array", + "items": { + "$ref": "#/definitions/linkage" + }, + "uniqueItems": true + }, + "empty": { + "description": "Describes an empty to-one relationship.", + "type": "null" + }, + "linkage": { + "description": "The \"type\" and \"id\" to non-empty members.", + "type": "object", + "required": [ + "type", + "id" + ], + "properties": { + "type": { + "type": "string" + }, + "id": { + "type": "string" + }, + "meta": { + "$ref": "#/definitions/meta" + } + }, + "additionalProperties": false + }, + "pagination": { + "type": "object", + "properties": { + "first": { + "description": "The first page of data", + "oneOf": [ + { + "type": "string", + "format": "uri" + }, + { + "type": "null" + } + ] + }, + "last": { + "description": "The last page of data", + "oneOf": [ + { + "type": "string", + "format": "uri" + }, + { + "type": "null" + } + ] + }, + "prev": { + "description": "The previous page of data", + "oneOf": [ + { + "type": "string", + "format": "uri" + }, + { + "type": "null" + } + ] + }, + "next": { + "description": "The next page of data", + "oneOf": [ + { + "type": "string", + "format": "uri" + }, + { + "type": "null" + } + ] + } + } + }, + "jsonapi": { + "description": "An object describing the server's implementation", + "type": "object", + "properties": { + "version": { + "type": "string" + }, + "meta": { + "$ref": "#/definitions/meta" + } + }, + "additionalProperties": false + }, + "error": { + "type": "object", + "properties": { + "id": { + "description": "A unique identifier for this particular occurrence of the problem.", + "type": "string" + }, + "links": { + "$ref": "#/definitions/links" + }, + "status": { + "description": "The HTTP status code applicable to this problem, expressed as a string value.", + "type": "string" + }, + "code": { + "description": "An application-specific error code, expressed as a string value.", + "type": "string" + }, + "title": { + "description": "A short, human-readable summary of the problem. It **SHOULD NOT** change from occurrence to occurrence of the problem, except for purposes of localization.", + "type": "string" + }, + "detail": { + "description": "A human-readable explanation specific to this occurrence of the problem.", + "type": "string" + }, + "source": { + "type": "object", + "properties": { + "pointer": { + "description": "A JSON Pointer [RFC6901] to the associated entity in the request document [e.g. \"/data\" for a primary data object, or \"/data/attributes/title\" for a specific attribute].", + "type": "string" + }, + "parameter": { + "description": "A string indicating which query parameter caused the error.", + "type": "string" + } + } + }, + "meta": { + "$ref": "#/definitions/meta" + } + }, + "additionalProperties": false + } + } +} + +function validate_json(data) { + return validate_func(JSON.parse(data)) +} + +module.exports = function() { + this.fs = require("fs") + this.request = require("request-promise-native") + this.chai = require("chai") + this.assert = require("assert") + this.chaiAsPromised = require("chai-as-promised") + chai.use(chaiAsPromised) + this.should = chai.should() + this.expect = chai.expect + this.Ajv = require("ajv") + this.ajv = new Ajv({$data: true, allErrors: true, extendRefs: true, verbose: true}) + this.validate_func = ajv.compile(json_api_schema) + this.validate = validate_json + this.base_url = "http://localhost:8989/v1" +} From ca62749f25b3fd405545d141a204d6157b961757 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Wed, 3 May 2017 13:32:43 +0300 Subject: [PATCH 11/55] MXS-1220: Factor out common code from JSON object creation The JSON objects that are created from the various core MaxScale objects share a lot of common code. Moving this into a separate files removes the redundant code. --- include/maxscale/config.h | 1 + include/maxscale/json_api.h | 64 +++++++++++++++++++++++++++++ server/core/CMakeLists.txt | 1 + server/core/config.cc | 1 + server/core/json_api.cc | 63 +++++++++++++++++++++++++++++ server/core/monitor.cc | 29 ++------------ server/core/server.cc | 21 ++-------- server/core/service.cc | 42 ++++--------------- server/core/session.cc | 80 ++++++------------------------------- 9 files changed, 156 insertions(+), 146 deletions(-) create mode 100644 include/maxscale/json_api.h create mode 100644 server/core/json_api.cc diff --git a/include/maxscale/config.h b/include/maxscale/config.h index 5282a4042..027d07938 100644 --- a/include/maxscale/config.h +++ b/include/maxscale/config.h @@ -102,6 +102,7 @@ extern const char CN_SERVERS[]; extern const char CN_SERVER[]; extern const char CN_SERVICES[]; extern const char CN_SERVICE[]; +extern const char CN_SESSIONS[]; extern const char CN_SKIP_PERMISSION_CHECKS[]; extern const char CN_SOCKET[]; extern const char CN_STATE[]; diff --git a/include/maxscale/json_api.h b/include/maxscale/json_api.h new file mode 100644 index 000000000..dfe5f1418 --- /dev/null +++ b/include/maxscale/json_api.h @@ -0,0 +1,64 @@ +#pragma once +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl11. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +/** + * @file Helper functions for creating JSON API conforming objects + */ + +#include +#include + +MXS_BEGIN_DECLS + +/** Resource endpoints */ +#define MXS_JSON_API_SERVERS "/servers/" +#define MXS_JSON_API_SERVICES "/services/" +#define MXS_JSON_API_FILTERS "/filters/" +#define MXS_JSON_API_MONITORS "/monitors/" +#define MXS_JSON_API_SESSIONS "/sessions/" +#define MXS_JSON_API_MAXSCALE "/maxscale/" + +/** + * @brief Create a JSON object + * + * The caller should add a `data` field to the returned object. + * + * @param host Hostname of this server + * @param self Endpoint of this resource + * @param data The JSON data, either an array or an object + * + * @return A valid top-level JSON API object + */ +json_t* mxs_json_resource(const char* host, const char* self, json_t* data); + +/** + * @brief Create an empty relationship object + * + * @param host Hostname of this server + * @param endpoint The endpoint for the resource's collection + * + * @return New relationship object + */ +json_t* mxs_json_relationship(const char* host, const char* endpoint); + +/** + * @brief Add an item to a relationship object + * + * @param rel Relationship created with mxs_json_relationship + * @param id The resource identifier + * @param type The resource type + */ +void mxs_json_add_relation(json_t* rel, const char* id, const char* type); + +MXS_END_DECLS diff --git a/server/core/CMakeLists.txt b/server/core/CMakeLists.txt index 736d67e71..78db4c51a 100644 --- a/server/core/CMakeLists.txt +++ b/server/core/CMakeLists.txt @@ -15,6 +15,7 @@ add_library(maxscale-common SHARED housekeeper.cc httprequest.cc httpresponse.cc + json_api.cc listener.cc load_utils.cc log_manager.cc diff --git a/server/core/config.cc b/server/core/config.cc index 50da51ff5..6e59791a8 100644 --- a/server/core/config.cc +++ b/server/core/config.cc @@ -110,6 +110,7 @@ const char CN_SERVERS[] = "servers"; const char CN_SERVER[] = "server"; const char CN_SERVICES[] = "services"; const char CN_SERVICE[] = "service"; +const char CN_SESSIONS[] = "sessions"; const char CN_SKIP_PERMISSION_CHECKS[] = "skip_permission_checks"; const char CN_SOCKET[] = "socket"; const char CN_STATE[] = "state"; diff --git a/server/core/json_api.cc b/server/core/json_api.cc new file mode 100644 index 000000000..2c820bfc7 --- /dev/null +++ b/server/core/json_api.cc @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl11. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include + +#include + +#include +#include + +using std::string; + +static json_t* self_link(const char* host, const char* endpoint) +{ + json_t* self_link = json_object(); + string links = host; + links += endpoint; + json_object_set_new(self_link, CN_SELF, json_string(links.c_str())); + + return self_link; +} + +json_t* mxs_json_resource(const char* host, const char* self, json_t* data) +{ + ss_dassert(data && (json_is_array(data) || json_is_object(data))); + json_t* rval = json_object(); + json_object_set_new(rval, CN_LINKS, self_link(host, self)); + json_object_set_new(rval, CN_DATA, data); + return rval; +} + +json_t* mxs_json_relationship(const char* host, const char* endpoint) +{ + json_t* rel = json_object(); + + /** Add the relation self link */ + json_object_set_new(rel, CN_LINKS, self_link(host, endpoint)); + + /** Add empty array of relations */ + json_object_set_new(rel, CN_DATA, json_array()); + return rel; +} + +void mxs_json_add_relation(json_t* rel, const char* id, const char* type) +{ + json_t* data = json_object_get(rel, CN_DATA); + ss_dassert(data && json_is_array(data)); + + json_t* obj = json_object(); + json_object_set_new(obj, CN_ID, json_string(id)); + json_object_set_new(obj, CN_TYPE, json_string(type)); + json_array_append(data, obj); +} diff --git a/server/core/monitor.cc b/server/core/monitor.cc index d2f67bfa7..13ea53081 100644 --- a/server/core/monitor.cc +++ b/server/core/monitor.cc @@ -36,6 +36,7 @@ #include "maxscale/externcmd.h" #include "maxscale/monitor.h" #include "maxscale/modules.h" +#include "maxscale/json_api.h" using std::string; using std::set; @@ -1566,31 +1567,9 @@ json_t* monitor_list_to_json(const char* host) return rval; } -static json_t* monitor_self_link(const char* host) -{ - json_t* rel_links = json_object(); - - string links = host; - links += "/monitors/"; - json_object_set_new(rel_links, CN_SELF, json_string(links.c_str())); - - return rel_links; -} - -static void add_monitor_relation(json_t* arr, const MXS_MONITOR* monitor) -{ - json_t* obj = json_object(); - json_object_set_new(obj, CN_ID, json_string(monitor->name)); - json_object_set_new(obj, CN_TYPE, json_string(CN_MONITORS)); - json_array_append_new(arr, obj); -} - json_t* monitor_relations_to_server(const SERVER* server, const char* host) { - json_t* rel = json_object(); - json_t* rel_data = json_array(); - - json_object_set_new(rel, CN_LINKS, monitor_self_link(host)); + json_t* rel = mxs_json_relationship(host, MXS_JSON_API_MONITORS); spinlock_acquire(&monLock); @@ -1602,7 +1581,7 @@ json_t* monitor_relations_to_server(const SERVER* server, const char* host) { if (db->server == server) { - add_monitor_relation(rel_data, mon); + mxs_json_add_relation(rel, mon->name, CN_MONITORS); break; } } @@ -1612,7 +1591,5 @@ json_t* monitor_relations_to_server(const SERVER* server, const char* host) spinlock_release(&monLock); - json_object_set_new(rel, CN_DATA, rel_data); - return rel; } diff --git a/server/core/server.cc b/server/core/server.cc index 7b7df38ec..08eaa7d37 100644 --- a/server/core/server.cc +++ b/server/core/server.cc @@ -36,6 +36,7 @@ #include #include #include +#include #include "maxscale/monitor.h" #include "maxscale/poll.h" @@ -1476,15 +1477,7 @@ static json_t* server_to_json_data(const SERVER* server, const char* host) json_t* server_to_json(const SERVER* server, const char* host) { - json_t* rval = json_object(); - - /** Top level self link */ - json_object_set_new(rval, CN_LINKS, server_self_link(host)); - - /** Add server data */ - json_object_set_new(rval, CN_DATA, server_to_json_data(server, host)); - - return rval; + return mxs_json_resource(host, MXS_JSON_API_SERVERS, server_to_json_data(server, host)); } json_t* server_list_to_json(const char* host) @@ -1503,13 +1496,5 @@ json_t* server_list_to_json(const char* host) spinlock_release(&server_spin); - json_t* rval = json_object(); - - /** Top level self link */ - json_object_set_new(rval, CN_LINKS, server_self_link(host)); - - /** Add server data array */ - json_object_set_new(rval, CN_DATA, data); - - return rval; + return mxs_json_resource(host, MXS_JSON_API_SERVERS, data); } diff --git a/server/core/service.cc b/server/core/service.cc index b38c2f7d7..6e139df1b 100644 --- a/server/core/service.cc +++ b/server/core/service.cc @@ -48,6 +48,7 @@ #include #include #include +#include #include "maxscale/config.h" #include "maxscale/filter.h" @@ -2563,40 +2564,18 @@ json_t* service_list_to_json(const char* host) if (svc) { - json_array_append_new(rval, svc); + json_array_append_new(arr, svc); } } spinlock_release(&service_spin); - return rval; -} - -static void add_service_relation(json_t* arr, const SERVICE* service) -{ - json_t* obj = json_object(); - json_object_set_new(obj, CN_ID, json_string(service->name)); - json_object_set_new(obj, CN_TYPE, json_string(CN_SERVICES)); - json_array_append_new(arr, obj); -} - -static json_t* service_self_link(const char* host) -{ - json_t* rel_links = json_object(); - - string links = host; - links += "/services/"; - json_object_set_new(rel_links, CN_SELF, json_string(links.c_str())); - - return rel_links; + return mxs_json_resource(host, MXS_JSON_API_SERVICES, arr); } json_t* service_relations_to_filter(const MXS_FILTER_DEF* filter, const char* host) { - json_t* rel = json_object(); - json_t* rel_data = json_array(); - - json_object_set_new(rel, CN_LINKS, service_self_link(host)); + json_t* rel = mxs_json_relationship(host, MXS_JSON_API_SERVICES); spinlock_acquire(&service_spin); @@ -2608,7 +2587,7 @@ json_t* service_relations_to_filter(const MXS_FILTER_DEF* filter, const char* ho { if (service->filters[i] == filter) { - add_service_relation(rel_data, service); + mxs_json_add_relation(rel, service->name, CN_SERVICES); } } @@ -2617,18 +2596,13 @@ json_t* service_relations_to_filter(const MXS_FILTER_DEF* filter, const char* ho spinlock_release(&service_spin); - json_object_set_new(rel, CN_DATA, rel_data); - return rel; } json_t* service_relations_to_server(const SERVER* server, const char* host) { - json_t* rel = json_object(); - json_t* rel_data = json_array(); - - json_object_set_new(rel, CN_LINKS, service_self_link(host)); + json_t* rel = mxs_json_relationship(host, MXS_JSON_API_SERVICES); spinlock_acquire(&service_spin); @@ -2640,7 +2614,7 @@ json_t* service_relations_to_server(const SERVER* server, const char* host) { if (ref->server == server && SERVER_REF_IS_ACTIVE(ref)) { - add_service_relation(rel_data, service); + mxs_json_add_relation(rel, service->name, CN_SERVICES); } } @@ -2649,7 +2623,5 @@ json_t* service_relations_to_server(const SERVER* server, const char* host) spinlock_release(&service_spin); - json_object_set_new(rel, CN_DATA, rel_data); - return rel; } diff --git a/server/core/session.cc b/server/core/session.cc index dc072c329..6d81247f6 100644 --- a/server/core/session.cc +++ b/server/core/session.cc @@ -48,6 +48,7 @@ #include #include #include +#include #include "maxscale/session.h" #include "maxscale/filter.h" @@ -1081,66 +1082,35 @@ json_t* session_json_data(const MXS_SESSION *session, const char *host) { json_t* data = json_object(); + /** ID must be a string */ stringstream ss; ss << session->ses_id; /** ID and type */ - json_object_set_new(data, "id", json_string(ss.str().c_str())); - json_object_set_new(data, "type", json_string("sessions")); + json_object_set_new(data, CN_ID, json_string(ss.str().c_str())); + json_object_set_new(data, CN_TYPE, json_string(CN_SESSIONS)); /** Relationships */ json_t* rel = json_object(); /** Service relationship (one-to-one) */ - string svc = host; - svc += "/services/"; - - json_t* services = json_object(); - - /** Service self link */ - json_t* services_links = json_object(); - json_object_set_new(services_links, "self", json_string(svc.c_str())); - - /** Only one value for data, the service where this session belongs to */ - json_t* services_data_value = json_object(); - json_object_set_new(services_data_value, "id", json_string(session->service->name)); - json_object_set_new(services_data_value, "type", json_string("services")); - - json_object_set_new(services, "links", services_links); - json_object_set_new(services, "data", services_data_value); - json_object_set_new(rel, "services", services); + json_t* services = mxs_json_relationship(host, MXS_JSON_API_SERVICES); + mxs_json_add_relation(services, session->service->name, CN_SERVICES); + json_object_set_new(rel, CN_SERVICES, services); /** Filter relationships (one-to-many) */ if (session->n_filters) { - json_t* filters = json_object(); - - /** Filter self link */ - string fil = host; - fil += "/filters/"; - json_t* filters_links = json_object(); - json_object_set_new(filters_links, "self", json_string(fil.c_str())); - - /** Array of data values */ - json_t* filters_data_array = json_array(); + json_t* filters = mxs_json_relationship(host, MXS_JSON_API_FILTERS); for (int i = 0; i < session->n_filters; i++) { - /** Each value is a reference to a filter */ - json_t* filters_data_value = json_object(); - - json_object_set_new(filters_data_value, "id", json_string(session->filters[i].filter->name)); - json_object_set_new(filters_data_value, "type", json_string("filters")); - - json_array_append_new(filters_data_array, filters_data_value); + mxs_json_add_relation(filters, session->filters[i].filter->name, CN_FILTERS); } - - json_object_set_new(filters, "data", filters_data_array); - json_object_set_new(filters, "links", filters_links); - json_object_set_new(rel, "filters", filters); + json_object_set_new(rel, CN_FILTERS, filters); } - json_object_set_new(data, "relationships", rel); + json_object_set_new(data, CN_RELATIONSHIPS, rel); /** Session attributes */ json_t* attr = json_object(); @@ -1178,19 +1148,7 @@ json_t* session_json_data(const MXS_SESSION *session, const char *host) json_t* session_to_json(const MXS_SESSION *session, const char *host) { - json_t* rval = json_object(); - - /** Create the top level self link */ - stringstream self; - self << host << "/sessions/" << session->ses_id; - json_t* links_self = json_object(); - json_object_set_new(links_self, "self", json_string(self.str().c_str())); - json_object_set_new(rval, "links", links_self); - - /** Only one value */ - json_object_set_new(rval, "data", session_json_data(session, host)); - - return rval; + return mxs_json_resource(host, MXS_JSON_API_SESSIONS, session_json_data(session, host)); } struct SessionListData @@ -1210,17 +1168,5 @@ json_t* session_list_to_json(const char* host) { SessionListData data = {json_array(), host}; dcb_foreach(seslist_cb, &data); - json_t* rval = json_object(); - - /** Create the top level self link */ - stringstream self; - self << host << "/sessions/"; - json_t* links_self = json_object(); - json_object_set_new(links_self, "self", json_string(self.str().c_str())); - json_object_set_new(rval, "links", links_self); - - /** Array of session values */ - json_object_set_new(rval, "data", data.json); - - return rval; + return mxs_json_resource(host, MXS_JSON_API_SESSIONS, data.json); } From fb7f283316cfee172a21cddbc486ed4e5bad23e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Wed, 3 May 2017 13:55:22 +0300 Subject: [PATCH 12/55] MXS-1220: Reorganize service resource The service resource now conforms to the JSON API schema. Added a test case for individual resources and resource collections. --- server/core/service.cc | 99 ++++++++++--------- .../test/rest-api/test/schema_validation.js | 3 + 2 files changed, 57 insertions(+), 45 deletions(-) diff --git a/server/core/service.cc b/server/core/service.cc index 6e139df1b..868ec3807 100644 --- a/server/core/service.cc +++ b/server/core/service.cc @@ -2446,16 +2446,25 @@ json_t* service_parameters_to_json(const SERVICE* service) return rval; } -json_t* service_to_json(const SERVICE* service, const char* host) +static inline bool have_active_servers(const SERVICE* service) { - spinlock_acquire(&service->spin); + for (SERVER_REF* ref = service->dbref; ref; ref = ref->next) + { + if (SERVER_REF_IS_ACTIVE(ref)) + { + return true; + } + } - json_t* rval = json_object(); + return false; +} - /** General service information */ - json_object_set_new(rval, CN_NAME, json_string(service->name)); - json_object_set_new(rval, CN_ROUTER, json_string(service->routerModule)); - json_object_set_new(rval, CN_STATE, json_string(service_state_to_string(service->state))); +json_t* service_attributes(const SERVICE* service) +{ + json_t* attr = json_object(); + + json_object_set_new(attr, CN_ROUTER, json_string(service->routerModule)); + json_object_set_new(attr, CN_STATE, json_string(service_state_to_string(service->state))); if (service->router && service->router_instance) { @@ -2463,7 +2472,7 @@ json_t* service_to_json(const SERVICE* service, const char* host) if (diag) { - json_object_set_new(rval, "router_diagnostics", diag); + json_object_set_new(attr, "router_diagnostics", diag); } } @@ -2473,12 +2482,12 @@ json_t* service_to_json(const SERVICE* service, const char* host) asctime_r(localtime_r(&service->stats.started, &result), timebuf); trim(timebuf); - json_object_set_new(rval, "started", json_string(timebuf)); - json_object_set_new(rval, "total_connections", json_integer(service->stats.n_sessions)); - json_object_set_new(rval, "connections", json_integer(service->stats.n_current)); + json_object_set_new(attr, "started", json_string(timebuf)); + json_object_set_new(attr, "total_connections", json_integer(service->stats.n_sessions)); + json_object_set_new(attr, "connections", json_integer(service->stats.n_current)); /** Add service parameters */ - json_object_set_new(rval, CN_PARAMETERS, service_parameters_to_json(service)); + json_object_set_new(attr, CN_PARAMETERS, service_parameters_to_json(service)); /** Add listeners */ json_t* arr = json_array(); @@ -2491,76 +2500,76 @@ json_t* service_to_json(const SERVICE* service, const char* host) } } - json_object_set_new(rval, CN_LISTENERS, arr); + json_object_set_new(attr, CN_LISTENERS, arr); + return attr; +} + +json_t* service_relationships(const SERVICE* service, const char* host) +{ /** Store relationships to other objects */ json_t* rel = json_object(); - string self = host; - self += "/services/"; - self += service->name; - json_object_set_new(rel, CN_SELF, json_string(self.c_str())); - if (service->n_filters) { - json_t* arr = json_array(); + json_t* filters = mxs_json_relationship(host, MXS_JSON_API_FILTERS); for (int i = 0; i < service->n_filters; i++) { - string filter = host; - filter += "/filters/"; - filter += service->filters[i]->name; - json_array_append_new(arr, json_string(filter.c_str())); + mxs_json_add_relation(filters, service->filters[i]->name, CN_FILTERS); } - json_object_set_new(rel, "filters", arr); + json_object_set_new(rel, CN_FILTERS, filters); } - bool active_servers = false; - - for (SERVER_REF* ref = service->dbref; ref; ref = ref->next) + if (have_active_servers(service)) { - if (SERVER_REF_IS_ACTIVE(ref)) - { - active_servers = true; - break; - } - } - - if (active_servers) - { - json_t* arr = json_array(); + json_t* servers = mxs_json_relationship(host, MXS_JSON_API_SERVERS); for (SERVER_REF* ref = service->dbref; ref; ref = ref->next) { if (SERVER_REF_IS_ACTIVE(ref)) { - string s = host; - s += "/servers/"; - s += ref->server->unique_name; - json_array_append_new(arr, json_string(s.c_str())); + mxs_json_add_relation(servers, ref->server->unique_name, CN_SERVERS); } } - json_object_set_new(rel, CN_SERVERS, arr); + json_object_set_new(rel, CN_SERVERS, servers); } - json_object_set_new(rval, CN_RELATIONSHIPS, rel); + return rel; +} + +json_t* service_json_data(const SERVICE* service, const char* host) +{ + json_t* rval = json_object(); + + spinlock_acquire(&service->spin); + + json_object_set_new(rval, CN_ID, json_string(service->name)); + json_object_set_new(rval, CN_TYPE, json_string(CN_SERVICES)); + json_object_set_new(rval, CN_ATTRIBUTES, service_attributes(service)); + json_object_set_new(rval, CN_RELATIONSHIPS, service_relationships(service, host)); spinlock_release(&service->spin); return rval; } +json_t* service_to_json(const SERVICE* service, const char* host) +{ + return mxs_json_resource(host, MXS_JSON_API_SERVICES, service_json_data(service, host)); +} + json_t* service_list_to_json(const char* host) { - json_t* rval = json_array(); + json_t* arr = json_array(); spinlock_acquire(&service_spin); for (SERVICE *service = allServices; service; service = service->next) { - json_t* svc = service_to_json(service, host); + json_t* svc = service_json_data(service, host); if (svc) { diff --git a/server/core/test/rest-api/test/schema_validation.js b/server/core/test/rest-api/test/schema_validation.js index f7ff12a31..6ffcfe51d 100644 --- a/server/core/test/rest-api/test/schema_validation.js +++ b/server/core/test/rest-api/test/schema_validation.js @@ -1,3 +1,5 @@ +// These tests use the server/test/maxscale_test.cnf configuration + require("../utils.js")() describe("Resource Collections", function(){ @@ -28,6 +30,7 @@ describe("Individual Resources", function(){ var tests = [ "/servers/server1", "/servers/server2", + "/services/RW-Split-Router", "/sessions/1", ] From b9a5d91fe6650518389ca7933c8723f507403e10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 4 May 2017 09:41:06 +0300 Subject: [PATCH 13/55] MXS-1220: Reorganize monitor resource The monitor resource now conforms to the JSON API schema. Added a test case for the individual monitor resource. --- server/core/monitor.cc | 47 +++++++++++-------- .../test/rest-api/test/schema_validation.js | 1 + 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/server/core/monitor.cc b/server/core/monitor.cc index 13ea53081..d67387a84 100644 --- a/server/core/monitor.cc +++ b/server/core/monitor.cc @@ -31,6 +31,7 @@ #include #include #include +#include #include "maxscale/config.h" #include "maxscale/externcmd.h" @@ -1497,16 +1498,22 @@ json_t* monitor_parameters_to_json(const MXS_MONITOR* monitor) return rval; } -json_t* monitor_to_json(const MXS_MONITOR* monitor, const char* host) +json_t* monitor_json_data(const MXS_MONITOR* monitor, const char* host) { json_t* rval = json_object(); - json_object_set_new(rval, CN_NAME, json_string(monitor->name)); - json_object_set_new(rval, CN_MODULE, json_string(monitor->module_name)); - json_object_set_new(rval, CN_STATE, json_string(monitor_state_to_string(monitor->state))); + spinlock_acquire(&monitor->lock); + + json_object_set_new(rval, CN_ID, json_string(monitor->name)); + json_object_set_new(rval, CN_TYPE, json_string(CN_MONITORS)); + + json_t* attr = json_object(); + + json_object_set_new(attr, CN_MODULE, json_string(monitor->module_name)); + json_object_set_new(attr, CN_STATE, json_string(monitor_state_to_string(monitor->state))); /** Monitor parameters */ - json_object_set_new(rval, CN_PARAMETERS, monitor_parameters_to_json(monitor)); + json_object_set_new(attr, CN_PARAMETERS, monitor_parameters_to_json(monitor)); if (monitor->handle && monitor->module->diagnostics) { @@ -1514,38 +1521,38 @@ json_t* monitor_to_json(const MXS_MONITOR* monitor, const char* host) if (diag) { - json_object_set_new(rval, "monitor_diagnostics", diag); + json_object_set_new(attr, "monitor_diagnostics", diag); } } json_t* rel = json_object(); - /** Store relationships to other objects */ - string self = host; - self += "/monitors/"; - self += monitor->name; - json_object_set_new(rel, CN_SELF, json_string(self.c_str())); if (monitor->databases) { - json_t* arr = json_array(); + json_t* mon_rel = mxs_json_relationship(host, MXS_JSON_API_SERVERS); for (MXS_MONITOR_SERVERS *db = monitor->databases; db; db = db->next) { - string s = host; - s += "/servers/"; - s += db->server->unique_name; - json_array_append_new(arr, json_string(s.c_str())); + mxs_json_add_relation(mon_rel, db->server->unique_name, CN_SERVERS); } - json_object_set_new(rel, "servers", arr); + json_object_set_new(rel, CN_SERVERS, mon_rel); } - json_object_set_new(rval, "relationships", rel); + spinlock_release(&monitor->lock); + + json_object_set_new(rval, CN_RELATIONSHIPS, rel); + json_object_set_new(rval, CN_ATTRIBUTES, attr); return rval; } +json_t* monitor_to_json(const MXS_MONITOR* monitor, const char* host) +{ + return mxs_json_resource(host, MXS_JSON_API_MONITORS, monitor_json_data(monitor, host)); +} + json_t* monitor_list_to_json(const char* host) { json_t* rval = json_array(); @@ -1554,7 +1561,7 @@ json_t* monitor_list_to_json(const char* host) for (MXS_MONITOR* mon = allMonitors; mon; mon = mon->next) { - json_t *json = monitor_to_json(mon, host); + json_t *json = monitor_json_data(mon, host); if (json) { @@ -1564,7 +1571,7 @@ json_t* monitor_list_to_json(const char* host) spinlock_release(&monLock); - return rval; + return mxs_json_resource(host, MXS_JSON_API_MONITORS, rval); } json_t* monitor_relations_to_server(const SERVER* server, const char* host) diff --git a/server/core/test/rest-api/test/schema_validation.js b/server/core/test/rest-api/test/schema_validation.js index 6ffcfe51d..e91f35619 100644 --- a/server/core/test/rest-api/test/schema_validation.js +++ b/server/core/test/rest-api/test/schema_validation.js @@ -31,6 +31,7 @@ describe("Individual Resources", function(){ "/servers/server1", "/servers/server2", "/services/RW-Split-Router", + "/monitors/MySQL-Monitor", "/sessions/1", ] From 432a6d6f28228766f087e0635cfc0b3cd6b32cff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 4 May 2017 09:48:43 +0300 Subject: [PATCH 14/55] MXS-1220: Reorganize filter resource The filter resource now conforms to the JSON API schema. Added a test case for the individual filter resource. --- server/core/filter.cc | 46 +++++++++---------- .../test/rest-api/test/schema_validation.js | 1 + 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/server/core/filter.cc b/server/core/filter.cc index 05180a64c..8ef97ff51 100644 --- a/server/core/filter.cc +++ b/server/core/filter.cc @@ -30,6 +30,7 @@ #include #include #include +#include #include "maxscale/config.h" #include "maxscale/modules.h" @@ -497,48 +498,43 @@ json_t* filter_parameters_to_json(const MXS_FILTER_DEF* filter) return rval; } -json_t* filter_to_json(const MXS_FILTER_DEF* filter, const char* host) +json_t* filter_json_data(const MXS_FILTER_DEF* filter, const char* host) { json_t* rval = json_object(); - json_object_set_new(rval, CN_NAME, json_string(filter->name)); - json_object_set_new(rval, CN_MODULE, json_string(filter->module)); - json_object_set_new(rval, CN_PARAMETERS, filter_parameters_to_json(filter)); + json_object_set_new(rval, CN_ID, json_string(filter->name)); + json_object_set_new(rval, CN_TYPE, json_string(CN_FILTERS)); - if (filter->obj && filter->filter) + json_t* attr = json_object(); + + json_object_set_new(attr, CN_MODULE, json_string(filter->module)); + json_object_set_new(attr, CN_PARAMETERS, filter_parameters_to_json(filter)); + + if (filter->obj && filter->filter && filter->obj->diagnostics_json) { json_t* diag = filter->obj->diagnostics_json(filter->filter, NULL); if (diag) { - json_object_set_new(rval, "filter_diagnostics", diag); + json_object_set_new(attr, "filter_diagnostics", diag); } } /** Store relationships to other objects */ json_t* rel = json_object(); + json_object_set_new(rel, CN_SERVICES, service_relations_to_filter(filter, host)); - string self = host; - self += "/filters/"; - self += filter->name; - json_object_set_new(rel, CN_SELF, json_string(self.c_str())); - - json_t* arr = service_relations_to_filter(filter, host); - - if (json_array_size(arr) > 0) - { - json_object_set_new(rel, "services", arr); - } - else - { - json_decref(arr); - } - - json_object_set_new(rval, "relationships", rel); + json_object_set_new(rval, CN_RELATIONSHIPS, rel); + json_object_set_new(rval, CN_ATTRIBUTES, attr); return rval; } +json_t* filter_to_json(const MXS_FILTER_DEF* filter, const char* host) +{ + return mxs_json_resource(host, MXS_JSON_API_FILTERS, filter_json_data(filter, host)); +} + json_t* filter_list_to_json(const char* host) { json_t* rval = json_array(); @@ -547,7 +543,7 @@ json_t* filter_list_to_json(const char* host) for (MXS_FILTER_DEF* f = allFilters; f; f = f->next) { - json_t* json = filter_to_json(f, host); + json_t* json = filter_json_data(f, host); if (json) { @@ -557,7 +553,7 @@ json_t* filter_list_to_json(const char* host) spinlock_release(&filter_spin); - return rval; + return mxs_json_resource(host, MXS_JSON_API_FILTERS, rval); } namespace maxscale diff --git a/server/core/test/rest-api/test/schema_validation.js b/server/core/test/rest-api/test/schema_validation.js index e91f35619..17efcb2ef 100644 --- a/server/core/test/rest-api/test/schema_validation.js +++ b/server/core/test/rest-api/test/schema_validation.js @@ -32,6 +32,7 @@ describe("Individual Resources", function(){ "/servers/server2", "/services/RW-Split-Router", "/monitors/MySQL-Monitor", + "/filters/Hint", "/sessions/1", ] From 2f8db4ec1a9efe15986ff5e5a15291868c11bd28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 4 May 2017 11:00:25 +0300 Subject: [PATCH 15/55] MXS-1220: Add JSON Pointer support for json_t objects Using the JSON Pointer syntax specified in RFC 6901 (https://tools.ietf.org/html/rfc6901) allows for a convenient way to access values deep in a JSON object. --- include/maxscale/json_api.h | 9 + server/core/json_api.cc | 70 ++++++++ server/core/test/CMakeLists.txt | 3 + server/core/test/testjson.cc | 297 ++++++++++++++++++++++++++++++++ 4 files changed, 379 insertions(+) create mode 100644 server/core/test/testjson.cc diff --git a/include/maxscale/json_api.h b/include/maxscale/json_api.h index dfe5f1418..d3d15ad31 100644 --- a/include/maxscale/json_api.h +++ b/include/maxscale/json_api.h @@ -61,4 +61,13 @@ json_t* mxs_json_relationship(const char* host, const char* endpoint); */ void mxs_json_add_relation(json_t* rel, const char* id, const char* type); +/** + * @brief Return value at provided JSON Pointer + * + * @param json JSON object + * @param json_ptr JSON Pointer to object + * @return Pointed value or NULL if no value is found + */ +json_t* mxs_json_pointer(json_t* json, const char* json_ptr); + MXS_END_DECLS diff --git a/server/core/json_api.cc b/server/core/json_api.cc index 2c820bfc7..5e856af16 100644 --- a/server/core/json_api.cc +++ b/server/core/json_api.cc @@ -61,3 +61,73 @@ void mxs_json_add_relation(json_t* rel, const char* id, const char* type) json_object_set_new(obj, CN_TYPE, json_string(type)); json_array_append(data, obj); } + +static string grab_next_component(string* s) +{ + std::string& str = *s; + + while (str.length() > 0 && str[0] == '/') + { + str.erase(str.begin()); + } + + size_t pos = str.find("/"); + string rval; + + if (pos != string::npos) + { + rval = str.substr(0, pos); + str.erase(0, pos); + return rval; + } + else + { + rval = str; + str.erase(0); + } + + return rval; +} + +static bool is_integer(const string& str) +{ + char* end; + return strtol(str.c_str(), &end, 10) >= 0 && *end == '\0'; +} + +static json_t* mxs_json_pointer_internal(json_t* json, string str) +{ + json_t* rval = NULL; + string comp = grab_next_component(&str); + + if (comp.length() == 0) + { + return json; + } + + if (json_is_array(json) && is_integer(comp)) + { + size_t idx = strtol(comp.c_str(), NULL, 10); + + if (idx < json_array_size(json)) + { + rval = mxs_json_pointer_internal(json_array_get(json, idx), str); + } + } + else if (json_is_object(json)) + { + json_t* obj = json_object_get(json, comp.c_str()); + + if (obj) + { + rval = mxs_json_pointer_internal(obj, str); + } + } + + return rval; +} + +json_t* mxs_json_pointer(json_t* json, const char* json_ptr) +{ + return mxs_json_pointer_internal(json, json_ptr); +} diff --git a/server/core/test/CMakeLists.txt b/server/core/test/CMakeLists.txt index 89ca4cc0a..33c47c1f6 100644 --- a/server/core/test/CMakeLists.txt +++ b/server/core/test/CMakeLists.txt @@ -23,6 +23,7 @@ add_executable(testmaxscalepcre2 testmaxscalepcre2.cc) add_executable(testmodulecmd testmodulecmd.cc) add_executable(testconfig testconfig.cc) add_executable(trxboundaryparser_profile trxboundaryparser_profile.cc) +add_executable(testjson testjson.cc) target_link_libraries(test_atomic maxscale-common) target_link_libraries(test_adminusers maxscale-common) target_link_libraries(test_buffer maxscale-common) @@ -48,6 +49,7 @@ target_link_libraries(testmaxscalepcre2 maxscale-common) target_link_libraries(testmodulecmd maxscale-common) target_link_libraries(testconfig maxscale-common) target_link_libraries(trxboundaryparser_profile maxscale-common) +target_link_libraries(testjson maxscale-common) add_test(TestAtomic test_atomic) add_test(TestAdminUsers test_adminusers) add_test(TestBuffer test_buffer) @@ -79,6 +81,7 @@ add_test(TestTrxCompare_Select test_trxcompare ${CMAKE_CURRENT_SOURCE_DIR}/../.. add_test(TestTrxCompare_Set test_trxcompare ${CMAKE_CURRENT_SOURCE_DIR}/../../../query_classifier/test/set.test) add_test(TestTrxCompare_Update test_trxcompare ${CMAKE_CURRENT_SOURCE_DIR}/../../../query_classifier/test/update.test) add_test(TestTrxCompare_MaxScale test_trxcompare ${CMAKE_CURRENT_SOURCE_DIR}/../../../query_classifier/test/maxscale.test) +add_test(TestJson testjson) # This test requires external dependencies and thus cannot be run # as a part of the core test set diff --git a/server/core/test/testjson.cc b/server/core/test/testjson.cc new file mode 100644 index 000000000..48df4ba57 --- /dev/null +++ b/server/core/test/testjson.cc @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl11. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include + +#include + +#include +#include +#include + +using std::string; + +const char* test1_json = + "{" + " \"links\": {" + " \"self\": \"http://localhost:8989/v1/servers/\"" + " }," + " \"data\": [" + " {" + " \"id\": \"server1\"," + " \"type\": \"servers\"," + " \"relationships\": {" + " \"services\": {" + " \"links\": {" + " \"self\": \"http://localhost:8989/v1/services/\"" + " }," + " \"data\": [" + " {" + " \"id\": \"RW-Split-Router\"," + " \"type\": \"services\"" + " }," + " {" + " \"id\": \"SchemaRouter-Router\"," + " \"type\": \"services\"" + " }," + " {" + " \"id\": \"RW-Split-Hint-Router\"," + " \"type\": \"services\"" + " }," + " {" + " \"id\": \"Read-Connection-Router\"," + " \"type\": \"services\"" + " }" + " ]" + " }," + " \"monitors\": {" + " \"links\": {" + " \"self\": \"http://localhost:8989/v1/monitors/\"" + " }," + " \"data\": [" + " {" + " \"id\": \"MySQL-Monitor\"," + " \"type\": \"monitors\"" + " }" + " ]" + " }" + " }," + " \"attributes\": {" + " \"parameters\": {" + " \"address\": \"127.0.0.1\"," + " \"port\": 3000," + " \"protocol\": \"MySQLBackend\"" + " }," + " \"status\": \"Master, Running\"," + " \"version_string\": \"10.1.19-MariaDB-1~jessie\"," + " \"node_id\": 3000," + " \"master_id\": -1," + " \"replication_depth\": 0," + " \"slaves\": [" + " 3001," + " 3002," + " 3003" + " ]," + " \"statictics\": {" + " \"connections\": 0," + " \"total_connections\": 0," + " \"active_operations\": 0" + " }" + " }" + " }," + " {" + " \"id\": \"server2\"," + " \"type\": \"servers\"," + " \"relationships\": {" + " \"services\": {" + " \"links\": {" + " \"self\": \"http://localhost:8989/v1/services/\"" + " }," + " \"data\": [" + " {" + " \"id\": \"RW-Split-Router\"," + " \"type\": \"services\"" + " }," + " {" + " \"id\": \"SchemaRouter-Router\"," + " \"type\": \"services\"" + " }," + " {" + " \"id\": \"RW-Split-Hint-Router\"," + " \"type\": \"services\"" + " }" + " ]" + " }," + " \"monitors\": {" + " \"links\": {" + " \"self\": \"http://localhost:8989/v1/monitors/\"" + " }," + " \"data\": [" + " {" + " \"id\": \"MySQL-Monitor\"," + " \"type\": \"monitors\"" + " }" + " ]" + " }" + " }," + " \"attributes\": {" + " \"parameters\": {" + " \"address\": \"127.0.0.1\"," + " \"port\": 3001," + " \"protocol\": \"MySQLBackend\"" + " }," + " \"status\": \"Slave, Running\"," + " \"version_string\": \"10.1.19-MariaDB-1~jessie\"," + " \"node_id\": 3001," + " \"master_id\": 3000," + " \"replication_depth\": 1," + " \"slaves\": []," + " \"statictics\": {" + " \"connections\": 0," + " \"total_connections\": 0," + " \"active_operations\": 0" + " }" + " }" + " }," + " {" + " \"id\": \"server3\"," + " \"type\": \"servers\"," + " \"relationships\": {" + " \"services\": {" + " \"links\": {" + " \"self\": \"http://localhost:8989/v1/services/\"" + " }," + " \"data\": [" + " {" + " \"id\": \"RW-Split-Router\"," + " \"type\": \"services\"" + " }," + " {" + " \"id\": \"SchemaRouter-Router\"," + " \"type\": \"services\"" + " }," + " {" + " \"id\": \"RW-Split-Hint-Router\"," + " \"type\": \"services\"" + " }" + " ]" + " }," + " \"monitors\": {" + " \"links\": {" + " \"self\": \"http://localhost:8989/v1/monitors/\"" + " }," + " \"data\": [" + " {" + " \"id\": \"MySQL-Monitor\"," + " \"type\": \"monitors\"" + " }" + " ]" + " }" + " }," + " \"attributes\": {" + " \"parameters\": {" + " \"address\": \"127.0.0.1\"," + " \"port\": 3002," + " \"protocol\": \"MySQLBackend\"" + " }," + " \"status\": \"Slave, Running\"," + " \"version_string\": \"10.1.19-MariaDB-1~jessie\"," + " \"node_id\": 3002," + " \"master_id\": 3000," + " \"replication_depth\": 1," + " \"slaves\": []," + " \"statictics\": {" + " \"connections\": 0," + " \"total_connections\": 0," + " \"active_operations\": 0" + " }" + " }" + " }," + " {" + " \"id\": \"server4\"," + " \"type\": \"servers\"," + " \"relationships\": {" + " \"services\": {" + " \"links\": {" + " \"self\": \"http://localhost:8989/v1/services/\"" + " }," + " \"data\": [" + " {" + " \"id\": \"RW-Split-Router\"," + " \"type\": \"services\"" + " }," + " {" + " \"id\": \"SchemaRouter-Router\"," + " \"type\": \"services\"" + " }," + " {" + " \"id\": \"RW-Split-Hint-Router\"," + " \"type\": \"services\"" + " }" + " ]" + " }," + " \"monitors\": {" + " \"links\": {" + " \"self\": \"http://localhost:8989/v1/monitors/\"" + " }," + " \"data\": [" + " {" + " \"id\": \"MySQL-Monitor\"," + " \"type\": \"monitors\"" + " }" + " ]" + " }" + " }," + " \"attributes\": {" + " \"parameters\": {" + " \"address\": \"127.0.0.1\"," + " \"port\": 3003," + " \"protocol\": \"MySQLBackend\"" + " }," + " \"status\": \"Slave, Running\"," + " \"version_string\": \"10.1.19-MariaDB-1~jessie\"," + " \"node_id\": 3003," + " \"master_id\": 3000," + " \"replication_depth\": 1," + " \"slaves\": []," + " \"statictics\": {" + " \"connections\": 0," + " \"total_connections\": 0," + " \"active_operations\": 0" + " }" + " }" + " }" + " ]" + "}"; + +int test1() +{ + json_error_t err = {}; + json_t* json = json_loads(test1_json, 0, &err); + + ss_dassert(json); + + ss_dassert(mxs_json_pointer(json, "") == json); + ss_dassert(mxs_json_pointer(json, "links") == json_object_get(json, "links")); + ss_dassert(json_is_string(mxs_json_pointer(json, "links/self"))); + + ss_dassert(mxs_json_pointer(json, "data") == json_object_get(json, "data")); + ss_dassert(json_is_array(mxs_json_pointer(json, "data"))); + + ss_dassert(json_is_object(mxs_json_pointer(json, "data/0"))); + ss_dassert(json_is_string(mxs_json_pointer(json, "data/0/id"))); + string s = json_string_value(mxs_json_pointer(json, "data/0/id")); + ss_dassert(s == "server1"); + + ss_dassert(json_is_object(mxs_json_pointer(json, "data/1"))); + ss_dassert(json_is_string(mxs_json_pointer(json, "data/1/id"))); + s = json_string_value(mxs_json_pointer(json, "data/1/id")); + ss_dassert(s == "server2"); + + ss_dassert(json_is_object(mxs_json_pointer(json, "data/0/attributes"))); + ss_dassert(json_is_object(mxs_json_pointer(json, "data/0/attributes/parameters"))); + ss_dassert(json_is_integer(mxs_json_pointer(json, "data/0/attributes/parameters/port"))); + int i = json_integer_value(mxs_json_pointer(json, "data/0/attributes/parameters/port")); + ss_dassert(i == 3000); + + ss_dassert(json_is_array(mxs_json_pointer(json, "data/0/attributes/slaves"))); + ss_dassert(json_array_size(mxs_json_pointer(json, "data/0/attributes/slaves")) == 3); + + return 0; +} + +int main(int argc, char** argv) +{ + test1(); + return 0; +} From 6c220a115104c30111cf03f0c01f0373177b9ff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 4 May 2017 14:25:31 +0300 Subject: [PATCH 16/55] MXS-1220: Migrate create/update server to JSON API format The creation and modification of servers now supports the JSON API conforming format generated by the GET endpoints. --- server/core/config_runtime.cc | 119 +++++++++++++++++----------------- 1 file changed, 58 insertions(+), 61 deletions(-) diff --git a/server/core/config_runtime.cc b/server/core/config_runtime.cc index e72adc204..e61b8ef71 100644 --- a/server/core/config_runtime.cc +++ b/server/core/config_runtime.cc @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -25,6 +26,7 @@ #include #include #include +#include #include "maxscale/config.h" #include "maxscale/monitor.h" @@ -32,9 +34,27 @@ #include "maxscale/service.h" using std::string; +using std::stringstream; using std::set; using mxs::Closer; +/** JSON Pointers to key parts of JSON objects */ +#define PTR_ID "/data/id" +#define PTR_PARAMETERS "/data/attributes/parameters" + +/** Pointers to relation lists */ +static const char PTR_RELATIONSHIPS_SERVERS[] = "/data/relationships/servers/data"; +static const char PTR_RELATIONSHIPS_SERVICES[] = "/data/relationships/services/data"; +static const char PTR_RELATIONSHIPS_MONITORS[] = "/data/relationships/monitors/data"; +static const char PTR_RELATIONSHIPS_FILTERS[] = "/data/relationships/filters/data"; + +/** Server JSON Pointers */ +static const char PTR_SRV_PORT[] = PTR_PARAMETERS "/port"; +static const char PTR_SRV_ADDRESS[] = PTR_PARAMETERS "/address"; +static const char PTR_SRV_PROTOCOL[] = PTR_PARAMETERS "/protocol"; +static const char PTR_SRV_AUTHENTICATOR[] = PTR_PARAMETERS "/authenticator"; +static const char PTR_SRV_AUTHENTICATOR_OPTIONS[] = PTR_PARAMETERS "/authenticator_options"; + static SPINLOCK crt_lock = SPINLOCK_INIT; bool runtime_link_server(SERVER *server, const char *target) @@ -316,12 +336,8 @@ bool runtime_alter_server(SERVER *server, const char *key, const char *value) } } - if (valid) + if (valid && server_serialize(server)) { - if (server->created_online) - { - server_serialize(server); - } MXS_NOTICE("Updated server '%s': %s=%s", server->unique_name, key, value); } @@ -756,47 +772,40 @@ static bool extract_relations(json_t* json, set& relations, bool (*relation_check)(const string&, const string&)) { bool rval = true; - json_t* rel; - if ((rel = json_object_get(json, CN_RELATIONSHIPS))) + for (int i = 0; relation_types[i]; i++) { - for (int i = 0; relation_types[i]; i++) + json_t* arr = mxs_json_pointer(json, relation_types[i]); + + if (arr && json_is_array(arr)) { - json_t* arr = json_object_get(rel, relation_types[i]); + size_t size = json_array_size(arr); - if (arr) + for (size_t j = 0; j < size; j++) { - size_t size = json_array_size(arr); + json_t* obj = json_array_get(arr, j); + json_t* id = json_object_get(obj, CN_ID); + json_t* type = mxs_json_pointer(obj, CN_TYPE); - for (size_t j = 0; j < size; j++) + if (id && json_is_string(id) && + type && json_is_string(type)) { - json_t* t = json_array_get(arr, j); + string id_value = json_string_value(id); + string type_value = json_string_value(type); - if (json_is_string(t)) + if (relation_check(type_value, id_value)) { - string value = json_string_value(t); - - // Remove the link part - size_t pos = value.find_last_of("/"); - if (pos != string::npos) - { - value.erase(0, pos + 1); - } - - if (relation_check(relation_types[i], value)) - { - relations.insert(value); - } - else - { - rval = false; - } + relations.insert(id_value); } else { rval = false; } } + else + { + rval = false; + } } } } @@ -804,10 +813,10 @@ static bool extract_relations(json_t* json, set& relations, return rval; } -static inline const char* string_or_null(json_t* json, const char* name) +static inline const char* string_or_null(json_t* json, const char* path) { const char* rval = NULL; - json_t* value = json_object_get(json, name); + json_t* value = mxs_json_pointer(json, path); if (value && json_is_string(value)) { @@ -819,29 +828,19 @@ static inline const char* string_or_null(json_t* json, const char* name) static bool server_contains_required_fields(json_t* json) { - bool rval = false; - json_t* value; + json_t* id = mxs_json_pointer(json, PTR_ID); + json_t* port = mxs_json_pointer(json, PTR_SRV_PORT); + json_t* address = mxs_json_pointer(json, PTR_SRV_ADDRESS); - if ((value = json_object_get(json, CN_NAME)) && json_is_string(value)) - { - /** Object has a name field */ - json_t* param = json_object_get(json, CN_PARAMETERS); - - if (param && - (value = json_object_get(param, CN_ADDRESS)) && json_is_string(value) && - (value = json_object_get(param, CN_PORT)) && json_is_integer(value)) - { - rval = true; - } - } - - return rval; + return (id && json_is_string(id) && + address && json_is_string(address) && + port && json_is_integer(port)); } const char* server_relation_types[] = { - CN_SERVICES, - CN_MONITORS, + PTR_RELATIONSHIPS_SERVICES, + PTR_RELATIONSHIPS_MONITORS, NULL }; @@ -889,20 +888,18 @@ SERVER* runtime_create_server_from_json(json_t* json) if (server_contains_required_fields(json)) { - const char* name = json_string_value(json_object_get(json, CN_NAME)); - json_t* params = json_object_get(json, CN_PARAMETERS); - const char* address = json_string_value(json_object_get(params, CN_ADDRESS)); + const char* name = json_string_value(mxs_json_pointer(json, PTR_ID)); + const char* address = json_string_value(mxs_json_pointer(json, PTR_SRV_ADDRESS)); /** The port needs to be in string format */ char port[200]; // Enough to store any port value - int i = json_integer_value(json_object_get(params, CN_PORT)); + int i = json_integer_value(mxs_json_pointer(json, PTR_SRV_PORT)); snprintf(port, sizeof(port), "%d", i); /** Optional parameters */ - const char* protocol = string_or_null(params, CN_PROTOCOL); - const char* authenticator = string_or_null(params, CN_AUTHENTICATOR); - const char* authenticator_options = string_or_null(params, CN_AUTHENTICATOR_OPTIONS); - + const char* protocol = string_or_null(json, PTR_SRV_PROTOCOL); + const char* authenticator = string_or_null(json, PTR_SRV_AUTHENTICATOR); + const char* authenticator_options = string_or_null(json, PTR_SRV_AUTHENTICATOR_OPTIONS); set relations; @@ -961,8 +958,8 @@ bool runtime_alter_server_from_json(SERVER* server, json_t* new_json) if (server_to_object_relations(server, old_json.get(), new_json)) { - json_t* parameters = json_object_get(new_json, CN_PARAMETERS); - json_t* old_parameters = json_object_get(old_json.get(), CN_PARAMETERS); + json_t* parameters = mxs_json_pointer(new_json, PTR_PARAMETERS); + json_t* old_parameters = mxs_json_pointer(old_json.get(), PTR_PARAMETERS); ss_dassert(old_parameters); From 6b0fabf83414ea4b4920ed8be32a12f20bb90b25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 4 May 2017 15:03:36 +0300 Subject: [PATCH 17/55] MXS-1220: Make REST API locally testable The `test_rest_api` make target creates a discardable installation of MaxScale which is used to launch a local instance of MaxScale. This local instance is then used to test the REST API. This is definitely not an efficient way to test the MaxScale but it allows local testing without virtual machines or containers. --- server/core/test/CMakeLists.txt | 2 + server/core/test/rest-api/CMakeLists.txt | 3 + server/core/test/rest-api/test_rest_api.sh | 73 ++++++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 server/core/test/rest-api/CMakeLists.txt create mode 100755 server/core/test/rest-api/test_rest_api.sh diff --git a/server/core/test/CMakeLists.txt b/server/core/test/CMakeLists.txt index 33c47c1f6..c2c674acc 100644 --- a/server/core/test/CMakeLists.txt +++ b/server/core/test/CMakeLists.txt @@ -89,3 +89,5 @@ if(TEST_FEEDBACK) add_test(TestFeedback testfeedback) set_tests_properties(TestFeedback PROPERTIES TIMEOUT 30) endif() + +add_subdirectory(rest-api) diff --git a/server/core/test/rest-api/CMakeLists.txt b/server/core/test/rest-api/CMakeLists.txt new file mode 100644 index 000000000..d7e871584 --- /dev/null +++ b/server/core/test/rest-api/CMakeLists.txt @@ -0,0 +1,3 @@ +add_custom_target(test_rest_api + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test_rest_api.sh ${CMAKE_SOURCE_DIR} + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/server/core/test/rest-api/test_rest_api.sh b/server/core/test/rest-api/test_rest_api.sh new file mode 100755 index 000000000..1f3306cbc --- /dev/null +++ b/server/core/test/rest-api/test_rest_api.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# This script builds and installs MaxScale, starts a MaxScale instance, runs the +# tests use npm and stops MaxScale. +# +# This is definitely not the most efficient way to test the binaries but it's a +# guaranteed method of creating a consistent and "safe" testing environment. +# +# TODO: Install and start a local MariaDB server for testing purposes + + +srcdir=$1 +maxscaledir=$PWD/maxscale_test/ +testdir=$PWD/local_test/ + +mkdir -p $testdir && cd $testdir + +# Currently all tests that use npm are for the REST API +cp -t $testdir -r $srcdir/server/core/test/rest-api/* +npm install + +mkdir -p $maxscaledir && cd $maxscaledir + +# Configure and install MaxScale +cmake $srcdir -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_INSTALL_PREFIX=$maxscaledir \ + -DBUILD_TESTS=Y \ + -DMAXSCALE_VARDIR=$maxscaledir \ + -DCMAKE_BUILD_TYPE=Debug \ + -DWITH_SCRIPTS=N \ + -DWITH_MAXSCALE_CNF=N \ + -DBUILD_CDC=Y \ + -DTARGET_COMPONENT=all \ + -DDEFAULT_MODULE_CONFIGDIR=$maxscaledir \ + -DDEFAULT_ADMIN_USER=`whoami` + +make install + +# Create required directories (we could run the postinst script but it's a bit too invasive) +mkdir -p $maxscaledir/lib64/maxscale +mkdir -p $maxscaledir/bin +mkdir -p $maxscaledir/share/maxscale +mkdir -p $maxscaledir/share/doc/MaxScale/maxscale +mkdir -p $maxscaledir/log/maxscale +mkdir -p $maxscaledir/lib/maxscale +mkdir -p $maxscaledir/cache/maxscale +mkdir -p $maxscaledir/run/maxscale +chmod 0755 $maxscaledir/log/maxscale +chmod 0755 $maxscaledir/lib/maxscale +chmod 0755 $maxscaledir/cache/maxscale +chmod 0755 $maxscaledir/run/maxscale + +# Start MaxScale +$maxscaledir/bin/maxscale -df $maxscaledir/maxscale.cnf >& $maxscaledir/maxscale.output & +pid=$! + +# Wait for MaxScale to start +for ((i=0;i<60;i++)) +do + $maxscaledir/bin/maxadmin help >& /dev/null && break + sleep 1 +done + +# Run tests +cd $testdir +npm test +rval=$? + +# Stop MaxScale +kill $pid +wait + +exit $rval From a3c683ab879845b2b18720f904a0c2cf332b797a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 4 May 2017 16:21:37 +0300 Subject: [PATCH 18/55] MXS-1220: Simplify test creation The tests now automatically start MaxScale before each test block and stop it and perform cleanup after the test. This is done by simply calling the `before.sh` and `after.sh` scripts before each test block. --- server/core/test/rest-api/after.sh | 38 ++++++++++++++++++++++ server/core/test/rest-api/before.sh | 21 ++++++++++++ server/core/test/rest-api/package.json | 2 +- server/core/test/rest-api/test_rest_api.sh | 16 ++------- server/core/test/rest-api/utils.js | 16 +++++++++ 5 files changed, 78 insertions(+), 15 deletions(-) create mode 100755 server/core/test/rest-api/after.sh create mode 100755 server/core/test/rest-api/before.sh diff --git a/server/core/test/rest-api/after.sh b/server/core/test/rest-api/after.sh new file mode 100755 index 000000000..2bf7ee21e --- /dev/null +++ b/server/core/test/rest-api/after.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# +# This script is run after each test block. It kills the MaxScale process +# and cleans up the directories that contain generated files. +# + +test -z "$MAXSCALE_DIR" && exit 1 + +maxscaledir=$MAXSCALE_DIR + +pid=`cat $maxscaledir/maxscale.pid` +echo $pid + +for ((i=0;i<60;i++)) +do + kill -0 $pid + + if [ $? -eq 0 ] + then + # Process is still up + kill $pid + + if [ $i -gt 3 ] + then + sleep 0.1 + fi + else + break + fi +done + +rm -r $maxscaledir/lib/maxscale +rm -r $maxscaledir/cache/maxscale +rm -r $maxscaledir/run/maxscale +mkdir -m 0755 -p $maxscaledir/lib/maxscale +mkdir -m 0755 -p $maxscaledir/cache/maxscale +mkdir -m 0755 -p $maxscaledir/run/maxscale diff --git a/server/core/test/rest-api/before.sh b/server/core/test/rest-api/before.sh new file mode 100755 index 000000000..1a05da0f9 --- /dev/null +++ b/server/core/test/rest-api/before.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# +# This script is run before each test block. It starts MaxScale and waits for it +# to become responsive. +# + +maxscaledir=$MAXSCALE_DIR + +test -z "$MAXSCALE_DIR" && exit 1 + +# Start MaxScale +$maxscaledir/bin/maxscale -df $maxscaledir/maxscale.cnf >& $maxscaledir/maxscale.output & +pid=$! + +# Wait for MaxScale to start +for ((i=0;i<60;i++)) +do + $maxscaledir/bin/maxadmin help >& /dev/null && break + sleep 0.1 +done diff --git a/server/core/test/rest-api/package.json b/server/core/test/rest-api/package.json index 6e24487e1..1e9348e6f 100644 --- a/server/core/test/rest-api/package.json +++ b/server/core/test/rest-api/package.json @@ -4,7 +4,7 @@ "repository": "https://github.com/mariadb-corporation/MaxScale", "description": "MaxScale REST API tests", "scripts": { - "test": "mocha" + "test": "mocha --timeout 60000 --slow 10000" }, "author": "", "license": "SEE LICENSE IN ../../../../LICENSE.txt", diff --git a/server/core/test/rest-api/test_rest_api.sh b/server/core/test/rest-api/test_rest_api.sh index 1f3306cbc..820937445 100755 --- a/server/core/test/rest-api/test_rest_api.sh +++ b/server/core/test/rest-api/test_rest_api.sh @@ -50,24 +50,12 @@ chmod 0755 $maxscaledir/lib/maxscale chmod 0755 $maxscaledir/cache/maxscale chmod 0755 $maxscaledir/run/maxscale -# Start MaxScale -$maxscaledir/bin/maxscale -df $maxscaledir/maxscale.cnf >& $maxscaledir/maxscale.output & -pid=$! - -# Wait for MaxScale to start -for ((i=0;i<60;i++)) -do - $maxscaledir/bin/maxadmin help >& /dev/null && break - sleep 1 -done +# This variable is used to start and stop MaxScale before each test +export MAXSCALE_DIR=$maxscaledir # Run tests cd $testdir npm test rval=$? -# Stop MaxScale -kill $pid -wait - exit $rval diff --git a/server/core/test/rest-api/utils.js b/server/core/test/rest-api/utils.js index dadb26586..3c4990ebb 100644 --- a/server/core/test/rest-api/utils.js +++ b/server/core/test/rest-api/utils.js @@ -405,6 +405,8 @@ function validate_json(data) { return validate_func(JSON.parse(data)) } +var child_process = require("child_process") + module.exports = function() { this.fs = require("fs") this.request = require("request-promise-native") @@ -419,4 +421,18 @@ module.exports = function() { this.validate_func = ajv.compile(json_api_schema) this.validate = validate_json this.base_url = "http://localhost:8989/v1" + this.before(function(done) { + child_process.execFile("./before.sh", function(err, stdout, stderr) { + if (process.env.MAXSCALE_DIR == null) { + throw new Error("MAXSCALE_DIR is not set"); + } + + done() + }) + }); + this.after(function(done) { + child_process.execFile("./after.sh", function(err, stdout, stderr) { + done() + }) + }); } From 9495438f2b2f426b4a588752d90ec9a63f669835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 4 May 2017 17:06:40 +0300 Subject: [PATCH 19/55] MXS-1220: Add server test The test creates, alters and destroys servers. --- server/core/httprequest.cc | 2 +- server/core/resource.cc | 2 + server/core/test/rest-api/after.sh | 22 ++--- .../test/rest-api/test/schema_validation.js | 4 + server/core/test/rest-api/test/server.js | 82 +++++++++++++++++++ server/core/test/rest-api/utils.js | 8 +- 6 files changed, 98 insertions(+), 22 deletions(-) create mode 100644 server/core/test/rest-api/test/server.js diff --git a/server/core/httprequest.cc b/server/core/httprequest.cc index f8580411b..5027b5a3c 100644 --- a/server/core/httprequest.cc +++ b/server/core/httprequest.cc @@ -105,8 +105,8 @@ HttpRequest::HttpRequest(struct MHD_Connection *connection, string url, string m HttpRequest::~HttpRequest() { - } + bool HttpRequest::validate_api_version() { bool rval = false; diff --git a/server/core/resource.cc b/server/core/resource.cc index 8af160e1e..701388668 100644 --- a/server/core/resource.cc +++ b/server/core/resource.cc @@ -573,5 +573,7 @@ static SpinLock resource_lock; HttpResponse resource_handle_request(const HttpRequest& request) { SpinLockGuard guard(resource_lock); + MXS_DEBUG("%s %s %s", request.get_verb().c_str(), request.get_uri().c_str(), + request.get_json_str().c_str()); return resources.process_request(request); } diff --git a/server/core/test/rest-api/after.sh b/server/core/test/rest-api/after.sh index 2bf7ee21e..de2ac6c55 100755 --- a/server/core/test/rest-api/after.sh +++ b/server/core/test/rest-api/after.sh @@ -9,27 +9,15 @@ test -z "$MAXSCALE_DIR" && exit 1 maxscaledir=$MAXSCALE_DIR -pid=`cat $maxscaledir/maxscale.pid` -echo $pid - for ((i=0;i<60;i++)) do - kill -0 $pid - - if [ $? -eq 0 ] - then - # Process is still up - kill $pid - - if [ $i -gt 3 ] - then - sleep 0.1 - fi - else - break - fi + pkill maxscale || break + sleep 0.5 done +# If it wasn't dead before, now it is +pgrep maxscale && pkill -9 maxscale + rm -r $maxscaledir/lib/maxscale rm -r $maxscaledir/cache/maxscale rm -r $maxscaledir/run/maxscale diff --git a/server/core/test/rest-api/test/schema_validation.js b/server/core/test/rest-api/test/schema_validation.js index 17efcb2ef..a7d363069 100644 --- a/server/core/test/rest-api/test/schema_validation.js +++ b/server/core/test/rest-api/test/schema_validation.js @@ -2,6 +2,8 @@ require("../utils.js")() +before(startMaxScale) + describe("Resource Collections", function(){ var tests = [ @@ -48,3 +50,5 @@ describe("Individual Resources", function(){ }); }) }); + +after(stopMaxScale) diff --git a/server/core/test/rest-api/test/server.js b/server/core/test/rest-api/test/server.js new file mode 100644 index 000000000..0d92eb16c --- /dev/null +++ b/server/core/test/rest-api/test/server.js @@ -0,0 +1,82 @@ +require("../utils.js")() + +var server = { + data: { + id: "test-server", + type: "servers", + attributes: { + parameters: { + port: 3003, + address: "127.0.0.1", + protocol: "MySQLBackend" + } + } + } +}; + +var rel = { + services: { + data: [ + { id: "RW-Split-Router", type: "services" }, + { id: "Read-Connection-Router", type: "services" }, + ] + } +}; + +describe("Creating a Server", function(){ + before(startMaxScale) + + it("create the server", function(){ + return request.post(base_url + "/servers/", {json: server }) + .should.be.fulfilled + }); + + it("request the created server", function(){ + return request.get(base_url + "/servers/" + server.data.id) + .should.be.fulfilled + }); + + it("update the created server", function(){ + server.data.attributes.parameters.weight = 10 + return request.put(base_url + "/servers/" + server.data.id, { json: server}) + .should.be.fulfilled + }); + + it("destroy the server", function(){ + return request.delete(base_url + "/servers/" + server.data.id) + .should.be.fulfilled + }); + + after(stopMaxScale) +}); + +describe("Creating a Server With Relationships", function(){ + before(startMaxScale) + + // We need a deep copy of the original server + var rel_server = JSON.parse(JSON.stringify(server)) + rel_server.data.relationships = rel + + it("create the server", function(){ + return request.post(base_url + "/servers/", {json: rel_server}) + .should.be.fulfilled + }); + + it("request the server", function(){ + return request.get(base_url + "/servers/" + rel_server.data.id) + .should.be.fulfilled + }); + + it("remove relationships", function(){ + delete rel_server.data["relationships"] + return request.put(base_url + "/servers/" + rel_server.data.id, {json: rel_server}) + .should.be.fulfilled + }); + + it("destroy the server", function(){ + return request.delete(base_url + "/servers/" + rel_server.data.id) + .should.be.fulfilled + }); + + after(stopMaxScale) +}); diff --git a/server/core/test/rest-api/utils.js b/server/core/test/rest-api/utils.js index 3c4990ebb..88664616e 100644 --- a/server/core/test/rest-api/utils.js +++ b/server/core/test/rest-api/utils.js @@ -421,7 +421,7 @@ module.exports = function() { this.validate_func = ajv.compile(json_api_schema) this.validate = validate_json this.base_url = "http://localhost:8989/v1" - this.before(function(done) { + this.startMaxScale = function(done) { child_process.execFile("./before.sh", function(err, stdout, stderr) { if (process.env.MAXSCALE_DIR == null) { throw new Error("MAXSCALE_DIR is not set"); @@ -429,10 +429,10 @@ module.exports = function() { done() }) - }); - this.after(function(done) { + }; + this.stopMaxScale = function(done) { child_process.execFile("./after.sh", function(err, stdout, stderr) { done() }) - }); + }; } From c1968aac2f393c91c6241a9e3b95a58c0b706376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 4 May 2017 19:06:18 +0300 Subject: [PATCH 20/55] MXS-1220: Migrate create/update monitor to JSON API format The creation and modification of moitor now supports the JSON API conforming format generated by the GET endpoints. Also added tests for creating and altering monitors via the REST API. --- server/core/config_runtime.cc | 23 ++++--- server/core/test/rest-api/test/monitor.js | 76 +++++++++++++++++++++++ 2 files changed, 89 insertions(+), 10 deletions(-) create mode 100644 server/core/test/rest-api/test/monitor.js diff --git a/server/core/config_runtime.cc b/server/core/config_runtime.cc index e61b8ef71..300b88211 100644 --- a/server/core/config_runtime.cc +++ b/server/core/config_runtime.cc @@ -55,6 +55,8 @@ static const char PTR_SRV_PROTOCOL[] = PTR_PARAMETERS "/protocol"; static const char PTR_SRV_AUTHENTICATOR[] = PTR_PARAMETERS "/authenticator"; static const char PTR_SRV_AUTHENTICATOR_OPTIONS[] = PTR_PARAMETERS "/authenticator_options"; +static const char PTR_MON_MODULE[] = "/data/attributes/module"; + static SPINLOCK crt_lock = SPINLOCK_INIT; bool runtime_link_server(SERVER *server, const char *target) @@ -991,7 +993,7 @@ bool runtime_alter_server_from_json(SERVER* server, json_t* new_json) const char* object_relation_types[] = { - CN_SERVERS, + PTR_RELATIONSHIPS_SERVERS, NULL }; @@ -1012,8 +1014,8 @@ static bool validate_monitor_json(json_t* json) bool rval = false; json_t* value; - if ((value = json_object_get(json, CN_NAME)) && json_is_string(value) && - (value = json_object_get(json, CN_MODULE)) && json_is_string(value)) + if ((value = mxs_json_pointer(json, PTR_ID)) && json_is_string(value) && + (value = json_object_get(json, PTR_MON_MODULE)) && json_is_string(value)) { set relations; if (extract_relations(json, relations, object_relation_types, object_relation_is_valid)) @@ -1068,8 +1070,8 @@ MXS_MONITOR* runtime_create_monitor_from_json(json_t* json) if (validate_monitor_json(json)) { - const char* name = json_string_value(json_object_get(json, CN_NAME)); - const char* module = json_string_value(json_object_get(json, CN_MODULE)); + const char* name = json_string_value(mxs_json_pointer(json, PTR_ID)); + const char* module = json_string_value(mxs_json_pointer(json, PTR_MON_MODULE)); if (runtime_create_monitor(name, module)) { @@ -1125,15 +1127,15 @@ bool runtime_alter_monitor_from_json(MXS_MONITOR* monitor, json_t* new_json) if (object_to_server_relations(monitor->name, old_json.get(), new_json)) { + rval = true; bool changed = false; - json_t* parameters = json_object_get(new_json, CN_PARAMETERS); - json_t* old_parameters = json_object_get(old_json.get(), CN_PARAMETERS); + json_t* parameters = mxs_json_pointer(new_json, PTR_PARAMETERS); + json_t* old_parameters = mxs_json_pointer(old_json.get(), PTR_PARAMETERS); ss_dassert(old_parameters); if (parameters) { - rval = true; const char* key; json_t* value; @@ -1191,8 +1193,8 @@ bool runtime_alter_service_from_json(SERVICE* service, json_t* new_json) if (object_to_server_relations(service->name, old_json.get(), new_json)) { bool changed = false; - json_t* parameters = json_object_get(new_json, CN_PARAMETERS); - json_t* old_parameters = json_object_get(old_json.get(), CN_PARAMETERS); + json_t* parameters = mxs_json_pointer(new_json, PTR_PARAMETERS); + json_t* old_parameters = mxs_json_pointer(old_json.get(), PTR_PARAMETERS); ss_dassert(old_parameters); @@ -1223,6 +1225,7 @@ bool runtime_alter_service_from_json(SERVICE* service, json_t* new_json) } else if (paramset.find(key) != paramset.end()) { + /** Parameter can be altered */ if (!runtime_alter_service(service, key, mxs::json_to_string(value).c_str())) { rval = false; diff --git a/server/core/test/rest-api/test/monitor.js b/server/core/test/rest-api/test/monitor.js new file mode 100644 index 000000000..340050883 --- /dev/null +++ b/server/core/test/rest-api/test/monitor.js @@ -0,0 +1,76 @@ +require("../utils.js")() + +var monitor = { + data: { + id: "test-monitor", + type: "monitors", + attributes: { + module: "mysqlmon" + } + } +} + +describe("Creating a Monitor", function() { + before(startMaxScale) + + it("create new monitor", function() { + return request.post(base_url + "/monitors/", {json: monitor}) + .should.be.fulfilled + }) + + it("request monitor", function() { + return request.get(base_url + "/monitors/" + monitor.data.id) + .should.be.fulfilled + }); + + it("alter monitor", function() { + monitor.data.attributes.parameters.monitor_interval = 1000 + return request.put(base_url + "/monitors/" + monitor.data.id, {json:monitor}) + .should.be.fulfilled + }); + + it("destroy created monitor", function() { + return request.delete(base_url + "/monitors/" + monitor.data.id) + .should.be.fulfilled + }); + + after(stopMaxScale) +} + +describe("Modifying Existing Monitor", function() { + before(startMaxScale) + + it("create new monitor", function() { + return request.post(base_url + "/monitors/", {json: monitor}) + .should.be.fulfilled + }) + + it("remove relationships from old monitor", function() { + + return request.get(base_url + "/monitors/MySQL-Monitor") + .then(function(resp) { + var mon = JSON.parse(resp) + delete mon.data.relationships + return request.put(base_url + "/monitors/MySQL-Monitor", {json: mon}) + }) + .should.be.fulfilled + }); + + it("add relationships to new monitor", function() { + + return request.get(base_url + "/monitors/" + monitor.data.id) + .then(function(resp) { + var mon = JSON.parse(resp) + mon.data.relationships.servers = [ + {id: "server1", type: "servers"}, + {id: "server2", type: "servers"}, + {id: "server3", type: "servers"}, + {id: "server4", type: "servers"}, + ] + return request.put(base_url + "/monitors/" + monitor.data.id, {json: mon}) + }) + .should.be.fulfilled + }); + + after(stopMaxScale) +} From 18b52adaebbd21b8628c476e5a2a290a19870ae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 4 May 2017 19:34:21 +0300 Subject: [PATCH 21/55] MXS-1220: Fix minor bug in monitor creation The JSON validation function wasn't using the JSON Pointer function for all values. --- server/core/config_runtime.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/core/config_runtime.cc b/server/core/config_runtime.cc index 300b88211..d3ad49727 100644 --- a/server/core/config_runtime.cc +++ b/server/core/config_runtime.cc @@ -720,6 +720,10 @@ bool runtime_create_monitor(const char *name, const char *module) } } } + else + { + MXS_INFO("Can't create monitor, it already exists"); + } spinlock_release(&crt_lock); return rval; @@ -1015,7 +1019,7 @@ static bool validate_monitor_json(json_t* json) json_t* value; if ((value = mxs_json_pointer(json, PTR_ID)) && json_is_string(value) && - (value = json_object_get(json, PTR_MON_MODULE)) && json_is_string(value)) + (value = mxs_json_pointer(json, PTR_MON_MODULE)) && json_is_string(value)) { set relations; if (extract_relations(json, relations, object_relation_types, object_relation_is_valid)) @@ -1085,6 +1089,10 @@ MXS_MONITOR* runtime_create_monitor_from_json(json_t* json) } } } + else + { + MXS_INFO("Invalid request JSON: %s", mxs::json_dump(json).c_str()); + } return rval; } From 7ce20b75d7b61b0e60bdc65fccbc4e904d986a38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 4 May 2017 19:35:53 +0300 Subject: [PATCH 22/55] MXS-1220: Extend and fix REST API tests Fixed a few broken tests and extended the monitor tests. --- server/core/test/rest-api/after.sh | 2 +- server/core/test/rest-api/package.json | 2 +- server/core/test/rest-api/test/monitor.js | 43 ++++++++++++++++--- .../test/rest-api/test/schema_validation.js | 26 +++++------ server/core/test/rest-api/test/server.js | 20 ++++----- 5 files changed, 63 insertions(+), 30 deletions(-) diff --git a/server/core/test/rest-api/after.sh b/server/core/test/rest-api/after.sh index de2ac6c55..982514de0 100755 --- a/server/core/test/rest-api/after.sh +++ b/server/core/test/rest-api/after.sh @@ -9,7 +9,7 @@ test -z "$MAXSCALE_DIR" && exit 1 maxscaledir=$MAXSCALE_DIR -for ((i=0;i<60;i++)) +for ((i=0;i<10;i++)) do pkill maxscale || break sleep 0.5 diff --git a/server/core/test/rest-api/package.json b/server/core/test/rest-api/package.json index 1e9348e6f..256817045 100644 --- a/server/core/test/rest-api/package.json +++ b/server/core/test/rest-api/package.json @@ -4,7 +4,7 @@ "repository": "https://github.com/mariadb-corporation/MaxScale", "description": "MaxScale REST API tests", "scripts": { - "test": "mocha --timeout 60000 --slow 10000" + "test": "mocha --timeout 30000 --slow 10000" }, "author": "", "license": "SEE LICENSE IN ../../../../LICENSE.txt", diff --git a/server/core/test/rest-api/test/monitor.js b/server/core/test/rest-api/test/monitor.js index 340050883..92b14cb05 100644 --- a/server/core/test/rest-api/test/monitor.js +++ b/server/core/test/rest-api/test/monitor.js @@ -10,7 +10,7 @@ var monitor = { } } -describe("Creating a Monitor", function() { +describe("Monitor", function() { before(startMaxScale) it("create new monitor", function() { @@ -24,7 +24,9 @@ describe("Creating a Monitor", function() { }); it("alter monitor", function() { - monitor.data.attributes.parameters.monitor_interval = 1000 + monitor.data.attributes.parameters = { + monitor_interval: 1000 + } return request.put(base_url + "/monitors/" + monitor.data.id, {json:monitor}) .should.be.fulfilled }); @@ -35,9 +37,9 @@ describe("Creating a Monitor", function() { }); after(stopMaxScale) -} +}) -describe("Modifying Existing Monitor", function() { +describe("Monitor Relationships", function() { before(startMaxScale) it("create new monitor", function() { @@ -50,7 +52,7 @@ describe("Modifying Existing Monitor", function() { return request.get(base_url + "/monitors/MySQL-Monitor") .then(function(resp) { var mon = JSON.parse(resp) - delete mon.data.relationships + delete mon.data.relationships.servers return request.put(base_url + "/monitors/MySQL-Monitor", {json: mon}) }) .should.be.fulfilled @@ -72,5 +74,34 @@ describe("Modifying Existing Monitor", function() { .should.be.fulfilled }); + it("move relationships back to old monitor", function() { + + return request.get(base_url + "/monitors/" + monitor.data.id) + .then(function(resp) { + var mon = JSON.parse(resp) + delete mon.data.relationships.servers + return request.put(base_url + "/monitors/" + monitor.data.id, {json: mon}) + }) + .then(function() { + return request.get(base_url + "/monitors/MySQL-Monitor") + }) + .then(function(resp) { + var mon = JSON.parse(resp) + mon.data.relationships.servers = [ + {id: "server1", type: "servers"}, + {id: "server2", type: "servers"}, + {id: "server3", type: "servers"}, + {id: "server4", type: "servers"}, + ] + return request.put(base_url + "/monitors/MySQL-Monitor", {json: mon}) + }) + .should.be.fulfilled + }); + + it("destroy created monitor", function() { + return request.delete(base_url + "/monitors/" + monitor.data.id) + .should.be.fulfilled + }); + after(stopMaxScale) -} +}) diff --git a/server/core/test/rest-api/test/schema_validation.js b/server/core/test/rest-api/test/schema_validation.js index a7d363069..18b3c97c4 100644 --- a/server/core/test/rest-api/test/schema_validation.js +++ b/server/core/test/rest-api/test/schema_validation.js @@ -2,9 +2,8 @@ require("../utils.js")() -before(startMaxScale) - -describe("Resource Collections", function(){ +describe("Resource Collections", function() { + before(startMaxScale) var tests = [ "/servers/", @@ -14,20 +13,23 @@ describe("Resource Collections", function(){ "/filters/", ] - tests.forEach(function(endpoint){ - it(endpoint + ': resource should be found', function() { + tests.forEach(function(endpoint) { + it(endpoint + ': resource found', function() { return request(base_url + endpoint) .should.be.fulfilled }); - it(endpoint + ': resource schema should be valid', function() { + it(endpoint + ': resource schema is valid', function() { return request(base_url + endpoint) .should.eventually.satisfy(validate) }); }) + + after(stopMaxScale) }); -describe("Individual Resources", function(){ +describe("Individual Resources", function() { + before(startMaxScale) var tests = [ "/servers/server1", @@ -38,17 +40,17 @@ describe("Individual Resources", function(){ "/sessions/1", ] - tests.forEach(function(endpoint){ - it(endpoint + ': resource should be found', function() { + tests.forEach(function(endpoint) { + it(endpoint + ': resource found', function() { return request(base_url + endpoint) .should.be.fulfilled }); - it(endpoint + ': resource schema should be valid', function() { + it(endpoint + ': resource schema is valid', function() { return request(base_url + endpoint) .should.eventually.satisfy(validate) }); }) -}); -after(stopMaxScale) + after(stopMaxScale) +}); diff --git a/server/core/test/rest-api/test/server.js b/server/core/test/rest-api/test/server.js index 0d92eb16c..30f4ae9d4 100644 --- a/server/core/test/rest-api/test/server.js +++ b/server/core/test/rest-api/test/server.js @@ -23,26 +23,26 @@ var rel = { } }; -describe("Creating a Server", function(){ +describe("Server", function() { before(startMaxScale) - it("create the server", function(){ + it("create new server", function() { return request.post(base_url + "/servers/", {json: server }) .should.be.fulfilled }); - it("request the created server", function(){ + it("request server", function() { return request.get(base_url + "/servers/" + server.data.id) .should.be.fulfilled }); - it("update the created server", function(){ + it("update server", function() { server.data.attributes.parameters.weight = 10 return request.put(base_url + "/servers/" + server.data.id, { json: server}) .should.be.fulfilled }); - it("destroy the server", function(){ + it("destroy server", function() { return request.delete(base_url + "/servers/" + server.data.id) .should.be.fulfilled }); @@ -50,30 +50,30 @@ describe("Creating a Server", function(){ after(stopMaxScale) }); -describe("Creating a Server With Relationships", function(){ +describe("Server Relationships", function() { before(startMaxScale) // We need a deep copy of the original server var rel_server = JSON.parse(JSON.stringify(server)) rel_server.data.relationships = rel - it("create the server", function(){ + it("create new server", function() { return request.post(base_url + "/servers/", {json: rel_server}) .should.be.fulfilled }); - it("request the server", function(){ + it("request server", function() { return request.get(base_url + "/servers/" + rel_server.data.id) .should.be.fulfilled }); - it("remove relationships", function(){ + it("remove relationships", function() { delete rel_server.data["relationships"] return request.put(base_url + "/servers/" + rel_server.data.id, {json: rel_server}) .should.be.fulfilled }); - it("destroy the server", function(){ + it("destroy server", function() { return request.delete(base_url + "/servers/" + rel_server.data.id) .should.be.fulfilled }); From 798ee840a8ddae5e1e4416825f8b90035eeccda8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 4 May 2017 20:41:45 +0300 Subject: [PATCH 23/55] MXS-1220: Make /maxscale/ resource conform to JSON API The /maxscale/ resource now conforms to the JSON API and a test checks that it the returned JSON is valid. --- server/core/config.cc | 33 +++++++++++-------- .../test/rest-api/test/schema_validation.js | 1 + 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/server/core/config.cc b/server/core/config.cc index 6e59791a8..f20c5a818 100644 --- a/server/core/config.cc +++ b/server/core/config.cc @@ -40,6 +40,7 @@ #include #include #include +#include #include "maxscale/config.h" #include "maxscale/filter.h" @@ -3855,20 +3856,24 @@ int config_parse_server_list(const char *servers, char ***output_array) json_t* config_paths_to_json(const char* host) { + json_t* attr = json_object(); + json_object_set_new(attr, "libdir", json_string(get_libdir())); + json_object_set_new(attr, "datadir", json_string(get_datadir())); + json_object_set_new(attr, "process_datadir", json_string(get_process_datadir())); + json_object_set_new(attr, "cachedir", json_string(get_cachedir())); + json_object_set_new(attr, "configdir", json_string(get_configdir())); + json_object_set_new(attr, "config_persistdir", json_string(get_config_persistdir())); + json_object_set_new(attr, "module_configdir", json_string(get_module_configdir())); + json_object_set_new(attr, "piddir", json_string(get_piddir())); + json_object_set_new(attr, "logdir", json_string(get_logdir())); + json_object_set_new(attr, "langdir", json_string(get_langdir())); + json_object_set_new(attr, "execdir", json_string(get_execdir())); + json_object_set_new(attr, "connector_plugindir", json_string(get_connector_plugindir())); + json_t* obj = json_object(); + json_object_set_new(obj, CN_ATTRIBUTES, attr); + json_object_set_new(obj, CN_ID, json_string(CN_MAXSCALE)); + json_object_set_new(obj, CN_TYPE, json_string(CN_MAXSCALE)); - json_object_set_new(obj, "libdir", json_string(get_libdir())); - json_object_set_new(obj, "datadir", json_string(get_datadir())); - json_object_set_new(obj, "process_datadir", json_string(get_process_datadir())); - json_object_set_new(obj, "cachedir", json_string(get_cachedir())); - json_object_set_new(obj, "configdir", json_string(get_configdir())); - json_object_set_new(obj, "config_persistdir", json_string(get_config_persistdir())); - json_object_set_new(obj, "module_configdir", json_string(get_module_configdir())); - json_object_set_new(obj, "piddir", json_string(get_piddir())); - json_object_set_new(obj, "logdir", json_string(get_logdir())); - json_object_set_new(obj, "langdir", json_string(get_langdir())); - json_object_set_new(obj, "execdir", json_string(get_execdir())); - json_object_set_new(obj, "connector_plugindir", json_string(get_connector_plugindir())); - - return obj; + return mxs_json_resource(host, MXS_JSON_API_MAXSCALE, obj); } diff --git a/server/core/test/rest-api/test/schema_validation.js b/server/core/test/rest-api/test/schema_validation.js index 18b3c97c4..8035d24f2 100644 --- a/server/core/test/rest-api/test/schema_validation.js +++ b/server/core/test/rest-api/test/schema_validation.js @@ -38,6 +38,7 @@ describe("Individual Resources", function() { "/monitors/MySQL-Monitor", "/filters/Hint", "/sessions/1", + "/maxscale/", ] tests.forEach(function(endpoint) { From 1d98b4b67ba65ead409a36c55a60db8bce02087d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 5 May 2017 05:39:23 +0300 Subject: [PATCH 24/55] MXS-1220: Return 403 Forbidden for invalid requests The JSON API specification suggests that the API returns the 403 Forbidden error when the user does an invalid request. The 400 Bad Request isn't the ideal error for cases where the syntax is correct but the action being performed is wrong. --- include/maxscale/json_api.h | 4 ++++ server/core/resource.cc | 23 ++++++++++--------- .../test/rest-api/test/schema_validation.js | 5 ++++ 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/include/maxscale/json_api.h b/include/maxscale/json_api.h index d3d15ad31..35c4fcc6f 100644 --- a/include/maxscale/json_api.h +++ b/include/maxscale/json_api.h @@ -28,6 +28,10 @@ MXS_BEGIN_DECLS #define MXS_JSON_API_MONITORS "/monitors/" #define MXS_JSON_API_SESSIONS "/sessions/" #define MXS_JSON_API_MAXSCALE "/maxscale/" +#define MXS_JSON_API_THREADS "/maxscale/threads/" +#define MXS_JSON_API_LOGS "/maxscale/logs/" +#define MXS_JSON_API_TASKS "/maxscale/tasks/" +#define MXS_JSON_API_MODULES "/maxscale/modules/" /** * @brief Create a JSON object diff --git a/server/core/resource.cc b/server/core/resource.cc index 701388668..9f3299fe8 100644 --- a/server/core/resource.cc +++ b/server/core/resource.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include "maxscale/httprequest.hh" #include "maxscale/httpresponse.hh" @@ -150,7 +151,7 @@ HttpResponse cb_create_server(const HttpRequest& request) } } - return HttpResponse(MHD_HTTP_BAD_REQUEST); + return HttpResponse(MHD_HTTP_FORBIDDEN); } HttpResponse cb_alter_server(const HttpRequest& request) @@ -167,7 +168,7 @@ HttpResponse cb_alter_server(const HttpRequest& request) } } - return HttpResponse(MHD_HTTP_BAD_REQUEST); + return HttpResponse(MHD_HTTP_FORBIDDEN); } HttpResponse cb_create_monitor(const HttpRequest& request) @@ -184,7 +185,7 @@ HttpResponse cb_create_monitor(const HttpRequest& request) } } - return HttpResponse(MHD_HTTP_BAD_REQUEST); + return HttpResponse(MHD_HTTP_FORBIDDEN); } HttpResponse cb_alter_monitor(const HttpRequest& request) @@ -201,7 +202,7 @@ HttpResponse cb_alter_monitor(const HttpRequest& request) } } - return HttpResponse(MHD_HTTP_BAD_REQUEST); + return HttpResponse(MHD_HTTP_FORBIDDEN); } HttpResponse cb_alter_service(const HttpRequest& request) @@ -218,7 +219,7 @@ HttpResponse cb_alter_service(const HttpRequest& request) } } - return HttpResponse(MHD_HTTP_BAD_REQUEST); + return HttpResponse(MHD_HTTP_FORBIDDEN); } HttpResponse cb_delete_server(const HttpRequest& request) @@ -230,7 +231,7 @@ HttpResponse cb_delete_server(const HttpRequest& request) return HttpResponse(MHD_HTTP_NO_CONTENT); } - return HttpResponse(MHD_HTTP_BAD_REQUEST); + return HttpResponse(MHD_HTTP_FORBIDDEN); } HttpResponse cb_delete_monitor(const HttpRequest& request) @@ -242,7 +243,7 @@ HttpResponse cb_delete_monitor(const HttpRequest& request) return HttpResponse(MHD_HTTP_NO_CONTENT); } - return HttpResponse(MHD_HTTP_BAD_REQUEST); + return HttpResponse(MHD_HTTP_FORBIDDEN); } HttpResponse cb_all_servers(const HttpRequest& request) @@ -355,7 +356,7 @@ HttpResponse cb_maxscale(const HttpRequest& request) HttpResponse cb_logs(const HttpRequest& request) { // TODO: Show logs - return HttpResponse(MHD_HTTP_OK); + return HttpResponse(MHD_HTTP_OK, mxs_json_resource(request.host(), MXS_JSON_API_LOGS, json_null())); } HttpResponse cb_flush(const HttpRequest& request) @@ -363,7 +364,7 @@ HttpResponse cb_flush(const HttpRequest& request) // Flush logs if (mxs_log_rotate() == 0) { - return HttpResponse(MHD_HTTP_OK); + return HttpResponse(MHD_HTTP_NO_CONTENT); } else { @@ -374,13 +375,13 @@ HttpResponse cb_flush(const HttpRequest& request) HttpResponse cb_threads(const HttpRequest& request) { // TODO: Show thread status - return HttpResponse(MHD_HTTP_OK); + return HttpResponse(MHD_HTTP_OK, mxs_json_resource(request.host(), MXS_JSON_API_THREADS, json_null())); } HttpResponse cb_tasks(const HttpRequest& request) { // TODO: Show housekeeper tasks - return HttpResponse(MHD_HTTP_OK); + return HttpResponse(MHD_HTTP_OK, mxs_json_resource(request.host(), MXS_JSON_API_TASKS, json_null())); } HttpResponse cb_all_modules(const HttpRequest& request) diff --git a/server/core/test/rest-api/test/schema_validation.js b/server/core/test/rest-api/test/schema_validation.js index 8035d24f2..2131a35c5 100644 --- a/server/core/test/rest-api/test/schema_validation.js +++ b/server/core/test/rest-api/test/schema_validation.js @@ -35,10 +35,15 @@ describe("Individual Resources", function() { "/servers/server1", "/servers/server2", "/services/RW-Split-Router", + "/services/RW-Split-Router/listeners", "/monitors/MySQL-Monitor", "/filters/Hint", "/sessions/1", "/maxscale/", + "maxscale/threads", + "maxscale/logs", + "maxscale/tasks", + "maxscale/modules", ] tests.forEach(function(endpoint) { From 08dca8c273aa50f0584af1961ff231ca8da64c80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 5 May 2017 06:38:45 +0300 Subject: [PATCH 25/55] MXS-1220: Treat service listeners as service sub-resources The listeners under the /services/:service/listeners collection are now fully JSON API compliant resources. The listeners could also be exposed as a /listeners collection to easily group all listener type resources in one place. This approach does has some semantical and practical problems, namely the fact that each listener has a many-to-one relationship with its service and listeners by themselves can't exist alone. --- include/maxscale/service.h | 10 +++++ server/core/json_api.cc | 2 +- server/core/listener.cc | 23 ++++++---- server/core/resource.cc | 10 +---- server/core/service.cc | 42 ++++++++++++------- .../test/rest-api/test/schema_validation.js | 20 +++++---- 6 files changed, 66 insertions(+), 41 deletions(-) diff --git a/include/maxscale/service.h b/include/maxscale/service.h index 5a01b8f6a..fbc01c090 100644 --- a/include/maxscale/service.h +++ b/include/maxscale/service.h @@ -304,6 +304,16 @@ json_t* service_to_json(const SERVICE* service, const char* host); */ json_t* service_list_to_json(const char* host); +/** + * @brief Convert service listeners to JSON + * + * @param service Service whose listeners are converted + * @param host Hostname of this server + * + * @return Array of JSON format listeners + */ +json_t* service_listeners_to_json(const SERVICE* service, const char* host); + /** * @brief Get links to services that relate to a server * diff --git a/server/core/json_api.cc b/server/core/json_api.cc index 5e856af16..de4688b38 100644 --- a/server/core/json_api.cc +++ b/server/core/json_api.cc @@ -32,7 +32,7 @@ static json_t* self_link(const char* host, const char* endpoint) json_t* mxs_json_resource(const char* host, const char* self, json_t* data) { - ss_dassert(data && (json_is_array(data) || json_is_object(data))); + ss_dassert(data && (json_is_array(data) || json_is_object(data) || json_is_null(data))); json_t* rval = json_object(); json_object_set_new(rval, CN_LINKS, self_link(host, self)); json_object_set_new(rval, CN_DATA, data); diff --git a/server/core/listener.cc b/server/core/listener.cc index 4006ffb23..8153f85d7 100644 --- a/server/core/listener.cc +++ b/server/core/listener.cc @@ -514,13 +514,12 @@ bool listener_serialize(const SERV_LISTENER *listener) json_t* listener_to_json(const SERV_LISTENER* listener) { - json_t* rval = json_object(); - json_object_set_new(rval, "name", json_string(listener->name)); - json_object_set_new(rval, "address", json_string(listener->address)); - json_object_set_new(rval, "port", json_integer(listener->port)); - json_object_set_new(rval, "protocol", json_string(listener->protocol)); - json_object_set_new(rval, "authenticator", json_string(listener->authenticator)); - json_object_set_new(rval, "auth_options", json_string(listener->auth_options)); + json_t* param = json_object(); + json_object_set_new(param, "address", json_string(listener->address)); + json_object_set_new(param, "port", json_integer(listener->port)); + json_object_set_new(param, "protocol", json_string(listener->protocol)); + json_object_set_new(param, "authenticator", json_string(listener->authenticator)); + json_object_set_new(param, "auth_options", json_string(listener->auth_options)); if (listener->ssl) { @@ -532,8 +531,16 @@ json_t* listener_to_json(const SERV_LISTENER* listener) json_object_set_new(ssl, "ssl_ca_cert", json_string(listener->ssl->ssl_ca_cert)); json_object_set_new(ssl, "ssl_key", json_string(listener->ssl->ssl_key)); - json_object_set_new(rval, "ssl", ssl); + json_object_set_new(param, "ssl", ssl); } + json_t* attr = json_object(); + json_object_set_new(attr, CN_PARAMETERS, param); + + json_t* rval = json_object(); + json_object_set_new(rval, CN_ATTRIBUTES, attr); + json_object_set_new(rval, CN_ID, json_string(listener->name)); + json_object_set_new(rval, CN_TYPE, json_string(CN_LISTENERS)); + return rval; } diff --git a/server/core/resource.cc b/server/core/resource.cc index 9f3299fe8..58b65bea0 100644 --- a/server/core/resource.cc +++ b/server/core/resource.cc @@ -283,15 +283,7 @@ HttpResponse cb_get_service(const HttpRequest& request) HttpResponse cb_get_service_listeners(const HttpRequest& request) { SERVICE* service = service_find(request.uri_part(1).c_str()); - json_t* json = service_to_json(service, request.host()); - - // The 'listeners' key is always defined - json_t* listeners = json_incref(json_object_get(json, CN_LISTENERS)); - ss_dassert(listeners); - - json_decref(json); - - return HttpResponse(MHD_HTTP_OK, listeners); + return HttpResponse(MHD_HTTP_OK, service_listeners_to_json(service, request.host())); } HttpResponse cb_all_filters(const HttpRequest& request) diff --git a/server/core/service.cc b/server/core/service.cc index 868ec3807..4aaac0b89 100644 --- a/server/core/service.cc +++ b/server/core/service.cc @@ -2459,6 +2459,21 @@ static inline bool have_active_servers(const SERVICE* service) return false; } +json_t* service_listeners_json_data(const SERVICE* service) +{ + json_t* arr = json_array(); + + if (service->ports) + { + for (SERV_LISTENER* p = service->ports; p; p = p->next) + { + json_array_append_new(arr, listener_to_json(p)); + } + } + + return arr; +} + json_t* service_attributes(const SERVICE* service) { json_t* attr = json_object(); @@ -2486,21 +2501,9 @@ json_t* service_attributes(const SERVICE* service) json_object_set_new(attr, "total_connections", json_integer(service->stats.n_sessions)); json_object_set_new(attr, "connections", json_integer(service->stats.n_current)); - /** Add service parameters */ + /** Add service parameters and listeners */ json_object_set_new(attr, CN_PARAMETERS, service_parameters_to_json(service)); - - /** Add listeners */ - json_t* arr = json_array(); - - if (service->ports) - { - for (SERV_LISTENER* p = service->ports; p; p = p->next) - { - json_array_append_new(arr, listener_to_json(p)); - } - } - - json_object_set_new(attr, CN_LISTENERS, arr); + json_object_set_new(attr, CN_LISTENERS, service_listeners_json_data(service)); return attr; } @@ -2561,6 +2564,17 @@ json_t* service_to_json(const SERVICE* service, const char* host) return mxs_json_resource(host, MXS_JSON_API_SERVICES, service_json_data(service, host)); } +json_t* service_listeners_to_json(const SERVICE* service, const char* host) +{ + /** This needs to be done here as the listeners are sort of sub-resources + * of the service. */ + string self = MXS_JSON_API_SERVICES; + self += service->name; + self += "/listeners"; + + return mxs_json_resource(host, self.c_str(), service_listeners_json_data(service)); +} + json_t* service_list_to_json(const char* host) { json_t* arr = json_array(); diff --git a/server/core/test/rest-api/test/schema_validation.js b/server/core/test/rest-api/test/schema_validation.js index 2131a35c5..752747fb7 100644 --- a/server/core/test/rest-api/test/schema_validation.js +++ b/server/core/test/rest-api/test/schema_validation.js @@ -6,11 +6,14 @@ describe("Resource Collections", function() { before(startMaxScale) var tests = [ - "/servers/", - "/sessions/", - "/services/", - "/monitors/", - "/filters/", + "/servers", + "/sessions", + "/services", + "/monitors", + "/filters", + "/maxscale/threads", + "/maxscale/modules", + "/maxscale/tasks", ] tests.forEach(function(endpoint) { @@ -40,10 +43,9 @@ describe("Individual Resources", function() { "/filters/Hint", "/sessions/1", "/maxscale/", - "maxscale/threads", - "maxscale/logs", - "maxscale/tasks", - "maxscale/modules", + "/maxscale/threads/0", + "/maxscale/logs", + "/maxscale/modules/readwritesplit", ] tests.forEach(function(endpoint) { From ac214435294a3e3468c09d86441ab0f6e7284da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 5 May 2017 09:41:25 +0300 Subject: [PATCH 26/55] MXS-1220: Make modules proper resources The /maxscale/modules/ endpoint is now a resource collection and the /maxscale/modules/:module endpoint exposes the individual resources. --- include/maxscale/config.h | 1 + server/core/config.cc | 1 + server/core/load_utils.cc | 50 +++++++++++++++++++++-------- server/core/maxscale/httprequest.hh | 10 ++++++ server/core/maxscale/modules.h | 11 +++++++ server/core/resource.cc | 10 +++++- 6 files changed, 69 insertions(+), 14 deletions(-) diff --git a/include/maxscale/config.h b/include/maxscale/config.h index 027d07938..07d467b5b 100644 --- a/include/maxscale/config.h +++ b/include/maxscale/config.h @@ -78,6 +78,7 @@ extern const char CN_MAXSCALE[]; extern const char CN_MAX_CONNECTIONS[]; extern const char CN_MAX_RETRY_INTERVAL[]; extern const char CN_MODULE[]; +extern const char CN_MODULES[]; extern const char CN_MONITORS[]; extern const char CN_MONITOR[]; extern const char CN_MS_TIMESTAMP[]; diff --git a/server/core/config.cc b/server/core/config.cc index f20c5a818..58fce45a9 100644 --- a/server/core/config.cc +++ b/server/core/config.cc @@ -87,6 +87,7 @@ const char CN_MAXSCALE[] = "maxscale"; const char CN_MAX_CONNECTIONS[] = "max_connections"; const char CN_MAX_RETRY_INTERVAL[] = "max_retry_interval"; const char CN_MODULE[] = "module"; +const char CN_MODULES[] = "modules"; const char CN_MONITORS[] = "monitors"; const char CN_MONITOR[] = "monitor"; const char CN_MS_TIMESTAMP[] = "ms_timestamp"; diff --git a/server/core/load_utils.cc b/server/core/load_utils.cc index 39d088b6b..c995dfb54 100644 --- a/server/core/load_utils.cc +++ b/server/core/load_utils.cc @@ -37,16 +37,17 @@ #include #include #include +#include +#include +#include #include #include #include #include -#include -#include -#include #include #include +#include #include "maxscale/modules.h" #include "maxscale/config.h" @@ -406,16 +407,19 @@ void dprintAllModules(DCB *dcb) dcb_printf(dcb, "----------------+-----------------+---------+-------+-------------------------\n\n"); } -static json_t* module_to_json(const LOADED_MODULE *mod, const char* host) +static json_t* module_json_data(const LOADED_MODULE *mod, const char* host) { json_t* obj = json_object(); - json_object_set_new(obj, "name", json_string(mod->module)); - json_object_set_new(obj, "type", json_string(mod->type)); - json_object_set_new(obj, "version", json_string(mod->info->version)); - json_object_set_new(obj, "description", json_string(mod->info->description)); - json_object_set_new(obj, "api", json_string(mxs_module_api_to_string(mod->info->modapi))); - json_object_set_new(obj, "status", json_string(mxs_module_status_to_string(mod->info->status))); + json_object_set_new(obj, CN_ID, json_string(mod->module)); + json_object_set_new(obj, CN_TYPE, json_string(CN_MODULE)); + + json_t* attr = json_object(); + json_object_set_new(attr, "module_type", json_string(mod->type)); + json_object_set_new(attr, "version", json_string(mod->info->version)); + json_object_set_new(attr, "description", json_string(mod->info->description)); + json_object_set_new(attr, "api", json_string(mxs_module_api_to_string(mod->info->modapi))); + json_object_set_new(attr, "status", json_string(mxs_module_status_to_string(mod->info->status))); json_t* params = json_array(); @@ -447,21 +451,41 @@ static json_t* module_to_json(const LOADED_MODULE *mod, const char* host) json_array_append_new(params, p); } - json_object_set_new(obj, CN_PARAMETERS, params); + json_object_set_new(attr, CN_PARAMETERS, params); + json_object_set_new(obj, CN_ATTRIBUTES, attr); return obj; } +json_t* module_to_json(const MXS_MODULE* module, const char* host) +{ + json_t* data = NULL; + + for (LOADED_MODULE *ptr = registered; ptr; ptr = ptr->next) + { + if (ptr->info == module) + { + data = module_json_data(ptr, host); + break; + } + } + + // This should always be non-NULL + ss_dassert(data); + + return mxs_json_resource(host, MXS_JSON_API_MODULES, data); +} + json_t* module_list_to_json(const char* host) { json_t* arr = json_array(); for (LOADED_MODULE *ptr = registered; ptr; ptr = ptr->next) { - json_array_append_new(arr, module_to_json(ptr, host)); + json_array_append_new(arr, module_json_data(ptr, host)); } - return arr; + return mxs_json_resource(host, MXS_JSON_API_MODULES, arr); } void moduleShowFeedbackReport(DCB *dcb) diff --git a/server/core/maxscale/httprequest.hh b/server/core/maxscale/httprequest.hh index c1cabe4fb..1cd405f05 100644 --- a/server/core/maxscale/httprequest.hh +++ b/server/core/maxscale/httprequest.hh @@ -159,6 +159,16 @@ public: return m_resource_parts.size(); } + /** + * @brief Return the last part of the URI + * + * @return The last URI part + */ + const std::string last_uri_part() const + { + return m_resource_parts.size() > 0 ? m_resource_parts[m_resource_parts.size() - 1] : ""; + } + const char* host() const { return m_hostname.c_str(); diff --git a/server/core/maxscale/modules.h b/server/core/maxscale/modules.h index 1eecff5fa..2d6748c16 100644 --- a/server/core/maxscale/modules.h +++ b/server/core/maxscale/modules.h @@ -161,10 +161,21 @@ bool mxs_module_iterator_has_next(const MXS_MODULE_ITERATOR* iterator); */ MXS_MODULE* mxs_module_iterator_get_next(MXS_MODULE_ITERATOR* iterator); +/** + * @brief Convert module to JSON + * + * @param module Module to convert + * @param host Hostname of this server + * + * @return The module in JSON format + */ +json_t* module_to_json(const MXS_MODULE* module, const char* host); + /** * @brief Convert all modules to JSON * * @param host The hostname of this server + * * @return Array of modules in JSON format */ json_t* module_list_to_json(const char* host); diff --git a/server/core/resource.cc b/server/core/resource.cc index 58b65bea0..c56db1ac7 100644 --- a/server/core/resource.cc +++ b/server/core/resource.cc @@ -89,7 +89,8 @@ bool Resource::matching_variable_path(const string& path, const string& target) if ((path == ":service" && service_find(target.c_str())) || (path == ":server" && server_find_by_unique_name(target.c_str())) || (path == ":filter" && filter_def_find(target.c_str())) || - (path == ":monitor" && monitor_find(target.c_str()))) + (path == ":monitor" && monitor_find(target.c_str())) || + (path == ":module" && get_module(target.c_str(), NULL))) { rval = true; } @@ -381,6 +382,12 @@ HttpResponse cb_all_modules(const HttpRequest& request) return HttpResponse(MHD_HTTP_OK, module_list_to_json(request.host())); } +HttpResponse cb_module(const HttpRequest& request) +{ + const MXS_MODULE* module = get_module(request.last_uri_part().c_str(), NULL); + return HttpResponse(MHD_HTTP_OK, module_to_json(module, request.host())); +} + HttpResponse cb_send_ok(const HttpRequest& request) { return HttpResponse(MHD_HTTP_OK); @@ -422,6 +429,7 @@ public: m_get.push_back(SResource(new Resource(cb_logs, 2, "maxscale", "logs"))); m_get.push_back(SResource(new Resource(cb_tasks, 2, "maxscale", "tasks"))); m_get.push_back(SResource(new Resource(cb_all_modules, 2, "maxscale", "modules"))); + m_get.push_back(SResource(new Resource(cb_module, 3, "maxscale", "modules", ":module"))); /** Create new resources */ m_post.push_back(SResource(new Resource(cb_flush, 3, "maxscale", "logs", "flush"))); From 0e57bec4efeb62d4a3a2b2a98e395e14df02c3b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 5 May 2017 09:48:30 +0300 Subject: [PATCH 27/55] MXS-1220: Add threads resource The threads are now a REST API resource exposed via the /maxscale/threads resource collection. --- include/maxscale/worker.h | 22 ++++++++ server/core/maxscale/worker.hh | 10 ++++ server/core/resource.cc | 35 +++++++++---- server/core/worker.cc | 93 ++++++++++++++++++++++++++++++++++ 4 files changed, 151 insertions(+), 9 deletions(-) diff --git a/include/maxscale/worker.h b/include/maxscale/worker.h index 711c5fbe4..f2a6d6f9f 100644 --- a/include/maxscale/worker.h +++ b/include/maxscale/worker.h @@ -15,6 +15,7 @@ #include #include #include +#include MXS_BEGIN_DECLS @@ -142,4 +143,25 @@ MXS_SESSION* mxs_worker_deregister_session(uint64_t id); */ MXS_SESSION* mxs_worker_find_session(uint64_t id); +/** + * @brief Convert a worker to JSON format + * + * @param host Hostname of this server + * @param id ID of the worker + * + * @return JSON resource representing the worker + */ +json_t* mxs_worker_to_json(const char* host, int id); + +/** + * Convert workers into JSON format + * + * @param host Hostname of this server + * + * @return A JSON resource collection of workers + * + * @see mxs_json_resource() + */ +json_t* mxs_worker_list_to_json(const char* host); + MXS_END_DECLS diff --git a/server/core/maxscale/worker.hh b/server/core/maxscale/worker.hh index 900b5f17c..35ff8ebd2 100644 --- a/server/core/maxscale/worker.hh +++ b/server/core/maxscale/worker.hh @@ -159,6 +159,16 @@ public: */ static int64_t get_one_statistic(POLL_STAT what); + /** + * Return this worker's statistics. + * + * @return Local statistics for this worker. + */ + const STATISTICS& get_local_statistics() const + { + return m_statistics; + } + /** * Add a file descriptor to the epoll instance of the worker. * diff --git a/server/core/resource.cc b/server/core/resource.cc index c56db1ac7..343ecb7a4 100644 --- a/server/core/resource.cc +++ b/server/core/resource.cc @@ -28,6 +28,7 @@ #include "maxscale/service.h" #include "maxscale/config_runtime.h" #include "maxscale/modules.h" +#include "maxscale/worker.h" using std::list; using std::string; @@ -105,6 +106,16 @@ bool Resource::matching_variable_path(const string& path, const string& target) rval = true; } } + else if (path == ":thread") + { + char* end; + int id = strtol(target.c_str(), &end, 10); + + if (*end == '\0' && mxs_worker_get(id)) + { + rval = true; + } + } } return rval; @@ -354,21 +365,26 @@ HttpResponse cb_logs(const HttpRequest& request) HttpResponse cb_flush(const HttpRequest& request) { + int code = MHD_HTTP_INTERNAL_SERVER_ERROR; + // Flush logs if (mxs_log_rotate() == 0) { - return HttpResponse(MHD_HTTP_NO_CONTENT); - } - else - { - return HttpResponse(MHD_HTTP_INTERNAL_SERVER_ERROR); + code = MHD_HTTP_NO_CONTENT; } + + return HttpResponse(code); } -HttpResponse cb_threads(const HttpRequest& request) +HttpResponse cb_all_threads(const HttpRequest& request) { - // TODO: Show thread status - return HttpResponse(MHD_HTTP_OK, mxs_json_resource(request.host(), MXS_JSON_API_THREADS, json_null())); + return HttpResponse(MHD_HTTP_OK, mxs_worker_list_to_json(request.host())); +} + +HttpResponse cb_thread(const HttpRequest& request) +{ + int id = atoi(request.last_uri_part().c_str()); + return HttpResponse(MHD_HTTP_OK, mxs_worker_to_json(request.host(), id)); } HttpResponse cb_tasks(const HttpRequest& request) @@ -425,7 +441,8 @@ public: m_get.push_back(SResource(new Resource(cb_get_session, 2, "sessions", ":session"))); m_get.push_back(SResource(new Resource(cb_maxscale, 1, "maxscale"))); - m_get.push_back(SResource(new Resource(cb_threads, 2, "maxscale", "threads"))); + m_get.push_back(SResource(new Resource(cb_all_threads, 2, "maxscale", "threads"))); + m_get.push_back(SResource(new Resource(cb_thread, 3, "maxscale", "threads", ":thread"))); m_get.push_back(SResource(new Resource(cb_logs, 2, "maxscale", "logs"))); m_get.push_back(SResource(new Resource(cb_tasks, 2, "maxscale", "tasks"))); m_get.push_back(SResource(new Resource(cb_all_modules, 2, "maxscale", "modules"))); diff --git a/server/core/worker.cc b/server/core/worker.cc index 9112a65ed..e0ed65819 100644 --- a/server/core/worker.cc +++ b/server/core/worker.cc @@ -12,11 +12,15 @@ */ #include "maxscale/worker.hh" + #include #include #include #include #include +#include +#include + #include #include #include @@ -24,13 +28,21 @@ #include #include #include +#include +#include + #include "maxscale/modules.h" #include "maxscale/poll.h" #include "maxscale/statistics.h" +#include "maxscale/workertask.hh" #define WORKER_ABSENT_ID -1 using maxscale::Worker; +using maxscale::Closer; +using maxscale::Semaphore; +using std::vector; +using std::stringstream; namespace { @@ -767,6 +779,87 @@ MXS_SESSION* Worker::find_session(uint64_t id) return rval; } +class WorkerInfoTask: public maxscale::WorkerTask +{ +public: + WorkerInfoTask(const char* host, uint32_t nthreads): + m_host(host) + { + m_data.resize(nthreads); + } + + void execute(Worker& worker) + { + json_t* stats = json_object(); + const Worker::STATISTICS& s = worker.get_local_statistics(); + json_object_set_new(stats, "reads", json_integer(s.n_read)); + json_object_set_new(stats, "writes", json_integer(s.n_write)); + json_object_set_new(stats, "errors", json_integer(s.n_error)); + json_object_set_new(stats, "hangups", json_integer(s.n_hup)); + json_object_set_new(stats, "accepts", json_integer(s.n_accept)); + json_object_set_new(stats, "blocking_polls", json_integer(s.blockingpolls)); + json_object_set_new(stats, "event_queue_length", json_integer(s.evq_length)); + json_object_set_new(stats, "max_event_queue_length", json_integer(s.evq_max)); + json_object_set_new(stats, "max_exec_time", json_integer(s.maxexectime)); + json_object_set_new(stats, "max_queue_time", json_integer(s.maxqtime)); + + json_t* attr = json_object(); + json_object_set_new(attr, "stats", stats); + + int idx = worker.get_current_id(); + stringstream ss; + ss << idx; + + json_t* json = json_object(); + json_object_set_new(json, CN_ID, json_string(ss.str().c_str())); + json_object_set_new(json, CN_TYPE, json_string(CN_THREADS)); + json_object_set_new(json, CN_ATTRIBUTES, attr); + + ss_dassert((size_t)idx < m_data.size()); + m_data[idx] = json; + } + + json_t* resource() + { + json_t* arr = json_array(); + + for (vector::iterator it = m_data.begin(); it != m_data.end(); it++) + { + json_array_append_new(arr, *it); + } + + return mxs_json_resource(m_host, MXS_JSON_API_THREADS, arr); + } + + json_t* resource(int id) + { + return mxs_json_resource(m_host, MXS_JSON_API_THREADS, m_data[id]); + } + +private: + vector m_data; + const char* m_host; +}; + +json_t* mxs_worker_to_json(const char* host, int id) +{ + Worker* target = Worker::get(id); + WorkerInfoTask task(host, id + 1); + Semaphore sem; + + target->post(&task, &sem); + sem.wait(); + + return task.resource(id); +} + +json_t* mxs_worker_list_to_json(const char* host) +{ + WorkerInfoTask task(host, config_threadcount()); + Worker::execute_concurrently(task); + return task.resource(); +} + void Worker::run() { this_thread.current_worker_id = m_id; From 16e597d43ebf2dce85c78dced22678bd7bfd64a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 5 May 2017 10:52:43 +0300 Subject: [PATCH 28/55] MXS-1220: Expand REST API test suite Added tests for services and extended the monitor test suite to test actions on monitors. --- server/core/test/rest-api/test/monitor.js | 28 +++++++++++ server/core/test/rest-api/test/service.js | 57 +++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 server/core/test/rest-api/test/service.js diff --git a/server/core/test/rest-api/test/monitor.js b/server/core/test/rest-api/test/monitor.js index 92b14cb05..4d28bec3e 100644 --- a/server/core/test/rest-api/test/monitor.js +++ b/server/core/test/rest-api/test/monitor.js @@ -105,3 +105,31 @@ describe("Monitor Relationships", function() { after(stopMaxScale) }) + +describe("Monitor Actions", function() { + before(startMaxScale) + + it("stop monitor", function() { + return request.put(base_url + "/monitors/MySQL-Monitor/stop") + .then(function() { + return request.get(base_url + "/monitors/MySQL-Monitor") + }) + .then(function(resp) { + var mon = JSON.parse(resp) + mon.data.attributes.state.should.be.equal("Stopped") + }) + }); + + it("start monitor", function() { + return request.put(base_url + "/monitors/MySQL-Monitor/start") + .then(function() { + return request.get(base_url + "/monitors/MySQL-Monitor") + }) + .then(function(resp) { + var mon = JSON.parse(resp) + mon.data.attributes.state.should.be.equal("Running") + }) + }); + + after(stopMaxScale) +}) diff --git a/server/core/test/rest-api/test/service.js b/server/core/test/rest-api/test/service.js new file mode 100644 index 000000000..d0e60ed2f --- /dev/null +++ b/server/core/test/rest-api/test/service.js @@ -0,0 +1,57 @@ +require("../utils.js")() + +describe("Service", function() { + before(startMaxScale) + + it("change service parameter", function() { + return request.get(base_url + "/services/RW-Split-Router") + .then(function(resp) { + var svc = JSON.parse(resp) + svc.data.attributes.parameters.enable_root_user = true + return request.put(base_url + "/services/RW-Split-Router", {json: svc}) + }) + .then(function(resp) { + var svc = resp + svc.data.attributes.parameters.enable_root_user.should.be.true + }) + }); + + + it("remove service relationship", function() { + return request.get(base_url + "/services/RW-Split-Router") + .then(function(resp) { + var svc = JSON.parse(resp) + delete svc.data.relationships + return request.put(base_url + "/services/RW-Split-Router", {json: svc}) + }) + .then(function(resp) { + var svc = resp + svc.data.relationships.should.be.empty + }) + }); + + it("add service relationship", function() { + return request.get(base_url + "/services/RW-Split-Router") + .then(function(resp) { + var svc = JSON.parse(resp) + svc.data.relationships = { + servers: { + data: [ + {id: "server1", type: "servers"}, + {id: "server2", type: "servers"}, + {id: "server3", type: "servers"}, + {id: "server4", type: "servers"}, + ] + } + } + + return request.put(base_url + "/services/RW-Split-Router", {json: svc}) + }) + .then(function(resp) { + var svc = resp + svc.data.relationships.servers.data[0].id.should.be.equal("server1") + }) + }); + + after(stopMaxScale) +}); From d4212b9c784c3045069e8f408380f25d1b1c1035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 5 May 2017 11:57:56 +0300 Subject: [PATCH 29/55] MXS-1220: Add request option test Currently only the `pretty=true` option is supported but it is also tested. --- server/core/test/rest-api/test/options.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 server/core/test/rest-api/test/options.js diff --git a/server/core/test/rest-api/test/options.js b/server/core/test/rest-api/test/options.js new file mode 100644 index 000000000..9ad56eae1 --- /dev/null +++ b/server/core/test/rest-api/test/options.js @@ -0,0 +1,12 @@ +require("../utils.js")() + +describe("Request Options", function() { + before(startMaxScale) + + it("pretty=true", function() { + return request.get(base_url + "/services/?pretty=true") + .should.eventually.satisfy(validate) + }) + + after(stopMaxScale) +}); From 1cb2783106d3efcac730d5ad60b8515a7a910614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 5 May 2017 11:59:46 +0300 Subject: [PATCH 30/55] MXS-1220: Implement /maxscale/logs resource The /maxscale/logs resource exposes information about the state of logging in MaxScale. --- include/maxscale/log_manager.h | 4 ++++ server/core/log_manager.cc | 31 ++++++++++++++++++++++++++++--- server/core/resource.cc | 3 +-- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/include/maxscale/log_manager.h b/include/maxscale/log_manager.h index 329155dc7..fe39cc049 100644 --- a/include/maxscale/log_manager.h +++ b/include/maxscale/log_manager.h @@ -13,11 +13,14 @@ */ #include + #include #include #include #include +#include + MXS_BEGIN_DECLS /** @@ -105,6 +108,7 @@ void mxs_log_set_augmentation(int bits); void mxs_log_set_throttling(const MXS_LOG_THROTTLING* throttling); void mxs_log_get_throttling(MXS_LOG_THROTTLING* throttling); +json_t* mxs_logs_to_json(const char* host); static inline bool mxs_log_priority_is_enabled(int priority) { diff --git a/server/core/log_manager.cc b/server/core/log_manager.cc index da9583749..b9119769c 100644 --- a/server/core/log_manager.cc +++ b/server/core/log_manager.cc @@ -11,10 +11,10 @@ * Public License. */ #include + #include #include #include - #include #include #include @@ -23,14 +23,17 @@ #include #include #include -#include -#include +#include +#include +#include #include +#include #include #include #include #include + #include "maxscale/mlist.h" #define MAX_PREFIXLEN 250 @@ -3013,3 +3016,25 @@ const char* mxs_strerror(int error) return strerror_r(error, errbuf, sizeof(errbuf)); } + +json_t* mxs_logs_to_json(const char* host) +{ + json_t* attr = json_object(); + json_object_set_new(attr, "highprecision", json_boolean(log_config.do_highprecision)); + json_object_set_new(attr, "maxlog", json_boolean(log_config.do_maxlog)); + json_object_set_new(attr, "syslog", json_boolean(log_config.do_syslog)); + + json_t* throttling = json_object(); + json_object_set_new(throttling, "count", json_integer(log_config.throttling.count)); + json_object_set_new(throttling, "suppress_ms", json_integer(log_config.throttling.suppress_ms)); + json_object_set_new(throttling, "window_ms", json_integer(log_config.throttling.window_ms)); + + json_object_set_new(attr, "throttling", throttling); + + json_t* data = json_object(); + json_object_set_new(data, CN_ATTRIBUTES, attr); + json_object_set_new(data, CN_ID, json_string("logs")); + json_object_set_new(data, CN_TYPE, json_string("logs")); + + return mxs_json_resource(host, MXS_JSON_API_LOGS, data); +} diff --git a/server/core/resource.cc b/server/core/resource.cc index 343ecb7a4..70e8207fa 100644 --- a/server/core/resource.cc +++ b/server/core/resource.cc @@ -359,8 +359,7 @@ HttpResponse cb_maxscale(const HttpRequest& request) HttpResponse cb_logs(const HttpRequest& request) { - // TODO: Show logs - return HttpResponse(MHD_HTTP_OK, mxs_json_resource(request.host(), MXS_JSON_API_LOGS, json_null())); + return HttpResponse(MHD_HTTP_OK, mxs_logs_to_json(request.host())); } HttpResponse cb_flush(const HttpRequest& request) From a384665141fc801edcb456b4b47b7763ad57b564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 5 May 2017 22:21:01 +0300 Subject: [PATCH 31/55] MXS-1220: Allow modification of logging options The logging options can now be modified with a PUT request. --- include/maxscale/config.h | 19 ++++ server/core/config_runtime.cc | 157 +++++++++++++++++++------- server/core/log_manager.cc | 12 +- server/core/maxscale/config_runtime.h | 9 ++ server/core/resource.cc | 15 ++- 5 files changed, 167 insertions(+), 45 deletions(-) diff --git a/include/maxscale/config.h b/include/maxscale/config.h index 07d467b5b..5efb52991 100644 --- a/include/maxscale/config.h +++ b/include/maxscale/config.h @@ -36,6 +36,25 @@ MXS_BEGIN_DECLS #define MAX_ADMIN_PW_LEN 1024 #define MAX_ADMIN_HOST_LEN 1024 +/** JSON Pointers to key parts of JSON objects */ +#define MXS_JSON_PTR_ID "/data/id" +#define MXS_JSON_PTR_PARAMETERS "/data/attributes/parameters" + +/** Pointers to relation lists */ +#define MXS_JSON_PTR_RELATIONSHIPS_SERVERS "/data/relationships/servers/data" +#define MXS_JSON_PTR_RELATIONSHIPS_SERVICES "/data/relationships/services/data" +#define MXS_JSON_PTR_RELATIONSHIPS_MONITORS "/data/relationships/monitors/data" +#define MXS_JSON_PTR_RELATIONSHIPS_FILTERS "/data/relationships/filters/data" + +/** Server JSON Pointers */ +#define MXS_JSON_PTR_SRV_PORT MXS_JSON_PTR_PARAMETERS "/port" +#define MXS_JSON_PTR_SRV_ADDRESS MXS_JSON_PTR_PARAMETERS "/address" +#define MXS_JSON_PTR_SRV_PROTOCOL MXS_JSON_PTR_PARAMETERS "/protocol" +#define MXS_JSON_PTR_SRV_AUTHENTICATOR MXS_JSON_PTR_PARAMETERS "/authenticator" +#define MXS_JSON_PTR_SRV_AUTHENTICATOR_OPTIONS MXS_JSON_PTR_PARAMETERS "/authenticator_options" + +#define PTR_MON_MODULE "/data/attributes/module" + /** * Common configuration parameters names * diff --git a/server/core/config_runtime.cc b/server/core/config_runtime.cc index d3ad49727..eec8406f8 100644 --- a/server/core/config_runtime.cc +++ b/server/core/config_runtime.cc @@ -38,25 +38,6 @@ using std::stringstream; using std::set; using mxs::Closer; -/** JSON Pointers to key parts of JSON objects */ -#define PTR_ID "/data/id" -#define PTR_PARAMETERS "/data/attributes/parameters" - -/** Pointers to relation lists */ -static const char PTR_RELATIONSHIPS_SERVERS[] = "/data/relationships/servers/data"; -static const char PTR_RELATIONSHIPS_SERVICES[] = "/data/relationships/services/data"; -static const char PTR_RELATIONSHIPS_MONITORS[] = "/data/relationships/monitors/data"; -static const char PTR_RELATIONSHIPS_FILTERS[] = "/data/relationships/filters/data"; - -/** Server JSON Pointers */ -static const char PTR_SRV_PORT[] = PTR_PARAMETERS "/port"; -static const char PTR_SRV_ADDRESS[] = PTR_PARAMETERS "/address"; -static const char PTR_SRV_PROTOCOL[] = PTR_PARAMETERS "/protocol"; -static const char PTR_SRV_AUTHENTICATOR[] = PTR_PARAMETERS "/authenticator"; -static const char PTR_SRV_AUTHENTICATOR_OPTIONS[] = PTR_PARAMETERS "/authenticator_options"; - -static const char PTR_MON_MODULE[] = "/data/attributes/module"; - static SPINLOCK crt_lock = SPINLOCK_INIT; bool runtime_link_server(SERVER *server, const char *target) @@ -834,9 +815,9 @@ static inline const char* string_or_null(json_t* json, const char* path) static bool server_contains_required_fields(json_t* json) { - json_t* id = mxs_json_pointer(json, PTR_ID); - json_t* port = mxs_json_pointer(json, PTR_SRV_PORT); - json_t* address = mxs_json_pointer(json, PTR_SRV_ADDRESS); + json_t* id = mxs_json_pointer(json, MXS_JSON_PTR_ID); + json_t* port = mxs_json_pointer(json, MXS_JSON_PTR_SRV_PORT); + json_t* address = mxs_json_pointer(json, MXS_JSON_PTR_SRV_ADDRESS); return (id && json_is_string(id) && address && json_is_string(address) && @@ -845,8 +826,8 @@ static bool server_contains_required_fields(json_t* json) const char* server_relation_types[] = { - PTR_RELATIONSHIPS_SERVICES, - PTR_RELATIONSHIPS_MONITORS, + MXS_JSON_PTR_RELATIONSHIPS_SERVICES, + MXS_JSON_PTR_RELATIONSHIPS_MONITORS, NULL }; @@ -894,18 +875,18 @@ SERVER* runtime_create_server_from_json(json_t* json) if (server_contains_required_fields(json)) { - const char* name = json_string_value(mxs_json_pointer(json, PTR_ID)); - const char* address = json_string_value(mxs_json_pointer(json, PTR_SRV_ADDRESS)); + const char* name = json_string_value(mxs_json_pointer(json, MXS_JSON_PTR_ID)); + const char* address = json_string_value(mxs_json_pointer(json, MXS_JSON_PTR_SRV_ADDRESS)); /** The port needs to be in string format */ char port[200]; // Enough to store any port value - int i = json_integer_value(mxs_json_pointer(json, PTR_SRV_PORT)); + int i = json_integer_value(mxs_json_pointer(json, MXS_JSON_PTR_SRV_PORT)); snprintf(port, sizeof(port), "%d", i); /** Optional parameters */ - const char* protocol = string_or_null(json, PTR_SRV_PROTOCOL); - const char* authenticator = string_or_null(json, PTR_SRV_AUTHENTICATOR); - const char* authenticator_options = string_or_null(json, PTR_SRV_AUTHENTICATOR_OPTIONS); + const char* protocol = string_or_null(json, MXS_JSON_PTR_SRV_PROTOCOL); + const char* authenticator = string_or_null(json, MXS_JSON_PTR_SRV_AUTHENTICATOR); + const char* authenticator_options = string_or_null(json, MXS_JSON_PTR_SRV_AUTHENTICATOR_OPTIONS); set relations; @@ -964,8 +945,8 @@ bool runtime_alter_server_from_json(SERVER* server, json_t* new_json) if (server_to_object_relations(server, old_json.get(), new_json)) { - json_t* parameters = mxs_json_pointer(new_json, PTR_PARAMETERS); - json_t* old_parameters = mxs_json_pointer(old_json.get(), PTR_PARAMETERS); + json_t* parameters = mxs_json_pointer(new_json, MXS_JSON_PTR_PARAMETERS); + json_t* old_parameters = mxs_json_pointer(old_json.get(), MXS_JSON_PTR_PARAMETERS); ss_dassert(old_parameters); @@ -997,7 +978,7 @@ bool runtime_alter_server_from_json(SERVER* server, json_t* new_json) const char* object_relation_types[] = { - PTR_RELATIONSHIPS_SERVERS, + MXS_JSON_PTR_RELATIONSHIPS_SERVERS, NULL }; @@ -1018,7 +999,7 @@ static bool validate_monitor_json(json_t* json) bool rval = false; json_t* value; - if ((value = mxs_json_pointer(json, PTR_ID)) && json_is_string(value) && + if ((value = mxs_json_pointer(json, MXS_JSON_PTR_ID)) && json_is_string(value) && (value = mxs_json_pointer(json, PTR_MON_MODULE)) && json_is_string(value)) { set relations; @@ -1074,7 +1055,7 @@ MXS_MONITOR* runtime_create_monitor_from_json(json_t* json) if (validate_monitor_json(json)) { - const char* name = json_string_value(mxs_json_pointer(json, PTR_ID)); + const char* name = json_string_value(mxs_json_pointer(json, MXS_JSON_PTR_ID)); const char* module = json_string_value(mxs_json_pointer(json, PTR_MON_MODULE)); if (runtime_create_monitor(name, module)) @@ -1137,8 +1118,8 @@ bool runtime_alter_monitor_from_json(MXS_MONITOR* monitor, json_t* new_json) { rval = true; bool changed = false; - json_t* parameters = mxs_json_pointer(new_json, PTR_PARAMETERS); - json_t* old_parameters = mxs_json_pointer(old_json.get(), PTR_PARAMETERS); + json_t* parameters = mxs_json_pointer(new_json, MXS_JSON_PTR_PARAMETERS); + json_t* old_parameters = mxs_json_pointer(old_json.get(), MXS_JSON_PTR_PARAMETERS); ss_dassert(old_parameters); @@ -1201,8 +1182,8 @@ bool runtime_alter_service_from_json(SERVICE* service, json_t* new_json) if (object_to_server_relations(service->name, old_json.get(), new_json)) { bool changed = false; - json_t* parameters = mxs_json_pointer(new_json, PTR_PARAMETERS); - json_t* old_parameters = mxs_json_pointer(old_json.get(), PTR_PARAMETERS); + json_t* parameters = mxs_json_pointer(new_json, MXS_JSON_PTR_PARAMETERS); + json_t* old_parameters = mxs_json_pointer(old_json.get(), MXS_JSON_PTR_PARAMETERS); ss_dassert(old_parameters); @@ -1245,3 +1226,101 @@ bool runtime_alter_service_from_json(SERVICE* service, json_t* new_json) return rval; } + +bool runtime_alter_logs_from_json(json_t* json) +{ + bool rval = false; + json_t* param = mxs_json_pointer(json, MXS_JSON_PTR_PARAMETERS); + + if (param && json_is_object(param)) + { + json_t* value; + rval = true; + + if ((value = mxs_json_pointer(param, "highprecision")) && json_is_boolean(value)) + { + if (json_is_boolean(value)) + { + mxs_log_set_highprecision_enabled(json_boolean_value(value)); + } + else + { + rval = false; + } + } + + if ((value = mxs_json_pointer(param, "maxlog")) && json_is_boolean(value)) + { + if (json_is_boolean(value)) + { + mxs_log_set_maxlog_enabled(json_boolean_value(value)); + } + else + { + rval = false; + } + } + + if ((value = mxs_json_pointer(param, "syslog"))) + { + if (json_is_boolean(value)) + { + mxs_log_set_syslog_enabled(json_boolean_value(value)); + } + else + { + rval = false; + } + } + + if ((param = mxs_json_pointer(param, "throttling")) && json_is_object(param)) + { + int intval; + MXS_LOG_THROTTLING throttle; + mxs_log_get_throttling(&throttle); + + if ((value = mxs_json_pointer(param, "count"))) + { + if (json_is_integer(value) && (intval = json_integer_value(value)) > 0) + { + throttle.count = intval; + } + else + { + rval = false; + } + } + + if ((value = mxs_json_pointer(param, "suppress_ms"))) + { + if (json_is_integer(value) && (intval = json_integer_value(value)) > 0) + { + throttle.suppress_ms = intval; + } + else + { + rval = false; + } + } + + if ((value = mxs_json_pointer(param, "window_ms"))) + { + if (json_is_integer(value) && (intval = json_integer_value(value)) > 0) + { + throttle.window_ms = intval; + } + else + { + rval = false; + } + } + + if (rval) + { + mxs_log_set_throttling(&throttle); + } + } + } + + return rval; +} diff --git a/server/core/log_manager.cc b/server/core/log_manager.cc index b9119769c..5626ec91a 100644 --- a/server/core/log_manager.cc +++ b/server/core/log_manager.cc @@ -3019,17 +3019,19 @@ const char* mxs_strerror(int error) json_t* mxs_logs_to_json(const char* host) { - json_t* attr = json_object(); - json_object_set_new(attr, "highprecision", json_boolean(log_config.do_highprecision)); - json_object_set_new(attr, "maxlog", json_boolean(log_config.do_maxlog)); - json_object_set_new(attr, "syslog", json_boolean(log_config.do_syslog)); + json_t* param = json_object(); + json_object_set_new(param, "highprecision", json_boolean(log_config.do_highprecision)); + json_object_set_new(param, "maxlog", json_boolean(log_config.do_maxlog)); + json_object_set_new(param, "syslog", json_boolean(log_config.do_syslog)); json_t* throttling = json_object(); json_object_set_new(throttling, "count", json_integer(log_config.throttling.count)); json_object_set_new(throttling, "suppress_ms", json_integer(log_config.throttling.suppress_ms)); json_object_set_new(throttling, "window_ms", json_integer(log_config.throttling.window_ms)); + json_object_set_new(param, "throttling", throttling); - json_object_set_new(attr, "throttling", throttling); + json_t* attr = json_object(); + json_object_set_new(attr, CN_PARAMETERS, param); json_t* data = json_object(); json_object_set_new(data, CN_ATTRIBUTES, attr); diff --git a/server/core/maxscale/config_runtime.h b/server/core/maxscale/config_runtime.h index d5473269d..b5600efb2 100644 --- a/server/core/maxscale/config_runtime.h +++ b/server/core/maxscale/config_runtime.h @@ -238,4 +238,13 @@ bool runtime_alter_monitor_from_json(MXS_MONITOR* monitor, json_t* new_json); */ bool runtime_alter_service_from_json(SERVICE* service, json_t* new_json); +/** + * @brief Alter logging options using JSON + * + * @param json JSON definition of the updated logging options + * + * @return True if the modifications were successful + */ +bool runtime_alter_logs_from_json(json_t* json); + MXS_END_DECLS diff --git a/server/core/resource.cc b/server/core/resource.cc index 70e8207fa..fd043d7cc 100644 --- a/server/core/resource.cc +++ b/server/core/resource.cc @@ -234,6 +234,18 @@ HttpResponse cb_alter_service(const HttpRequest& request) return HttpResponse(MHD_HTTP_FORBIDDEN); } +HttpResponse cb_alter_logs(const HttpRequest& request) +{ + json_t* json = request.get_json(); + + if (json && runtime_alter_logs_from_json(json)) + { + return HttpResponse(MHD_HTTP_NO_CONTENT); + } + + return HttpResponse(MHD_HTTP_FORBIDDEN); +} + HttpResponse cb_delete_server(const HttpRequest& request) { SERVER* server = server_find_by_unique_name(request.uri_part(1).c_str()); @@ -419,7 +431,7 @@ public: RootResource() { // Special resources required by OPTION etc. - m_get.push_back(SResource(new Resource(cb_send_ok, 1, "/"))); + m_get.push_back(SResource(new Resource(cb_send_ok, 0))); m_get.push_back(SResource(new Resource(cb_send_ok, 1, "*"))); m_get.push_back(SResource(new Resource(cb_all_servers, 1, "servers"))); @@ -456,6 +468,7 @@ public: m_put.push_back(SResource(new Resource(cb_alter_server, 2, "servers", ":server"))); m_put.push_back(SResource(new Resource(cb_alter_monitor, 2, "monitors", ":monitor"))); m_put.push_back(SResource(new Resource(cb_alter_service, 2, "services", ":service"))); + m_put.push_back(SResource(new Resource(cb_alter_logs, 2, "maxscale", "logs"))); /** Change resource states */ m_put.push_back(SResource(new Resource(cb_stop_monitor, 3, "monitors", ":monitor", "stop"))); From efc5461daa2adaf330cf0537dcc303943a90c2e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Sat, 6 May 2017 08:37:19 +0300 Subject: [PATCH 32/55] MXS-1220: Return 204 No Content for PUT and POST request Returning 204 No Content removes the cost of always sending back the modified resource. If the modified resource is required, a GET request should be made to retrieve it. Updated tests to account for this change. --- server/core/maxscale/http.hh | 18 +++++++++++++++++ server/core/resource.cc | 24 +++++++---------------- server/core/test/rest-api/test/service.js | 15 +++++++++++--- 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/server/core/maxscale/http.hh b/server/core/maxscale/http.hh index 6a37e8664..c529d1f4a 100644 --- a/server/core/maxscale/http.hh +++ b/server/core/maxscale/http.hh @@ -34,3 +34,21 @@ static inline std::string http_get_date() return std::string(buf); } + +/** + * @brief Convert a time_t value into a HTTP-date string + * + * @param t Time to convert + * + * @return The time converted to a HTTP-date string + */ +static inline std::string http_to_date(time_t t) +{ + struct tm tm; + char buf[200]; // Enough to store all dates + + gmtime_r(&t, &tm); + strftime(buf, sizeof(buf), "%a, %d %b %y %T GMT", &tm); + + return std::string(buf); +} diff --git a/server/core/resource.cc b/server/core/resource.cc index fd043d7cc..142ced7b3 100644 --- a/server/core/resource.cc +++ b/server/core/resource.cc @@ -153,14 +153,9 @@ HttpResponse cb_create_server(const HttpRequest& request) { json_t* json = request.get_json(); - if (json) + if (json && runtime_create_server_from_json(json)) { - SERVER* server = runtime_create_server_from_json(json); - - if (server) - { - return HttpResponse(MHD_HTTP_OK, server_to_json(server, request.host())); - } + return HttpResponse(MHD_HTTP_NO_CONTENT); } return HttpResponse(MHD_HTTP_FORBIDDEN); @@ -176,7 +171,7 @@ HttpResponse cb_alter_server(const HttpRequest& request) if (server && runtime_alter_server_from_json(server, json)) { - return HttpResponse(MHD_HTTP_OK, server_to_json(server, request.host())); + return HttpResponse(MHD_HTTP_NO_CONTENT); } } @@ -187,14 +182,9 @@ HttpResponse cb_create_monitor(const HttpRequest& request) { json_t* json = request.get_json(); - if (json) + if (json && runtime_create_monitor_from_json(json)) { - MXS_MONITOR* monitor = runtime_create_monitor_from_json(json); - - if (monitor) - { - return HttpResponse(MHD_HTTP_OK, monitor_to_json(monitor, request.host())); - } + return HttpResponse(MHD_HTTP_NO_CONTENT); } return HttpResponse(MHD_HTTP_FORBIDDEN); @@ -210,7 +200,7 @@ HttpResponse cb_alter_monitor(const HttpRequest& request) if (monitor && runtime_alter_monitor_from_json(monitor, json)) { - return HttpResponse(MHD_HTTP_OK, monitor_to_json(monitor, request.host())); + return HttpResponse(MHD_HTTP_NO_CONTENT); } } @@ -227,7 +217,7 @@ HttpResponse cb_alter_service(const HttpRequest& request) if (service && runtime_alter_service_from_json(service, json)) { - return HttpResponse(MHD_HTTP_OK, service_to_json(service, request.host())); + return HttpResponse(MHD_HTTP_NO_CONTENT); } } diff --git a/server/core/test/rest-api/test/service.js b/server/core/test/rest-api/test/service.js index d0e60ed2f..5b51efd5d 100644 --- a/server/core/test/rest-api/test/service.js +++ b/server/core/test/rest-api/test/service.js @@ -11,7 +11,10 @@ describe("Service", function() { return request.put(base_url + "/services/RW-Split-Router", {json: svc}) }) .then(function(resp) { - var svc = resp + return request.get(base_url + "/services/RW-Split-Router") + }) + .then(function(resp) { + var svc = JSON.parse(resp) svc.data.attributes.parameters.enable_root_user.should.be.true }) }); @@ -25,7 +28,10 @@ describe("Service", function() { return request.put(base_url + "/services/RW-Split-Router", {json: svc}) }) .then(function(resp) { - var svc = resp + return request.get(base_url + "/services/RW-Split-Router") + }) + .then(function(resp) { + var svc = JSON.parse(resp) svc.data.relationships.should.be.empty }) }); @@ -48,7 +54,10 @@ describe("Service", function() { return request.put(base_url + "/services/RW-Split-Router", {json: svc}) }) .then(function(resp) { - var svc = resp + return request.get(base_url + "/services/RW-Split-Router") + }) + .then(function(resp) { + var svc = JSON.parse(resp) svc.data.relationships.servers.data[0].id.should.be.equal("server1") }) }); From e3ff3b56bcf034fa4942488a40c59c952cd633cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Sat, 6 May 2017 09:46:47 +0300 Subject: [PATCH 33/55] MXS-1220: Add test for /maxscale/logs/ endpoint The test alters logging options and flushes the logs. --- server/core/test/rest-api/test/logs.js | 42 ++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 server/core/test/rest-api/test/logs.js diff --git a/server/core/test/rest-api/test/logs.js b/server/core/test/rest-api/test/logs.js new file mode 100644 index 000000000..10bd2d4ed --- /dev/null +++ b/server/core/test/rest-api/test/logs.js @@ -0,0 +1,42 @@ +require("../utils.js")() + +describe("Logs", function() { + before(startMaxScale) + + it("change logging options", function() { + return request.get(base_url + "/maxscale/logs") + .then(function(resp) { + var logs = JSON.parse(resp) + logs.data.attributes.parameters.maxlog.should.be.true + logs.data.attributes.parameters.syslog.should.be.true + logs.data.attributes.parameters.highprecision.should.be.false + logs.data.attributes.parameters.maxlog = false + logs.data.attributes.parameters.syslog = false + logs.data.attributes.parameters.highprecision = true + logs.data.attributes.parameters.throttling.count = 1 + logs.data.attributes.parameters.throttling.suppress_ms = 1 + logs.data.attributes.parameters.throttling.window_ms = 1 + + return request.put(base_url + "/maxscale/logs", {json: logs}) + }) + .then(function(resp) { + return request.get(base_url + "/maxscale/logs") + }) + .then(function(resp) { + var logs = JSON.parse(resp) + logs.data.attributes.parameters.maxlog.should.be.false + logs.data.attributes.parameters.syslog.should.be.false + logs.data.attributes.parameters.highprecision.should.be.true + logs.data.attributes.parameters.throttling.count.should.be.equal(1) + logs.data.attributes.parameters.throttling.suppress_ms.should.be.equal(1) + logs.data.attributes.parameters.throttling.window_ms.should.be.equal(1) + }) + }); + + it("flush logs", function() { + return request.post(base_url + "/maxscale/logs/flush") + .should.be.fulfilled + }) + + after(stopMaxScale) +}); From 3e1ff70d7d92bff016b7ea44a532be39ed0a390f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Sat, 6 May 2017 10:05:59 +0300 Subject: [PATCH 34/55] MXS-1220: Respond with 200 OK to root level requests If a request to the `/` resource is made, the API responds with an 200 OK. This is done to make it possible to use the HTTP health check mechanism found in many cloud load balancers. --- server/core/admin.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/core/admin.cc b/server/core/admin.cc index 26ae490ea..7f2f4ca11 100644 --- a/server/core/admin.cc +++ b/server/core/admin.cc @@ -104,7 +104,12 @@ int Client::process(string url, string method, const char* upload_data, size_t * HttpRequest request(m_connection, url, method, json); HttpResponse reply(MHD_HTTP_NOT_FOUND); - if (request.validate_api_version()) + if (url == "/") + { + // Respond to pings with 200 OK + reply = HttpResponse(MHD_HTTP_OK); + } + else if (request.validate_api_version()) { reply = resource_handle_request(request); } From 461cd6afd9cb4560c1060acb3f25e364ad6e791e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Sun, 7 May 2017 10:31:32 +0300 Subject: [PATCH 35/55] MXS-1220: Add creation of listeners via REST API Listeners can now be created via the REST API by doing a POST request to the service listener resource. --- include/maxscale/config.h | 19 +++-- server/core/config_runtime.cc | 85 +++++++++++++++++++---- server/core/maxscale/config_runtime.h | 10 +++ server/core/resource.cc | 15 ++++ server/core/test/rest-api/test/service.js | 23 ++++++ 5 files changed, 133 insertions(+), 19 deletions(-) diff --git a/include/maxscale/config.h b/include/maxscale/config.h index 5efb52991..2afffa84d 100644 --- a/include/maxscale/config.h +++ b/include/maxscale/config.h @@ -46,14 +46,19 @@ MXS_BEGIN_DECLS #define MXS_JSON_PTR_RELATIONSHIPS_MONITORS "/data/relationships/monitors/data" #define MXS_JSON_PTR_RELATIONSHIPS_FILTERS "/data/relationships/filters/data" -/** Server JSON Pointers */ -#define MXS_JSON_PTR_SRV_PORT MXS_JSON_PTR_PARAMETERS "/port" -#define MXS_JSON_PTR_SRV_ADDRESS MXS_JSON_PTR_PARAMETERS "/address" -#define MXS_JSON_PTR_SRV_PROTOCOL MXS_JSON_PTR_PARAMETERS "/protocol" -#define MXS_JSON_PTR_SRV_AUTHENTICATOR MXS_JSON_PTR_PARAMETERS "/authenticator" -#define MXS_JSON_PTR_SRV_AUTHENTICATOR_OPTIONS MXS_JSON_PTR_PARAMETERS "/authenticator_options" +/** Parameter value JSON Pointers */ +#define MXS_JSON_PTR_PARAM_PORT MXS_JSON_PTR_PARAMETERS "/port" +#define MXS_JSON_PTR_PARAM_ADDRESS MXS_JSON_PTR_PARAMETERS "/address" +#define MXS_JSON_PTR_PARAM_PROTOCOL MXS_JSON_PTR_PARAMETERS "/protocol" +#define MXS_JSON_PTR_PARAM_AUTHENTICATOR MXS_JSON_PTR_PARAMETERS "/authenticator" +#define MXS_JSON_PTR_PARAM_AUTHENTICATOR_OPTIONS MXS_JSON_PTR_PARAMETERS "/authenticator_options" +#define MXS_JSON_PTR_PARAM_SSL_KEY MXS_JSON_PTR_PARAMETERS "/ssl_key" +#define MXS_JSON_PTR_PARAM_SSL_CERT MXS_JSON_PTR_PARAMETERS "/ssl_cert" +#define MXS_JSON_PTR_PARAM_SSL_CA_CERT MXS_JSON_PTR_PARAMETERS "/ssl_ca_cert" +#define MXS_JSON_PTR_PARAM_SSL_VERSION MXS_JSON_PTR_PARAMETERS "/ssl_version" +#define MXS_JSON_PTR_PARAM_SSL_CERT_VERIFY_DEPTH MXS_JSON_PTR_PARAMETERS "/ssl_cert_verify_depth" -#define PTR_MON_MODULE "/data/attributes/module" +#define MXS_JSON_PTR_MODULE "/data/attributes/module" /** * Common configuration parameters names diff --git a/server/core/config_runtime.cc b/server/core/config_runtime.cc index eec8406f8..bd361a8d2 100644 --- a/server/core/config_runtime.cc +++ b/server/core/config_runtime.cc @@ -816,8 +816,8 @@ static inline const char* string_or_null(json_t* json, const char* path) static bool server_contains_required_fields(json_t* json) { json_t* id = mxs_json_pointer(json, MXS_JSON_PTR_ID); - json_t* port = mxs_json_pointer(json, MXS_JSON_PTR_SRV_PORT); - json_t* address = mxs_json_pointer(json, MXS_JSON_PTR_SRV_ADDRESS); + json_t* port = mxs_json_pointer(json, MXS_JSON_PTR_PARAM_PORT); + json_t* address = mxs_json_pointer(json, MXS_JSON_PTR_PARAM_ADDRESS); return (id && json_is_string(id) && address && json_is_string(address) && @@ -869,6 +869,14 @@ static bool link_server_to_objects(SERVER* server, set& relations) return rval; } +static string json_int_to_string(json_t* json) +{ + char str[25]; // Enough to store any 64-bit integer value + int64_t i = json_integer_value(json); + snprintf(str, sizeof(str), "%ld", i); + return string(str); +} + SERVER* runtime_create_server_from_json(json_t* json) { SERVER* rval = NULL; @@ -876,22 +884,20 @@ SERVER* runtime_create_server_from_json(json_t* json) if (server_contains_required_fields(json)) { const char* name = json_string_value(mxs_json_pointer(json, MXS_JSON_PTR_ID)); - const char* address = json_string_value(mxs_json_pointer(json, MXS_JSON_PTR_SRV_ADDRESS)); + const char* address = json_string_value(mxs_json_pointer(json, MXS_JSON_PTR_PARAM_ADDRESS)); /** The port needs to be in string format */ - char port[200]; // Enough to store any port value - int i = json_integer_value(mxs_json_pointer(json, MXS_JSON_PTR_SRV_PORT)); - snprintf(port, sizeof(port), "%d", i); + string port = json_int_to_string(mxs_json_pointer(json, MXS_JSON_PTR_PARAM_PORT)); /** Optional parameters */ - const char* protocol = string_or_null(json, MXS_JSON_PTR_SRV_PROTOCOL); - const char* authenticator = string_or_null(json, MXS_JSON_PTR_SRV_AUTHENTICATOR); - const char* authenticator_options = string_or_null(json, MXS_JSON_PTR_SRV_AUTHENTICATOR_OPTIONS); + const char* protocol = string_or_null(json, MXS_JSON_PTR_PARAM_PROTOCOL); + const char* authenticator = string_or_null(json, MXS_JSON_PTR_PARAM_AUTHENTICATOR); + const char* authenticator_options = string_or_null(json, MXS_JSON_PTR_PARAM_AUTHENTICATOR_OPTIONS); set relations; if (extract_relations(json, relations, server_relation_types, server_relation_is_valid) && - runtime_create_server(name, address, port, protocol, authenticator, authenticator_options)) + runtime_create_server(name, address, port.c_str(), protocol, authenticator, authenticator_options)) { rval = server_find_by_unique_name(name); ss_dassert(rval); @@ -1000,7 +1006,7 @@ static bool validate_monitor_json(json_t* json) json_t* value; if ((value = mxs_json_pointer(json, MXS_JSON_PTR_ID)) && json_is_string(value) && - (value = mxs_json_pointer(json, PTR_MON_MODULE)) && json_is_string(value)) + (value = mxs_json_pointer(json, MXS_JSON_PTR_MODULE)) && json_is_string(value)) { set relations; if (extract_relations(json, relations, object_relation_types, object_relation_is_valid)) @@ -1056,7 +1062,7 @@ MXS_MONITOR* runtime_create_monitor_from_json(json_t* json) if (validate_monitor_json(json)) { const char* name = json_string_value(mxs_json_pointer(json, MXS_JSON_PTR_ID)); - const char* module = json_string_value(mxs_json_pointer(json, PTR_MON_MODULE)); + const char* module = json_string_value(mxs_json_pointer(json, MXS_JSON_PTR_MODULE)); if (runtime_create_monitor(name, module)) { @@ -1324,3 +1330,58 @@ bool runtime_alter_logs_from_json(json_t* json) return rval; } + +static bool validate_listener_json(json_t* json) +{ + bool rval = false; + json_t* param; + + if ((param = mxs_json_pointer(json, MXS_JSON_PTR_ID)) && json_is_string(param) && + (param = mxs_json_pointer(json, MXS_JSON_PTR_PARAMETERS)) && json_is_object(param)) + { + json_t* value; + + if ((value = mxs_json_pointer(param, CN_PORT)) && json_is_integer(value) && + (!(value = mxs_json_pointer(param, CN_ADDRESS)) || json_is_string(value)) && + (!(value = mxs_json_pointer(param, CN_AUTHENTICATOR)) || json_is_string(value)) && + (!(value = mxs_json_pointer(param, CN_AUTHENTICATOR_OPTIONS)) || json_is_string(value)) && + (!(value = mxs_json_pointer(param, CN_SSL_KEY)) || json_is_string(value)) && + (!(value = mxs_json_pointer(param, CN_SSL_CERT)) || json_is_string(value)) && + (!(value = mxs_json_pointer(param, CN_SSL_CA_CERT)) || json_is_string(value)) && + (!(value = mxs_json_pointer(param, CN_SSL_VERSION)) || json_is_string(value)) && + (!(value = mxs_json_pointer(param, CN_SSL_CERT_VERIFY_DEPTH)) || json_is_integer(value))) + { + rval = true; + } + } + + return rval; +} + +bool runtime_create_listener_from_json(SERVICE* service, json_t* json) +{ + bool rval = false; + + if (validate_listener_json(json)) + { + string port = json_int_to_string(mxs_json_pointer(json, MXS_JSON_PTR_PARAM_PORT)); + + const char* id = string_or_null(json, MXS_JSON_PTR_ID); + const char* address = string_or_null(json, MXS_JSON_PTR_PARAM_ADDRESS); + const char* protocol = string_or_null(json, MXS_JSON_PTR_PARAM_PROTOCOL); + const char* authenticator = string_or_null(json, MXS_JSON_PTR_PARAM_AUTHENTICATOR); + const char* authenticator_options = string_or_null(json, MXS_JSON_PTR_PARAM_AUTHENTICATOR_OPTIONS); + const char* ssl_key = string_or_null(json, MXS_JSON_PTR_PARAM_SSL_KEY); + const char* ssl_cert = string_or_null(json, MXS_JSON_PTR_PARAM_SSL_CERT); + const char* ssl_ca_cert = string_or_null(json, MXS_JSON_PTR_PARAM_SSL_CA_CERT); + const char* ssl_version = string_or_null(json, MXS_JSON_PTR_PARAM_SSL_VERSION); + const char* ssl_cert_verify_depth = string_or_null(json, MXS_JSON_PTR_PARAM_SSL_CERT_VERIFY_DEPTH); + + rval = runtime_create_listener(service, id, address, port.c_str(), protocol, + authenticator, authenticator_options, + ssl_key, ssl_cert, ssl_ca_cert, ssl_version, + ssl_cert_verify_depth); + } + + return rval; +} diff --git a/server/core/maxscale/config_runtime.h b/server/core/maxscale/config_runtime.h index b5600efb2..4043251f0 100644 --- a/server/core/maxscale/config_runtime.h +++ b/server/core/maxscale/config_runtime.h @@ -238,6 +238,16 @@ bool runtime_alter_monitor_from_json(MXS_MONITOR* monitor, json_t* new_json); */ bool runtime_alter_service_from_json(SERVICE* service, json_t* new_json); +/** + * @brief Create a listener from JSON + * + * @param service Service where the listener is created + * @param json JSON definition of the new listener + * + * @return True if the listener was successfully created and started + */ +bool runtime_create_listener_from_json(SERVICE* service, json_t* json); + /** * @brief Alter logging options using JSON * diff --git a/server/core/resource.cc b/server/core/resource.cc index 142ced7b3..bf51ebc05 100644 --- a/server/core/resource.cc +++ b/server/core/resource.cc @@ -190,6 +190,19 @@ HttpResponse cb_create_monitor(const HttpRequest& request) return HttpResponse(MHD_HTTP_FORBIDDEN); } +HttpResponse cb_create_service_listener(const HttpRequest& request) +{ + json_t* json = request.get_json(); + SERVICE* service = service_find(request.uri_part(1).c_str()); + + if (service && json && runtime_create_listener_from_json(service, json)) + { + return HttpResponse(MHD_HTTP_NO_CONTENT); + } + + return HttpResponse(MHD_HTTP_FORBIDDEN); +} + HttpResponse cb_alter_monitor(const HttpRequest& request) { json_t* json = request.get_json(); @@ -453,6 +466,8 @@ public: m_post.push_back(SResource(new Resource(cb_flush, 3, "maxscale", "logs", "flush"))); m_post.push_back(SResource(new Resource(cb_create_server, 1, "servers"))); m_post.push_back(SResource(new Resource(cb_create_monitor, 1, "monitors"))); + m_post.push_back(SResource(new Resource(cb_create_service_listener, 3, + "services", ":service", "listeners"))); /** Update resources */ m_put.push_back(SResource(new Resource(cb_alter_server, 2, "servers", ":server"))); diff --git a/server/core/test/rest-api/test/service.js b/server/core/test/rest-api/test/service.js index 5b51efd5d..2aac1fdff 100644 --- a/server/core/test/rest-api/test/service.js +++ b/server/core/test/rest-api/test/service.js @@ -62,5 +62,28 @@ describe("Service", function() { }) }); + it("create a listener", function() { + var listener = { + "links": { + "self": "http://localhost:8989/v1/services/RW-Split-Router/listeners" + }, + "data": { + "attributes": { + "parameters": { + "port": 4012, + "protocol": "MySQLClient", + "authenticator": "MySQLAuth", + "address": "127.0.0.1" + } + }, + "id": "RW-Split-Listener-2", + "type": "listeners" + } + } + + return request.post(base_url + "/services/RW-Split-Router/listeners", {json: listener}) + .should.be.fulfilled + }); + after(stopMaxScale) }); From 717f88383996dab18c0a1fce06b7791d05c1701f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Sun, 7 May 2017 11:18:54 +0300 Subject: [PATCH 36/55] Make service ports lock-free The service port list is now iterated in a safe and lock-free manner. This makes the handling of service ports somewhat simpler since once an item has been added to the list it will never be removed. It also removes the need to lock the service when checking whether a service listens on a port which caused potential deadlocks. --- include/maxscale/listener.h | 1 + server/core/listener.cc | 1 + server/core/service.cc | 147 +++++++++++++++++------------------- 3 files changed, 71 insertions(+), 78 deletions(-) diff --git a/include/maxscale/listener.h b/include/maxscale/listener.h index d52cd8f11..a854b9387 100644 --- a/include/maxscale/listener.h +++ b/include/maxscale/listener.h @@ -47,6 +47,7 @@ typedef struct servlistener struct users *users; /**< The user data for this listener */ struct service* service; /**< The service which used by this listener */ SPINLOCK lock; + int active; /**< True if the port has not been deleted */ struct servlistener *next; /**< Next service protocol */ } SERV_LISTENER; diff --git a/server/core/listener.cc b/server/core/listener.cc index 8153f85d7..816ba41e3 100644 --- a/server/core/listener.cc +++ b/server/core/listener.cc @@ -119,6 +119,7 @@ listener_alloc(struct service* service, const char* name, const char *protocol, return NULL; } + proto->active = 1; proto->name = my_name; proto->listener = NULL; proto->service = service; diff --git a/server/core/service.cc b/server/core/service.cc index 4aaac0b89..a9c451ef7 100644 --- a/server/core/service.cc +++ b/server/core/service.cc @@ -94,6 +94,11 @@ static void service_internal_restart(void *data); static void service_queue_check(void *data); static void service_calculate_weights(SERVICE *service); +static inline SERV_LISTENER* load_port(SERV_LISTENER const *const *const port) +{ + return (SERV_LISTENER*)atomic_load_ptr((void**)port); +} + SERVICE* service_alloc(const char *name, const char *router) { char *my_name = MXS_STRDUP(name); @@ -499,36 +504,22 @@ int serviceInitialize(SERVICE *service) } /** - * @brief Remove a failed listener + * @brief Remove a listener from use * - * This should only be called when a newly created listener fails to start. - * - * @note The service spinlock must be held when this function is called. + * @note This does not free the memory * * @param service Service where @c port points to * @param port Port to remove */ -void serviceRemoveListener(SERVICE *service, SERV_LISTENER *port) +void serviceRemoveListener(SERVICE *service, SERV_LISTENER *target) { - - if (service->ports == port) + for (SERV_LISTENER *port = load_port(&service->ports); + port; port = load_port(&port->next)) { - service->ports = service->ports->next; - } - else - { - SERV_LISTENER *prev = service->ports; - SERV_LISTENER *current = service->ports->next; - - while (current) + if (port == target) { - if (current == port) - { - prev->next = current->next; - break; - } - prev = current; - current = current->next; + atomic_store_int32(&port->active, 0); + break; } } } @@ -544,7 +535,6 @@ bool serviceLaunchListener(SERVICE *service, SERV_LISTENER *port) { /** Failed to start the listener */ serviceRemoveListener(service, port); - listener_free(port); rval = false; } @@ -557,11 +547,10 @@ bool serviceStopListener(SERVICE *service, const char *name) { bool rval = false; - spinlock_acquire(&service->spin); - - for (SERV_LISTENER *port = service->ports; port; port = port->next) + for (SERV_LISTENER *port = load_port(&service->ports); + port; port = load_port(&port->next)) { - if (strcmp(port->name, name) == 0) + if (atomic_load_int32(&port->active) && strcmp(port->name, name) == 0) { if (poll_remove_dcb(port->listener) == 0) { @@ -572,8 +561,6 @@ bool serviceStopListener(SERVICE *service, const char *name) } } - spinlock_release(&service->spin); - return rval; } @@ -581,9 +568,8 @@ bool serviceStartListener(SERVICE *service, const char *name) { bool rval = false; - spinlock_acquire(&service->spin); - - for (SERV_LISTENER *port = service->ports; port; port = port->next) + for (SERV_LISTENER *port = load_port(&service->ports); + port; port = load_port(&port->next)) { if (strcmp(port->name, name) == 0) { @@ -597,8 +583,6 @@ bool serviceStartListener(SERVICE *service, const char *name) } } - spinlock_release(&service->spin); - return rval; } @@ -632,9 +616,11 @@ bool serviceStop(SERVICE *service) if (service) { - for (SERV_LISTENER * port = service->ports; port; port = port->next) + for (SERV_LISTENER *port = load_port(&service->ports); + port; port = load_port(&port->next)) { - if (port->listener && port->listener->session->state == SESSION_STATE_LISTENER) + if (atomic_load_int32(&port->active) && + port->listener && port->listener->session->state == SESSION_STATE_LISTENER) { if (poll_remove_dcb(port->listener) == 0) { @@ -664,9 +650,11 @@ bool serviceStart(SERVICE *service) if (service) { - for (SERV_LISTENER* port = service->ports; port; port = port->next) + for (SERV_LISTENER *port = load_port(&service->ports); + port; port = load_port(&port->next)) { - if (port->listener && port->listener->session->state == SESSION_STATE_LISTENER_STOPPED) + if (atomic_load_int32(&port->active) && + port->listener && port->listener->session->state == SESSION_STATE_LISTENER_STOPPED) { if (poll_add_dcb(port->listener) == 0) { @@ -748,10 +736,11 @@ SERV_LISTENER* serviceCreateListener(SERVICE *service, const char *name, const c if (proto) { - spinlock_acquire(&service->spin); - proto->next = service->ports; - service->ports = proto; - spinlock_release(&service->spin); + do + { + proto->next = load_port(&service->ports); + } + while (!atomic_cas_ptr((void**)&service->ports, (void**)&proto->next, proto)); } return proto; @@ -769,23 +758,20 @@ SERV_LISTENER* serviceCreateListener(SERVICE *service, const char *name, const c bool serviceHasListener(SERVICE *service, const char *protocol, const char* address, unsigned short port) { - SERV_LISTENER *proto; - spinlock_acquire(&service->spin); - proto = service->ports; - while (proto) + for (SERV_LISTENER *proto = load_port(&service->ports); + proto; proto = load_port(&proto->next)) { - if (strcmp(proto->protocol, protocol) == 0 && proto->port == port && + if (atomic_load_int32(&proto->active) && + strcmp(proto->protocol, protocol) == 0 && proto->port == port && ((address && proto->address && strcmp(proto->address, address) == 0) || (address == NULL && proto->address == NULL))) { - break; + return true; } - proto = proto->next; } - spinlock_release(&service->spin); - return proto != NULL; + return false; } /** @@ -1523,7 +1509,7 @@ void dListListeners(DCB *dcb) { SERVICE *service; - SERV_LISTENER *lptr; + SERV_LISTENER *port; spinlock_acquire(&service_spin); service = allServices; @@ -1539,19 +1525,20 @@ dListListeners(DCB *dcb) } while (service) { - lptr = service->ports; - while (lptr) + for (SERV_LISTENER *port = load_port(&service->ports); + port; port = load_port(&port->next)) { - dcb_printf(dcb, "%-20s | %-19s | %-18s | %-15s | %5d | %s\n", - lptr->name, service->name, lptr->protocol, - (lptr && lptr->address) ? lptr->address : "*", - lptr->port, - (!lptr->listener || - !lptr->listener->session || - lptr->listener->session->state == SESSION_STATE_LISTENER_STOPPED) ? - "Stopped" : "Running"); - - lptr = lptr->next; + if (atomic_load_int32(&port->active)) + { + dcb_printf(dcb, "%-20s | %-19s | %-18s | %-15s | %5d | %s\n", + port->name, service->name, port->protocol, + (port && port->address) ? port->address : "*", + port->port, + (!port->listener || + !port->listener->session || + port->listener->session->state == SESSION_STATE_LISTENER_STOPPED) ? + "Stopped" : "Running"); + } } service = service->next; } @@ -1639,10 +1626,12 @@ int service_refresh_users(SERVICE *service) ret = 0; - for (SERV_LISTENER *port = service->ports; port; port = port->next) + for (SERV_LISTENER *port = load_port(&service->ports); + port; port = load_port(&port->next)) { /** Load the authentication users before before starting the listener */ - if (port->listener && port->listener->authfunc.loadusers) + if (atomic_load_int32(&port->active) && + port->listener && port->listener->authfunc.loadusers) { switch (port->listener->authfunc.loadusers(port)) { @@ -1903,6 +1892,8 @@ serviceSessionCountAll() * Provide a row to the result set that defines the set of service * listeners * + * TODO: Replace these + * * @param set The result set * @param data The index of the row to send * @return The next row or NULL @@ -2086,7 +2077,7 @@ bool service_all_services_have_listeners() while (service) { - if (service->ports == NULL) + if (load_port(&service->ports) == NULL) { MXS_ERROR("Service '%s' has no listeners.", service->name); rval = false; @@ -2341,9 +2332,11 @@ bool service_serialize_servers(const SERVICE *service) void service_print_users(DCB *dcb, const SERVICE *service) { - for (SERV_LISTENER *port = service->ports; port; port = port->next) + for (SERV_LISTENER *port = load_port(&service->ports); + port; port = load_port(&port->next)) { - if (port->listener && port->listener->authfunc.diagnostic) + if (atomic_load_int32(&port->active) && + port->listener && port->listener->authfunc.diagnostic) { port->listener->authfunc.diagnostic(dcb, port); } @@ -2357,18 +2350,15 @@ bool service_port_is_used(unsigned short port) for (SERVICE *service = allServices; service && !rval; service = service->next) { - spinlock_acquire(&service->spin); - - for (SERV_LISTENER *proto = service->ports; proto; proto = proto->next) + for (SERV_LISTENER *proto = load_port(&service->ports); + proto; proto = load_port(&proto->next)) { - if (proto->port == port) + if (atomic_load_int32(&proto->active) && proto->port == port) { rval = true; break; } } - - spinlock_release(&service->spin); } spinlock_release(&service_spin); @@ -2463,11 +2453,12 @@ json_t* service_listeners_json_data(const SERVICE* service) { json_t* arr = json_array(); - if (service->ports) + for (SERV_LISTENER *port = load_port(&service->ports); + port; port = load_port(&port->next)) { - for (SERV_LISTENER* p = service->ports; p; p = p->next) + if (atomic_load_int32(&port->active)) { - json_array_append_new(arr, listener_to_json(p)); + json_array_append_new(arr, listener_to_json(port)); } } From 3deb4973944761a3aae78d565ba289a0fd422d29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Sun, 7 May 2017 23:32:18 +0300 Subject: [PATCH 37/55] MXS-1220: Add Last-Modified and ETag headers The resource system now tracks both the time when a resource was last modified and the revision number of the resource. This allows working Last-Modified and ETag headers to be generated by the REST API. The If-Modified-Since and If-None-Match request headers are not yet processed and using them will always return the resource instead of a 304 Not Modified response. --- server/core/httpresponse.cc | 3 - server/core/maxscale/resource.hh | 4 +- server/core/resource.cc | 111 ++++++++++++++++++++++++- server/core/test/rest-api/test/http.js | 59 +++++++++++++ 4 files changed, 170 insertions(+), 7 deletions(-) create mode 100644 server/core/test/rest-api/test/http.js diff --git a/server/core/httpresponse.cc b/server/core/httpresponse.cc index 8708d7c11..557ce4c8e 100644 --- a/server/core/httpresponse.cc +++ b/server/core/httpresponse.cc @@ -30,9 +30,6 @@ HttpResponse::HttpResponse(int code, json_t* response): { string http_date = http_get_date(); add_header(HTTP_RESPONSE_HEADER_DATE, http_date); - add_header(HTTP_RESPONSE_HEADER_LAST_MODIFIED, http_date); - // This ETag is the base64 encoding of `not-yet-implemented` - add_header(HTTP_RESPONSE_HEADER_ETAG, "bm90LXlldC1pbXBsZW1lbnRlZAo"); if (m_body) { diff --git a/server/core/maxscale/resource.hh b/server/core/maxscale/resource.hh index b3e80498d..c92c77611 100644 --- a/server/core/maxscale/resource.hh +++ b/server/core/maxscale/resource.hh @@ -62,8 +62,8 @@ private: bool matching_variable_path(const std::string& path, const std::string& target) const; - ResourceCallback m_cb; /**< Resource handler callback */ - std::deque m_path; /**< Path components */ + ResourceCallback m_cb; /**< Resource handler callback */ + std::deque m_path; /**< Path components */ }; /** diff --git a/server/core/resource.cc b/server/core/resource.cc index bf51ebc05..e68262a38 100644 --- a/server/core/resource.cc +++ b/server/core/resource.cc @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -29,13 +30,76 @@ #include "maxscale/config_runtime.h" #include "maxscale/modules.h" #include "maxscale/worker.h" +#include "maxscale/http.hh" using std::list; +using std::map; using std::string; using std::stringstream; using mxs::SpinLock; using mxs::SpinLockGuard; +/** + * Class that keeps track of resource modification times + */ +class ResourceWatcher +{ +public: + + ResourceWatcher() : + m_init(time(NULL)) + { + } + + void modify(const string& path) + { + map::iterator it = m_etag.find(path); + + if (it != m_etag.end()) + { + it->second++; + } + else + { + // First modification + m_etag[path] = 1; + } + + m_last_modified[path] = time(NULL); + } + + time_t last_modified(const string& path) const + { + map::const_iterator it = m_last_modified.find(path); + + if (it != m_last_modified.end()) + { + return it->second; + } + + // Resource has not yet been updated + return m_init; + } + + time_t etag(const string& path) const + { + map::const_iterator it = m_etag.find(path); + + if (it != m_etag.end()) + { + return it->second; + } + + // Resource has not yet been updated + return 0; + } + +private: + time_t m_init; + map m_last_modified; + map m_etag; +}; + Resource::Resource(ResourceCallback cb, int components, ...) : m_cb(cb) { @@ -603,12 +667,55 @@ private: }; static RootResource resources; /**< Core resource set */ +static ResourceWatcher watcher; /**< Modification watcher */ static SpinLock resource_lock; +static bool request_modifies_data(const string& verb) +{ + return verb == MHD_HTTP_METHOD_POST || + verb == MHD_HTTP_METHOD_PUT || + verb == MHD_HTTP_METHOD_DELETE; +} + +static bool request_reads_data(const string& verb) +{ + return verb == MHD_HTTP_METHOD_GET || + verb == MHD_HTTP_METHOD_HEAD; +} + HttpResponse resource_handle_request(const HttpRequest& request) { - SpinLockGuard guard(resource_lock); MXS_DEBUG("%s %s %s", request.get_verb().c_str(), request.get_uri().c_str(), request.get_json_str().c_str()); - return resources.process_request(request); + + SpinLockGuard guard(resource_lock); + HttpResponse rval = resources.process_request(request); + + if (request_modifies_data(request.get_verb())) + { + switch (rval.get_code()) + { + case MHD_HTTP_OK: + case MHD_HTTP_NO_CONTENT: + case MHD_HTTP_CREATED: + watcher.modify(request.get_uri()); + break; + + default: + break; + } + } + else if (request_reads_data(request.get_verb())) + { + const string& uri = request.get_uri(); + + rval.add_header(HTTP_RESPONSE_HEADER_LAST_MODIFIED, + http_to_date(watcher.last_modified(uri))); + + stringstream ss; + ss << watcher.etag(uri); + rval.add_header(HTTP_RESPONSE_HEADER_ETAG, ss.str()); + } + + return rval; } diff --git a/server/core/test/rest-api/test/http.js b/server/core/test/rest-api/test/http.js new file mode 100644 index 000000000..c919cc8f1 --- /dev/null +++ b/server/core/test/rest-api/test/http.js @@ -0,0 +1,59 @@ +require("../utils.js")() + + +describe("HTTP Headers", function() { + before(startMaxScale) + + it("ETag changes after modification", function() { + return request.get(base_url + "/servers/server1", {resolveWithFullResponse: true}) + .then(function(resp) { + resp.headers.etag.should.be.equal("0") + var srv = JSON.parse(resp.body) + delete srv.data.relationships + return request.put(base_url + "/servers/server1", {json: srv}) + }) + .then(function() { + return request.get(base_url + "/servers/server1", {resolveWithFullResponse: true}) + }) + .then(function(resp) { + resp.headers.etag.should.be.equal("1") + }) + }); + + it("Last-Modified changes after modification", function() { + var date; + + return request.get(base_url + "/servers/server1", {resolveWithFullResponse: true}) + .then(function(resp) { + + // Store the current modification time + resp.headers["last-modified"].should.not.be.null + date = resp.headers["last-modified"] + + // Modify resource after three seconds + setTimeout(function() { + var srv = JSON.parse(resp.body) + + srv.data.relationships = { + services: { + data: [ + {id: "RW-Split-Router", type: "services"} + ] + } + } + + request.put(base_url + "/servers/server1", {json: srv}) + .then(function() { + return request.get(base_url + "/servers/server1", {resolveWithFullResponse: true}) + }) + .then(function(resp) { + resp.headers["last-modified"].should.not.be.null + resp.headers["last-modified"].should.not.be.equal(date) + }) + + }, 3000) + }) + }); + + after(stopMaxScale) +}); From 778631a8608dbd235805cd14083233cf89157375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 8 May 2017 08:22:59 +0300 Subject: [PATCH 38/55] MXS-1220: Add support for conditional HTTP requests The REST API now supports the If-Modified-Since, If-Unmodified-Since, If-Match and If-None-Match headers and returns the correct response if the conditional fails. Added tests for the date parsing and expanded the HTTP header tests in the REST API test suite. --- server/core/maxscale/http.hh | 37 ++++++++++- server/core/resource.cc | 89 ++++++++++++++++++++------ server/core/test/CMakeLists.txt | 3 + server/core/test/rest-api/test/http.js | 77 ++++++++++++++++++++-- server/core/test/testhttp.cc | 39 +++++++++++ 5 files changed, 220 insertions(+), 25 deletions(-) create mode 100644 server/core/test/testhttp.cc diff --git a/server/core/maxscale/http.hh b/server/core/maxscale/http.hh index c529d1f4a..1632827be 100644 --- a/server/core/maxscale/http.hh +++ b/server/core/maxscale/http.hh @@ -15,6 +15,7 @@ #include #include +#include #include @@ -48,7 +49,41 @@ static inline std::string http_to_date(time_t t) char buf[200]; // Enough to store all dates gmtime_r(&t, &tm); - strftime(buf, sizeof(buf), "%a, %d %b %y %T GMT", &tm); + strftime(buf, sizeof(buf), "%a, %d %b %Y %T GMT", &tm); return std::string(buf); } + +/** + * @brief Convert a HTTP-date string into time_t + * + * @param str HTTP-date formatted string to convert + * + * @return The time converted to time_t + */ +static inline time_t http_from_date(const std::string& str) +{ + struct tm tm = {}; + + /** First get the GMT time in time_t format */ + strptime(str.c_str(), "%a, %d %b %Y %T GMT", &tm); + time_t t = mktime(&tm); + + /** Then convert it to local time by calculating the difference between + * the local time and the GMT time */ + struct tm local_tm = {}; + struct tm gmt_tm = {}; + time_t epoch = 0; + + /** Call tzset() for the sake of portability */ + tzset(); + gmtime_r(&epoch, &gmt_tm); + localtime_r(&epoch, &local_tm); + + time_t gmt_t = mktime(&gmt_tm); + time_t local_t = mktime(&local_tm); + + /** The value of `(gmt_t - local_t)` will be the number of seconds west + * from GMT. For timezones east of GMT, it will be negative. */ + return t - (gmt_t - local_t); +} diff --git a/server/core/resource.cc b/server/core/resource.cc index e68262a38..d8cc88302 100644 --- a/server/core/resource.cc +++ b/server/core/resource.cc @@ -683,38 +683,89 @@ static bool request_reads_data(const string& verb) verb == MHD_HTTP_METHOD_HEAD; } +bool request_precondition_met(const HttpRequest& request, HttpResponse& response) +{ + bool rval = true; + string str; + const string& uri = request.get_uri(); + + if ((str = request.get_header(MHD_HTTP_HEADER_IF_MODIFIED_SINCE)).length()) + { + if (watcher.last_modified(uri) <= http_from_date(str)) + { + rval = false; + response = HttpResponse(MHD_HTTP_NOT_MODIFIED); + } + } + else if ((str = request.get_header(MHD_HTTP_HEADER_IF_UNMODIFIED_SINCE)).length()) + { + if (watcher.last_modified(uri) > http_from_date(str)) + { + rval = false; + response = HttpResponse(MHD_HTTP_PRECONDITION_FAILED); + } + } + else if ((str = request.get_header(MHD_HTTP_HEADER_IF_MATCH)).length()) + { + str = str.substr(1, str.length() - 2); + + if (watcher.etag(uri) != strtol(str.c_str(), NULL, 10)) + { + rval = false; + response = HttpResponse(MHD_HTTP_PRECONDITION_FAILED); + } + } + else if ((str = request.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH)).length()) + { + str = str.substr(1, str.length() - 2); + + if (watcher.etag(uri) == strtol(str.c_str(), NULL, 10)) + { + rval = false; + response = HttpResponse(MHD_HTTP_NOT_MODIFIED); + } + } + + return rval; +} + HttpResponse resource_handle_request(const HttpRequest& request) { MXS_DEBUG("%s %s %s", request.get_verb().c_str(), request.get_uri().c_str(), request.get_json_str().c_str()); SpinLockGuard guard(resource_lock); - HttpResponse rval = resources.process_request(request); + HttpResponse rval; - if (request_modifies_data(request.get_verb())) + if (request_precondition_met(request, rval)) { - switch (rval.get_code()) + rval = resources.process_request(request); + + if (request_modifies_data(request.get_verb())) { - case MHD_HTTP_OK: - case MHD_HTTP_NO_CONTENT: - case MHD_HTTP_CREATED: - watcher.modify(request.get_uri()); - break; + switch (rval.get_code()) + { + case MHD_HTTP_OK: + case MHD_HTTP_NO_CONTENT: + case MHD_HTTP_CREATED: + watcher.modify(request.get_uri()); + break; - default: - break; + default: + break; + } } - } - else if (request_reads_data(request.get_verb())) - { - const string& uri = request.get_uri(); + else if (request_reads_data(request.get_verb())) + { + const string& uri = request.get_uri(); - rval.add_header(HTTP_RESPONSE_HEADER_LAST_MODIFIED, - http_to_date(watcher.last_modified(uri))); + rval.add_header(HTTP_RESPONSE_HEADER_LAST_MODIFIED, + http_to_date(watcher.last_modified(uri))); - stringstream ss; - ss << watcher.etag(uri); - rval.add_header(HTTP_RESPONSE_HEADER_ETAG, ss.str()); + stringstream ss; + ss << "\"" << watcher.etag(uri) << "\""; + rval.add_header(HTTP_RESPONSE_HEADER_ETAG, ss.str()); + } } return rval; diff --git a/server/core/test/CMakeLists.txt b/server/core/test/CMakeLists.txt index c2c674acc..e265a3cc8 100644 --- a/server/core/test/CMakeLists.txt +++ b/server/core/test/CMakeLists.txt @@ -24,6 +24,7 @@ add_executable(testmodulecmd testmodulecmd.cc) add_executable(testconfig testconfig.cc) add_executable(trxboundaryparser_profile trxboundaryparser_profile.cc) add_executable(testjson testjson.cc) +add_executable(testhttp testhttp.cc) target_link_libraries(test_atomic maxscale-common) target_link_libraries(test_adminusers maxscale-common) target_link_libraries(test_buffer maxscale-common) @@ -50,6 +51,7 @@ target_link_libraries(testmodulecmd maxscale-common) target_link_libraries(testconfig maxscale-common) target_link_libraries(trxboundaryparser_profile maxscale-common) target_link_libraries(testjson maxscale-common) +target_link_libraries(testhttp maxscale-common) add_test(TestAtomic test_atomic) add_test(TestAdminUsers test_adminusers) add_test(TestBuffer test_buffer) @@ -82,6 +84,7 @@ add_test(TestTrxCompare_Set test_trxcompare ${CMAKE_CURRENT_SOURCE_DIR}/../../.. add_test(TestTrxCompare_Update test_trxcompare ${CMAKE_CURRENT_SOURCE_DIR}/../../../query_classifier/test/update.test) add_test(TestTrxCompare_MaxScale test_trxcompare ${CMAKE_CURRENT_SOURCE_DIR}/../../../query_classifier/test/maxscale.test) add_test(TestJson testjson) +add_test(TestHttp testhttp) # This test requires external dependencies and thus cannot be run # as a part of the core test set diff --git a/server/core/test/rest-api/test/http.js b/server/core/test/rest-api/test/http.js index c919cc8f1..8287e5826 100644 --- a/server/core/test/rest-api/test/http.js +++ b/server/core/test/rest-api/test/http.js @@ -7,7 +7,7 @@ describe("HTTP Headers", function() { it("ETag changes after modification", function() { return request.get(base_url + "/servers/server1", {resolveWithFullResponse: true}) .then(function(resp) { - resp.headers.etag.should.be.equal("0") + resp.headers.etag.should.be.equal("\"0\"") var srv = JSON.parse(resp.body) delete srv.data.relationships return request.put(base_url + "/servers/server1", {json: srv}) @@ -16,14 +16,14 @@ describe("HTTP Headers", function() { return request.get(base_url + "/servers/server1", {resolveWithFullResponse: true}) }) .then(function(resp) { - resp.headers.etag.should.be.equal("1") + resp.headers.etag.should.be.equal("\"1\"") }) }); - it("Last-Modified changes after modification", function() { + it("Last-Modified changes after modification", function(done) { var date; - return request.get(base_url + "/servers/server1", {resolveWithFullResponse: true}) + request.get(base_url + "/servers/server1", {resolveWithFullResponse: true}) .then(function(resp) { // Store the current modification time @@ -49,11 +49,78 @@ describe("HTTP Headers", function() { .then(function(resp) { resp.headers["last-modified"].should.not.be.null resp.headers["last-modified"].should.not.be.equal(date) + done() + }) + .catch(function(e) { + done(e) }) - }, 3000) + }, 2000) + }) + .catch(function(e) { + done(e) }) }); + var oldtime = new Date(new Date().getTime() - 1000000).toUTCString(); + var newtime = new Date(new Date().getTime() + 1000000).toUTCString(); + + it("request with older If-Modified-Since value", function() { + return request.get(base_url + "/servers/server1", { + headers: { "If-Modified-Since": oldtime} + }).should.be.fulfilled + }) + + it("request with newer If-Modified-Since value", function() { + return request.get(base_url + "/servers/server1", { + resolveWithFullResponse: true, + headers: { "If-Modified-Since": newtime } + }).should.be.rejected + }) + + it("request with older If-Unmodified-Since value", function() { + return request.get(base_url + "/servers/server1", { + headers: { "If-Unmodified-Since": oldtime} + }).should.be.rejected + }) + + it("request with newer If-Unmodified-Since value", function() { + return request.get(base_url + "/servers/server1", { + headers: { "If-Unmodified-Since": newtime} + }).should.be.fulfilled + }) + + it("request with mismatching If-Match value", function() { + return request.get(base_url + "/servers/server1", { + headers: { "If-Match": "\"0\""} + }).should.be.rejected + }) + + it("request with matching If-Match value", function() { + return request.get(base_url + "/servers/server1", { resolveWithFullResponse: true }) + .then(function(resp) { + return request.get(base_url + "/servers/server1", { + headers: { "If-Match": resp.headers["etag"]} + }) + }) + .should.be.fulfilled + }) + + it("request with mismatching If-None-Match value", function() { + return request.get(base_url + "/servers/server1", { + headers: { "If-None-Match": "\"0\""} + }).should.be.fulfilled + }) + + it("request with matching If-None-Match value", function() { + return request.get(base_url + "/servers/server1", { resolveWithFullResponse: true }) + .then(function(resp) { + return request.get(base_url + "/servers/server1", { + headers: { "If-None-Match": resp.headers["etag"]} + }) + }) + .should.be.rejected + }) + after(stopMaxScale) }); diff --git a/server/core/test/testhttp.cc b/server/core/test/testhttp.cc new file mode 100644 index 000000000..04e439459 --- /dev/null +++ b/server/core/test/testhttp.cc @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl11. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include + +#include +#include "../maxscale/http.hh" + +using std::string; +using std::cout; +using std::endl; + +int main(int argc, char** argv) +{ + time_t now = time(NULL); + string date = http_to_date(now); + time_t converted_now = http_from_date(date); + string converted_date = http_to_date(converted_now); + + cout << "Current linux time: " << now << endl; + cout << "HTTP-date from current time: " << date << endl; + cout << "Converted Linux time: " << converted_now << endl; + cout << "Converted HTTP-date: " << converted_date << endl; + + ss_dassert(now == converted_now); + ss_dassert(date == converted_date); + + return 0; +} From c30b817abc75855aaec21eddfa6b2c7082b810b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 8 May 2017 10:00:44 +0300 Subject: [PATCH 39/55] MXS-1220: Add warnings for invalid request JSON Converted info level log messages to warnings and added a missing warning to server creation. --- server/core/config_runtime.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/core/config_runtime.cc b/server/core/config_runtime.cc index bd361a8d2..0a6e7e840 100644 --- a/server/core/config_runtime.cc +++ b/server/core/config_runtime.cc @@ -703,7 +703,7 @@ bool runtime_create_monitor(const char *name, const char *module) } else { - MXS_INFO("Can't create monitor, it already exists"); + MXS_WARNING("Can't create monitor, it already exists"); } spinlock_release(&crt_lock); @@ -909,6 +909,10 @@ SERVER* runtime_create_server_from_json(json_t* json) } } } + else + { + MXS_WARNING("Invalid request JSON: %s", mxs::json_dump(json).c_str()); + } return rval; } @@ -1078,7 +1082,7 @@ MXS_MONITOR* runtime_create_monitor_from_json(json_t* json) } else { - MXS_INFO("Invalid request JSON: %s", mxs::json_dump(json).c_str()); + MXS_WARNING("Invalid request JSON: %s", mxs::json_dump(json).c_str()); } return rval; From 613b924f2e4b572e5a15c668052b5633188b9a6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 8 May 2017 14:17:14 +0300 Subject: [PATCH 40/55] Add listener iterator The listener iterator hides the details of the listener iteration behind a small set of functions. The following for-loop demonstrates the main use-case for the iterator: LISTENER_ITERATOR iter; for (SERV_LISTENER *listener = listener_iterator_init(service, &iter); listener; listener = listener_iterator_next(&iter)) { /** Do something with the listener */ } As the listeners are mostly iterated to either check a fact about them or print their information, the functions cater to that use-case. For this reason, they should always be allocated off the stack. --- include/maxscale/listener.h | 43 ++++++++- server/core/listener.cc | 34 +++++++ server/core/service.cc | 175 +++++++++++++++++++++--------------- 3 files changed, 179 insertions(+), 73 deletions(-) diff --git a/include/maxscale/listener.h b/include/maxscale/listener.h index a854b9387..b7b56db6c 100644 --- a/include/maxscale/listener.h +++ b/include/maxscale/listener.h @@ -49,7 +49,12 @@ typedef struct servlistener SPINLOCK lock; int active; /**< True if the port has not been deleted */ struct servlistener *next; /**< Next service protocol */ -} SERV_LISTENER; +} SERV_LISTENER; // TODO: Rename to LISTENER + +typedef struct listener_iterator +{ + SERV_LISTENER* current; +} LISTENER_ITERATOR; /** * @brief Serialize a listener to a file @@ -80,4 +85,40 @@ int listener_set_ssl_version(SSL_LISTENER *ssl_listener, char* version); void listener_set_certificates(SSL_LISTENER *ssl_listener, char* cert, char* key, char* ca_cert); int listener_init_SSL(SSL_LISTENER *ssl_listener); +/** + * @brief Check if listener is active + * + * @param listener Listener to check + * + * @return True if listener is active + */ +bool listener_is_active(SERV_LISTENER* listener); + +/** + * @brief Modify listener active state + * + * @param listener Listener to modify + * @param active True to activate, false to disable + */ +void listener_set_active(SERV_LISTENER* listener, bool active); + +/** + * @brief Initialize a listener iterator for iterating service listeners + * + * @param service Service whose listeners are iterated + * @param iter Pointer to iterator to initialize + * + * @return The first value pointed by the iterator + */ +SERV_LISTENER* listener_iterator_init(const struct service* service, LISTENER_ITERATOR* iter); + +/** + * @brief Get the next listener + * + * @param iter Listener iterator + * + * @return The next listener or NULL on end of list + */ +SERV_LISTENER* listener_iterator_next(LISTENER_ITERATOR* iter); + MXS_END_DECLS diff --git a/server/core/listener.cc b/server/core/listener.cc index 816ba41e3..3f537b83e 100644 --- a/server/core/listener.cc +++ b/server/core/listener.cc @@ -545,3 +545,37 @@ json_t* listener_to_json(const SERV_LISTENER* listener) return rval; } + +void listener_set_active(SERV_LISTENER* listener, bool active) +{ + atomic_store_int32(&listener->active, active ? 1 : 0); +} + +bool listener_is_active(SERV_LISTENER* listener) +{ + return atomic_load_int32(&listener->active); +} + +static inline SERV_LISTENER* load_port(SERV_LISTENER const *const *const port) +{ + return (SERV_LISTENER*)atomic_load_ptr((void**)port); +} + +SERV_LISTENER* listener_iterator_init(const SERVICE* service, LISTENER_ITERATOR* iter) +{ + ss_dassert(iter); + iter->current = load_port(&service->ports); + return iter->current; +} + +SERV_LISTENER* listener_iterator_next(LISTENER_ITERATOR* iter) +{ + ss_dassert(iter); + + if (iter->current) + { + iter->current = load_port(&iter->current->next); + } + + return iter->current; +} diff --git a/server/core/service.cc b/server/core/service.cc index a9c451ef7..2df6b9c0b 100644 --- a/server/core/service.cc +++ b/server/core/service.cc @@ -94,11 +94,6 @@ static void service_internal_restart(void *data); static void service_queue_check(void *data); static void service_calculate_weights(SERVICE *service); -static inline SERV_LISTENER* load_port(SERV_LISTENER const *const *const port) -{ - return (SERV_LISTENER*)atomic_load_ptr((void**)port); -} - SERVICE* service_alloc(const char *name, const char *router) { char *my_name = MXS_STRDUP(name); @@ -513,12 +508,14 @@ int serviceInitialize(SERVICE *service) */ void serviceRemoveListener(SERVICE *service, SERV_LISTENER *target) { - for (SERV_LISTENER *port = load_port(&service->ports); - port; port = load_port(&port->next)) + LISTENER_ITERATOR iter; + + for (SERV_LISTENER *listener = listener_iterator_init(service, &iter); + listener; listener = listener_iterator_next(&iter)) { - if (port == target) + if (listener == target) { - atomic_store_int32(&port->active, 0); + listener_set_active(listener, false); break; } } @@ -546,15 +543,16 @@ bool serviceLaunchListener(SERVICE *service, SERV_LISTENER *port) bool serviceStopListener(SERVICE *service, const char *name) { bool rval = false; + LISTENER_ITERATOR iter; - for (SERV_LISTENER *port = load_port(&service->ports); - port; port = load_port(&port->next)) + for (SERV_LISTENER *listener = listener_iterator_init(service, &iter); + listener; listener = listener_iterator_next(&iter)) { - if (atomic_load_int32(&port->active) && strcmp(port->name, name) == 0) + if (listener_is_active(listener) && strcmp(listener->name, name) == 0) { - if (poll_remove_dcb(port->listener) == 0) + if (poll_remove_dcb(listener->listener) == 0) { - port->listener->session->state = SESSION_STATE_LISTENER_STOPPED; + listener->listener->session->state = SESSION_STATE_LISTENER_STOPPED; rval = true; } break; @@ -567,16 +565,17 @@ bool serviceStopListener(SERVICE *service, const char *name) bool serviceStartListener(SERVICE *service, const char *name) { bool rval = false; + LISTENER_ITERATOR iter; - for (SERV_LISTENER *port = load_port(&service->ports); - port; port = load_port(&port->next)) + for (SERV_LISTENER *listener = listener_iterator_init(service, &iter); + listener; listener = listener_iterator_next(&iter)) { - if (strcmp(port->name, name) == 0) + if (listener_is_active(listener) && strcmp(listener->name, name) == 0) { - if (port->listener && port->listener->session->state == SESSION_STATE_LISTENER_STOPPED && - poll_add_dcb(port->listener) == 0) + if (listener->listener && listener->listener->session->state == SESSION_STATE_LISTENER_STOPPED && + poll_add_dcb(listener->listener) == 0) { - port->listener->session->state = SESSION_STATE_LISTENER; + listener->listener->session->state = SESSION_STATE_LISTENER; rval = true; } break; @@ -616,15 +615,17 @@ bool serviceStop(SERVICE *service) if (service) { - for (SERV_LISTENER *port = load_port(&service->ports); - port; port = load_port(&port->next)) + LISTENER_ITERATOR iter; + + for (SERV_LISTENER *listener = listener_iterator_init(service, &iter); + listener; listener = listener_iterator_next(&iter)) { - if (atomic_load_int32(&port->active) && - port->listener && port->listener->session->state == SESSION_STATE_LISTENER) + if (listener_is_active(listener) && + listener->listener && listener->listener->session->state == SESSION_STATE_LISTENER) { - if (poll_remove_dcb(port->listener) == 0) + if (poll_remove_dcb(listener->listener) == 0) { - port->listener->session->state = SESSION_STATE_LISTENER_STOPPED; + listener->listener->session->state = SESSION_STATE_LISTENER_STOPPED; listeners++; } } @@ -650,15 +651,17 @@ bool serviceStart(SERVICE *service) if (service) { - for (SERV_LISTENER *port = load_port(&service->ports); - port; port = load_port(&port->next)) + LISTENER_ITERATOR iter; + + for (SERV_LISTENER *listener = listener_iterator_init(service, &iter); + listener; listener = listener_iterator_next(&iter)) { - if (atomic_load_int32(&port->active) && - port->listener && port->listener->session->state == SESSION_STATE_LISTENER_STOPPED) + if (listener_is_active(listener) && + listener->listener && listener->listener->session->state == SESSION_STATE_LISTENER_STOPPED) { - if (poll_add_dcb(port->listener) == 0) + if (poll_add_dcb(listener->listener) == 0) { - port->listener->session->state = SESSION_STATE_LISTENER; + listener->listener->session->state = SESSION_STATE_LISTENER; listeners++; } } @@ -715,6 +718,25 @@ void service_free(SERVICE *service) MXS_FREE(service); } +/** + * Add a listener to a service + * + * @param service Service where listener is added + * @param proto Listener to add + */ +static void service_add_listener(SERVICE* service, SERV_LISTENER* proto) +{ + do + { + /** Read the current value of the list's head. This will be our expected + * value for the following compare-and-swap operation. */ + proto->next = (SERV_LISTENER*)atomic_load_ptr((void**)&service->ports); + } + /** Compare the current value to our expected value and if they match, replace + * the current value with our new value. */ + while (!atomic_cas_ptr((void**)&service->ports, (void**)&proto->next, proto)); +} + /** * Create a listener for the service * @@ -736,11 +758,7 @@ SERV_LISTENER* serviceCreateListener(SERVICE *service, const char *name, const c if (proto) { - do - { - proto->next = load_port(&service->ports); - } - while (!atomic_cas_ptr((void**)&service->ports, (void**)&proto->next, proto)); + service_add_listener(service, proto); } return proto; @@ -758,14 +776,15 @@ SERV_LISTENER* serviceCreateListener(SERVICE *service, const char *name, const c bool serviceHasListener(SERVICE *service, const char *protocol, const char* address, unsigned short port) { + LISTENER_ITERATOR iter; - for (SERV_LISTENER *proto = load_port(&service->ports); - proto; proto = load_port(&proto->next)) + for (SERV_LISTENER *listener = listener_iterator_init(service, &iter); + listener; listener = listener_iterator_next(&iter)) { - if (atomic_load_int32(&proto->active) && - strcmp(proto->protocol, protocol) == 0 && proto->port == port && - ((address && proto->address && strcmp(proto->address, address) == 0) || - (address == NULL && proto->address == NULL))) + if (listener_is_active(listener) && + strcmp(listener->protocol, protocol) == 0 && listener->port == port && + ((address && listener->address && strcmp(listener->address, address) == 0) || + (address == NULL && listener->address == NULL))) { return true; } @@ -1525,18 +1544,20 @@ dListListeners(DCB *dcb) } while (service) { - for (SERV_LISTENER *port = load_port(&service->ports); - port; port = load_port(&port->next)) + LISTENER_ITERATOR iter; + + for (SERV_LISTENER *listener = listener_iterator_init(service, &iter); + listener; listener = listener_iterator_next(&iter)) { - if (atomic_load_int32(&port->active)) + if (listener_is_active(listener)) { dcb_printf(dcb, "%-20s | %-19s | %-18s | %-15s | %5d | %s\n", - port->name, service->name, port->protocol, - (port && port->address) ? port->address : "*", - port->port, - (!port->listener || - !port->listener->session || - port->listener->session->state == SESSION_STATE_LISTENER_STOPPED) ? + listener->name, service->name, listener->protocol, + (listener && listener->address) ? listener->address : "*", + listener->port, + (!listener->listener || + !listener->listener->session || + listener->listener->session->state == SESSION_STATE_LISTENER_STOPPED) ? "Stopped" : "Running"); } } @@ -1625,25 +1646,26 @@ int service_refresh_users(SERVICE *service) } ret = 0; + LISTENER_ITERATOR iter; - for (SERV_LISTENER *port = load_port(&service->ports); - port; port = load_port(&port->next)) + for (SERV_LISTENER *listener = listener_iterator_init(service, &iter); + listener; listener = listener_iterator_next(&iter)) { /** Load the authentication users before before starting the listener */ - if (atomic_load_int32(&port->active) && - port->listener && port->listener->authfunc.loadusers) + if (listener_is_active(listener) && listener->listener && + listener->listener->authfunc.loadusers) { - switch (port->listener->authfunc.loadusers(port)) + switch (listener->listener->authfunc.loadusers(listener)) { case MXS_AUTH_LOADUSERS_FATAL: MXS_ERROR("[%s] Fatal error when loading users for listener '%s'," - " authentication will not work.", service->name, port->name); + " authentication will not work.", service->name, listener->name); ret = 1; break; case MXS_AUTH_LOADUSERS_ERROR: MXS_WARNING("[%s] Failed to load users for listener '%s', authentication" - " might not work.", service->name, port->name); + " might not work.", service->name, listener->name); ret = 1; break; @@ -2077,11 +2099,15 @@ bool service_all_services_have_listeners() while (service) { - if (load_port(&service->ports) == NULL) + LISTENER_ITERATOR iter; + SERV_LISTENER *listener = listener_iterator_init(service, &iter); + + if (listener == NULL) { MXS_ERROR("Service '%s' has no listeners.", service->name); rval = false; } + service = service->next; } @@ -2332,13 +2358,15 @@ bool service_serialize_servers(const SERVICE *service) void service_print_users(DCB *dcb, const SERVICE *service) { - for (SERV_LISTENER *port = load_port(&service->ports); - port; port = load_port(&port->next)) + LISTENER_ITERATOR iter; + + for (SERV_LISTENER *listener = listener_iterator_init(service, &iter); + listener; listener = listener_iterator_next(&iter)) { - if (atomic_load_int32(&port->active) && - port->listener && port->listener->authfunc.diagnostic) + if (listener_is_active(listener) && listener->listener && + listener->listener->authfunc.diagnostic) { - port->listener->authfunc.diagnostic(dcb, port); + listener->listener->authfunc.diagnostic(dcb, listener); } } } @@ -2350,10 +2378,12 @@ bool service_port_is_used(unsigned short port) for (SERVICE *service = allServices; service && !rval; service = service->next) { - for (SERV_LISTENER *proto = load_port(&service->ports); - proto; proto = load_port(&proto->next)) + LISTENER_ITERATOR iter; + + for (SERV_LISTENER *listener = listener_iterator_init(service, &iter); + listener; listener = listener_iterator_next(&iter)) { - if (atomic_load_int32(&proto->active) && proto->port == port) + if (listener_is_active(listener) && listener->port == port) { rval = true; break; @@ -2452,13 +2482,14 @@ static inline bool have_active_servers(const SERVICE* service) json_t* service_listeners_json_data(const SERVICE* service) { json_t* arr = json_array(); + LISTENER_ITERATOR iter; - for (SERV_LISTENER *port = load_port(&service->ports); - port; port = load_port(&port->next)) + for (SERV_LISTENER *listener = listener_iterator_init(service, &iter); + listener; listener = listener_iterator_next(&iter)) { - if (atomic_load_int32(&port->active)) + if (listener_is_active(listener)) { - json_array_append_new(arr, listener_to_json(port)); + json_array_append_new(arr, listener_to_json(listener)); } } From 9fee283a3f3a8c941d6d552ab26d233348a9d3c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 9 May 2017 08:50:26 +0300 Subject: [PATCH 41/55] MXS-1220: Add /maxscale/tasks resource The resource shows all active tasks inside MaxScale. --- include/maxscale/housekeeper.h | 60 +++++++++++++++-- server/core/housekeeper.cc | 116 ++++++++++++++------------------- server/core/resource.cc | 4 +- 3 files changed, 107 insertions(+), 73 deletions(-) diff --git a/include/maxscale/housekeeper.h b/include/maxscale/housekeeper.h index f6396763e..bfd11be17 100644 --- a/include/maxscale/housekeeper.h +++ b/include/maxscale/housekeeper.h @@ -68,9 +68,61 @@ extern void hkshutdown(); */ extern void hkfinish(); -extern int hktask_add(const char *name, void (*task)(void *), void *data, int frequency); -extern int hktask_oneshot(const char *name, void (*task)(void *), void *data, int when); -extern int hktask_remove(const char *name); -extern void hkshow_tasks(DCB *pdcb); +/** + * @brief Add a new task + * + * The task will be first run @c frequency seconds after this call is + * made and will the be executed repeatedly every frequency seconds + * until the task is removed. + * + * Task names must be unique. + * + * @param name Task name + * @param task Function to execute + * @param data Data passed to function as the parameter + * @param frequency Frequency of execution + * + * @return 1 if task was added + */ +int hktask_add(const char *name, void (*task)(void *) , void *data, int frequency); + +/** + * @brief Add oneshot task + * + * The task will only execute once. + * + * @param name Task name + * @param task Function to execute + * @param data Data passed to function as the parameter + * @param when Number of seconds to wait until task is executed + * + * @return 1 if task was added + */ +int hktask_oneshot(const char *name, void (*task)(void *) , void *data, int when); + +/** + * @brief Remove a task + * + * @param name Task name + * + * @return 1 if the task was removed + */ +int hktask_remove(const char *name); + +/** + * @brief Show the tasks that are scheduled for the house keeper + * + * @param pdcb The DCB to send to output + */ +void hkshow_tasks(DCB *pdcb); + +/** + * @brief Show tasks as JSON resource + * + * @param host Hostname of this server + * + * @return Collection of JSON formatted task resources + */ +json_t* hk_tasks_json(const char* host); MXS_END_DECLS diff --git a/server/core/housekeeper.cc b/server/core/housekeeper.cc index d8425e333..092e6e494 100644 --- a/server/core/housekeeper.cc +++ b/server/core/housekeeper.cc @@ -11,34 +11,29 @@ * Public License. */ #include + #include #include + #include #include +#include #include #include #include +#include /** * @file housekeeper.c Provide a mechanism to run periodic tasks * * The housekeeper provides a mechanism to allow for tasks, function - * calls basically, to be run on a tiem basis. A task may be run + * calls basically, to be run on a time basis. A task may be run * repeatedly, with a given frequency (in seconds), or may be a one * shot task that will only be run once after a specified number of * seconds. * * The housekeeper also maintains a global variable, hkheartbeat, that * is incremented every 100ms. - * - * @verbatim - * Revision History - * - * Date Who Description - * 29/08/14 Mark Riddoch Initial implementation - * 22/10/14 Mark Riddoch Addition of one-shot tasks - * - * @endverbatim */ /** @@ -57,8 +52,7 @@ static THREAD hk_thr_handle; static void hkthread(void *); -bool -hkinit() +bool hkinit() { bool inited = false; @@ -74,25 +68,7 @@ hkinit() return inited; } -/** - * Add a new task to the housekeepers lists of tasks that should be - * run periodically. - * - * The task will be first run frequency seconds after this call is - * made and will the be executed repeatedly every frequency seconds - * until the task is removed. - * - * Task names must be unique. - * - * @param name The unique name for this housekeeper task - * @param taskfn The function to call for the task - * @param data Data to pass to the task function - * @param frequency How often to run the task, expressed in seconds - * @return Return the time in seconds when the task will be first run - * if the task was added, otherwise 0 - */ -int -hktask_add(const char *name, void (*taskfn)(void *), void *data, int frequency) +int hktask_add(const char *name, void (*taskfn)(void *), void *data, int frequency) { HKTASK *task, *ptr; @@ -144,21 +120,7 @@ hktask_add(const char *name, void (*taskfn)(void *), void *data, int frequency) return task->nextdue; } -/** - * Add a one-shot task to the housekeeper task list - * - * Task names must be unique. - * - * @param name The unique name for this housekeeper task - * @param taskfn The function to call for the task - * @param data Data to pass to the task function - * @param when How many second until the task is executed - * @return Return the time in seconds when the task will be first run - * if the task was added, otherwise 0 - * - */ -int -hktask_oneshot(const char *name, void (*taskfn)(void *), void *data, int when) +int hktask_oneshot(const char *name, void (*taskfn)(void *), void *data, int when) { HKTASK *task, *ptr; @@ -196,15 +158,7 @@ hktask_oneshot(const char *name, void (*taskfn)(void *), void *data, int when) return task->nextdue; } - -/** - * Remove a named task from the housekeepers task list - * - * @param name The task name to remove - * @return Returns 0 if the task could not be removed - */ -int -hktask_remove(const char *name) +int hktask_remove(const char *name) { HKTASK *ptr, *lptr = NULL; @@ -237,7 +191,6 @@ hktask_remove(const char *name) } } - /** * The housekeeper thread implementation. * @@ -253,8 +206,7 @@ hktask_remove(const char *name) * * @param data Unused, here to satisfy the thread system */ -void -hkthread(void *data) +void hkthread(void *data) { HKTASK *ptr; time_t now; @@ -304,8 +256,7 @@ hkthread(void *data) MXS_NOTICE("Housekeeper shutting down."); } -void -hkshutdown() +void hkshutdown() { do_shutdown = true; atomic_synchronize(); @@ -321,13 +272,7 @@ void hkfinish() MXS_NOTICE("Housekeeper has shut down."); } -/** - * Show the tasks that are scheduled for the house keeper - * - * @param pdcb The DCB to send to output - */ -void -hkshow_tasks(DCB *pdcb) +void hkshow_tasks(DCB *pdcb) { HKTASK *ptr; struct tm tm; @@ -350,3 +295,40 @@ hkshow_tasks(DCB *pdcb) } spinlock_release(&tasklock); } + +json_t* hk_tasks_json(const char* host) +{ + json_t* arr = json_array(); + + spinlock_acquire(&tasklock); + + for (HKTASK* ptr = tasks; ptr; ptr = ptr->next) + { + struct tm tm; + char buf[40]; + localtime_r(&ptr->nextdue, &tm); + asctime_r(&tm, buf); + char* nl = strchr(buf, '\n'); + ss_dassert(nl); + *nl = '\0'; + + const char* task_type = ptr->type == HK_REPEATED ? "Repeated" : "One-Shot"; + + json_t* obj = json_object(); + + json_object_set_new(obj, CN_ID, json_string(ptr->name)); + json_object_set_new(obj, CN_TYPE, json_string("tasks")); + + json_t* attr = json_object(); + json_object_set_new(attr, "task_type", json_string(task_type)); + json_object_set_new(attr, "frequency", json_integer(ptr->frequency)); + json_object_set_new(attr, "next_execution", json_string(buf)); + + json_object_set_new(obj, CN_ATTRIBUTES, attr); + json_array_append_new(arr, obj); + } + + spinlock_release(&tasklock); + + return mxs_json_resource(host, MXS_JSON_API_TASKS, arr); +} diff --git a/server/core/resource.cc b/server/core/resource.cc index d8cc88302..1f78f5750 100644 --- a/server/core/resource.cc +++ b/server/core/resource.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include "maxscale/httprequest.hh" #include "maxscale/httpresponse.hh" @@ -467,8 +468,7 @@ HttpResponse cb_thread(const HttpRequest& request) HttpResponse cb_tasks(const HttpRequest& request) { - // TODO: Show housekeeper tasks - return HttpResponse(MHD_HTTP_OK, mxs_json_resource(request.host(), MXS_JSON_API_TASKS, json_null())); + return HttpResponse(MHD_HTTP_OK, hk_tasks_json(request.host())); } HttpResponse cb_all_modules(const HttpRequest& request) From 157b7f853be84192dcec62eb1bacf857dc7faddf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 9 May 2017 09:32:21 +0300 Subject: [PATCH 42/55] MXS-1220: Fix resource self links The top level resource self links pointed to the collection instead of the resource itself. The individual resoures now also have a links field that contains the self link to the resource. This should make navigation of the API easier as all objects have valid links in them. --- include/maxscale/json_api.h | 13 ++++++ server/core/filter.cc | 5 ++- server/core/json_api.cc | 14 ++++++ server/core/load_utils.cc | 1 + server/core/monitor.cc | 5 ++- server/core/server.cc | 16 ++----- server/core/service.cc | 5 ++- server/core/session.cc | 9 ++-- .../test/rest-api/test/schema_validation.js | 44 +++++++++++++++++++ server/core/worker.cc | 5 ++- 10 files changed, 98 insertions(+), 19 deletions(-) diff --git a/include/maxscale/json_api.h b/include/maxscale/json_api.h index 35c4fcc6f..5f7bc5865 100644 --- a/include/maxscale/json_api.h +++ b/include/maxscale/json_api.h @@ -65,6 +65,19 @@ json_t* mxs_json_relationship(const char* host, const char* endpoint); */ void mxs_json_add_relation(json_t* rel, const char* id, const char* type); +/** + * @brief Create self link object + * + * The self link points to the object itself. + * + * @param host Hostname of this server + * @param path Base path to the resource collection + * @param id The identified of this resource + * + * @return New self link object + */ +json_t* mxs_json_self_link(const char* host, const char* path, const char* id); + /** * @brief Return value at provided JSON Pointer * diff --git a/server/core/filter.cc b/server/core/filter.cc index 8ef97ff51..845ca7d11 100644 --- a/server/core/filter.cc +++ b/server/core/filter.cc @@ -526,13 +526,16 @@ json_t* filter_json_data(const MXS_FILTER_DEF* filter, const char* host) json_object_set_new(rval, CN_RELATIONSHIPS, rel); json_object_set_new(rval, CN_ATTRIBUTES, attr); + json_object_set_new(rval, CN_LINKS, mxs_json_self_link(host, CN_FILTERS, filter->name)); return rval; } json_t* filter_to_json(const MXS_FILTER_DEF* filter, const char* host) { - return mxs_json_resource(host, MXS_JSON_API_FILTERS, filter_json_data(filter, host)); + string self = MXS_JSON_API_FILTERS; + self += filter->name; + return mxs_json_resource(host, self.c_str(), filter_json_data(filter, host)); } json_t* filter_list_to_json(const char* host) diff --git a/server/core/json_api.cc b/server/core/json_api.cc index de4688b38..21f1a6ded 100644 --- a/server/core/json_api.cc +++ b/server/core/json_api.cc @@ -131,3 +131,17 @@ json_t* mxs_json_pointer(json_t* json, const char* json_ptr) { return mxs_json_pointer_internal(json, json_ptr); } + +json_t* mxs_json_self_link(const char* host, const char* path, const char* id) +{ + json_t* links = json_object(); + + string self = host; + self += "/"; + self += path; + self += "/"; + self += id; + json_object_set_new(links, CN_SELF, json_string(self.c_str())); + + return links; +} diff --git a/server/core/load_utils.cc b/server/core/load_utils.cc index c995dfb54..c5ace36f1 100644 --- a/server/core/load_utils.cc +++ b/server/core/load_utils.cc @@ -453,6 +453,7 @@ static json_t* module_json_data(const LOADED_MODULE *mod, const char* host) json_object_set_new(attr, CN_PARAMETERS, params); json_object_set_new(obj, CN_ATTRIBUTES, attr); + json_object_set_new(obj, CN_LINKS, mxs_json_self_link(host, CN_MODULES, mod->module)); return obj; } diff --git a/server/core/monitor.cc b/server/core/monitor.cc index d67387a84..7e35a0c99 100644 --- a/server/core/monitor.cc +++ b/server/core/monitor.cc @@ -1544,13 +1544,16 @@ json_t* monitor_json_data(const MXS_MONITOR* monitor, const char* host) json_object_set_new(rval, CN_RELATIONSHIPS, rel); json_object_set_new(rval, CN_ATTRIBUTES, attr); + json_object_set_new(rval, CN_LINKS, mxs_json_self_link(host, CN_MONITORS, monitor->name)); return rval; } json_t* monitor_to_json(const MXS_MONITOR* monitor, const char* host) { - return mxs_json_resource(host, MXS_JSON_API_MONITORS, monitor_json_data(monitor, host)); + string self = MXS_JSON_API_MONITORS; + self += monitor->name; + return mxs_json_resource(host, self.c_str(), monitor_json_data(monitor, host)); } json_t* monitor_list_to_json(const char* host) diff --git a/server/core/server.cc b/server/core/server.cc index 08eaa7d37..8d5aa164e 100644 --- a/server/core/server.cc +++ b/server/core/server.cc @@ -1361,16 +1361,6 @@ bool server_is_mxs_service(const SERVER *server) return rval; } -json_t* server_self_link(const char* host) -{ - json_t* links = json_object(); - string self = host; - self += "/servers/"; - json_object_set_new(links, CN_SELF, json_string(self.c_str())); - - return links; -} - static json_t* server_json_attributes(const SERVER* server) { /** Resource attributes */ @@ -1468,16 +1458,18 @@ static json_t* server_to_json_data(const SERVER* server, const char* host) json_object_set_new(rel, CN_SERVICES, service_relations_to_server(server, host)); json_object_set_new(rel, CN_MONITORS, monitor_relations_to_server(server, host)); json_object_set_new(rval, CN_RELATIONSHIPS, rel); - /** Attributes */ json_object_set_new(rval, CN_ATTRIBUTES, server_json_attributes(server)); + json_object_set_new(rval, CN_LINKS, mxs_json_self_link(host, CN_SERVERS, server->unique_name)); return rval; } json_t* server_to_json(const SERVER* server, const char* host) { - return mxs_json_resource(host, MXS_JSON_API_SERVERS, server_to_json_data(server, host)); + string self = MXS_JSON_API_SERVERS; + self += server->unique_name; + return mxs_json_resource(host, self.c_str(), server_to_json_data(server, host)); } json_t* server_list_to_json(const char* host) diff --git a/server/core/service.cc b/server/core/service.cc index 2df6b9c0b..237f256c8 100644 --- a/server/core/service.cc +++ b/server/core/service.cc @@ -2575,6 +2575,7 @@ json_t* service_json_data(const SERVICE* service, const char* host) json_object_set_new(rval, CN_TYPE, json_string(CN_SERVICES)); json_object_set_new(rval, CN_ATTRIBUTES, service_attributes(service)); json_object_set_new(rval, CN_RELATIONSHIPS, service_relationships(service, host)); + json_object_set_new(rval, CN_LINKS, mxs_json_self_link(host, CN_SERVICES, service->name)); spinlock_release(&service->spin); @@ -2583,7 +2584,9 @@ json_t* service_json_data(const SERVICE* service, const char* host) json_t* service_to_json(const SERVICE* service, const char* host) { - return mxs_json_resource(host, MXS_JSON_API_SERVICES, service_json_data(service, host)); + string self = MXS_JSON_API_SERVICES; + self += service->name; + return mxs_json_resource(host, self.c_str(), service_json_data(service, host)); } json_t* service_listeners_to_json(const SERVICE* service, const char* host) diff --git a/server/core/session.cc b/server/core/session.cc index 6d81247f6..0ed98a34b 100644 --- a/server/core/session.cc +++ b/server/core/session.cc @@ -1118,7 +1118,7 @@ json_t* session_json_data(const MXS_SESSION *session, const char *host) if (session->client_dcb->user) { - json_object_set_new(attr, "user", json_string(session->client_dcb->user)); + json_object_set_new(attr, CN_USER, json_string(session->client_dcb->user)); } if (session->client_dcb->remote) @@ -1141,14 +1141,17 @@ json_t* session_json_data(const MXS_SESSION *session, const char *host) json_object_set_new(attr, "idle", json_real(idle)); } - json_object_set_new(data, "attributes", attr); + json_object_set_new(data, CN_ATTRIBUTES, attr); + json_object_set_new(data, CN_LINKS, mxs_json_self_link(host, CN_SESSIONS, ss.str().c_str())); return data; } json_t* session_to_json(const MXS_SESSION *session, const char *host) { - return mxs_json_resource(host, MXS_JSON_API_SESSIONS, session_json_data(session, host)); + stringstream ss; + ss << MXS_JSON_API_SESSIONS << session->ses_id; + return mxs_json_resource(host, ss.str().c_str(), session_json_data(session, host)); } struct SessionListData diff --git a/server/core/test/rest-api/test/schema_validation.js b/server/core/test/rest-api/test/schema_validation.js index 752747fb7..ba8ed0091 100644 --- a/server/core/test/rest-api/test/schema_validation.js +++ b/server/core/test/rest-api/test/schema_validation.js @@ -62,3 +62,47 @@ describe("Individual Resources", function() { after(stopMaxScale) }); + +describe("Resource Self Links", function() { + before(startMaxScale) + + var tests = [ + "/servers", + "/sessions", + "/services", + "/monitors", + "/filters", + "/maxscale/threads", + "/maxscale/modules", + "/maxscale/tasks", + "/servers/server1", + "/servers/server2", + "/services/RW-Split-Router", + "/services/RW-Split-Router/listeners", + "/monitors/MySQL-Monitor", + "/filters/Hint", + "/sessions/1", + "/maxscale/", + "/maxscale/threads/0", + "/maxscale/logs", + "/maxscale/modules/readwritesplit", + ] + + tests.forEach(function(endpoint) { + it(endpoint + ': correct self link', function() { + var obj = null; + return request.get(base_url + endpoint) + .then(function(resp) { + obj = JSON.parse(resp) + return request.get(obj.links.self) + }) + .then(function(resp) { + var obj_self = JSON.parse(resp) + obj_self.links.self.should.be.equal(obj.links.self) + }) + .should.be.fulfilled + }); + }) + + after(stopMaxScale) +}); diff --git a/server/core/worker.cc b/server/core/worker.cc index e0ed65819..1c8840103 100644 --- a/server/core/worker.cc +++ b/server/core/worker.cc @@ -814,6 +814,7 @@ public: json_object_set_new(json, CN_ID, json_string(ss.str().c_str())); json_object_set_new(json, CN_TYPE, json_string(CN_THREADS)); json_object_set_new(json, CN_ATTRIBUTES, attr); + json_object_set_new(json, CN_LINKS, mxs_json_self_link(m_host, CN_THREADS, ss.str().c_str())); ss_dassert((size_t)idx < m_data.size()); m_data[idx] = json; @@ -833,7 +834,9 @@ public: json_t* resource(int id) { - return mxs_json_resource(m_host, MXS_JSON_API_THREADS, m_data[id]); + stringstream self; + self << MXS_JSON_API_THREADS << id; + return mxs_json_resource(m_host, self.str().c_str(), m_data[id]); } private: From 370cff35288d4bfd3db24ab02c171bbe485b63ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 9 May 2017 16:22:55 +0300 Subject: [PATCH 43/55] Remove dependencies on FLEX/BISON targets The FLEX and BISON targets don't seem to be actual CMake targets which cause warnings to be emitted. --- server/modules/filter/dbfwfilter/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/modules/filter/dbfwfilter/CMakeLists.txt b/server/modules/filter/dbfwfilter/CMakeLists.txt index 93c08c498..8f03ad2b0 100644 --- a/server/modules/filter/dbfwfilter/CMakeLists.txt +++ b/server/modules/filter/dbfwfilter/CMakeLists.txt @@ -10,13 +10,11 @@ if(BISON_FOUND AND FLEX_FOUND) target_link_libraries(dbfwfilter maxscale-common) set_target_properties(dbfwfilter PROPERTIES VERSION "1.0.0") install_module(dbfwfilter core) - add_dependencies(dbfwfilter token ruleparser) # The offline rule check utility add_executable(dbfwchk dbfw_rule_check.c ${BISON_ruleparser_OUTPUTS} ${FLEX_token_OUTPUTS}) target_link_libraries(dbfwchk maxscale-common) install_executable(dbfwchk core) - add_dependencies(dbfwchk token ruleparser) else() message(FATAL_ERROR "Could not find Bison or Flex: ${BISON_EXECUTABLE} ${FLEX_EXECUTABLE}") From fb5ca5256547ce31f41e2accf4566cf0f0ff9503 Mon Sep 17 00:00:00 2001 From: Dong Young Yoon Date: Tue, 10 Jan 2017 17:15:53 -0500 Subject: [PATCH 44/55] TPMfilter now prints latencies of individual statements. --- server/modules/filter/tpmfilter/tpmfilter.c | 39 +++++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/server/modules/filter/tpmfilter/tpmfilter.c b/server/modules/filter/tpmfilter/tpmfilter.c index b2eec23cb..739929c5e 100644 --- a/server/modules/filter/tpmfilter/tpmfilter.c +++ b/server/modules/filter/tpmfilter/tpmfilter.c @@ -66,7 +66,8 @@ /* The maximum size for query statements in a transaction (64MB) */ static size_t sql_size_limit = 64 * 1024 * 1024; - +/* The size of the buffer for recording latency of individual statements */ +static size_t latency_buf_size = 64 * 1024; static const int default_sql_size = 4 * 1024; #define DEFAULT_QUERY_DELIMITER "@@@" @@ -125,14 +126,17 @@ typedef struct char *clientHost; char *userName; char* sql; + char* latency; struct timeval start; char *current; int n_statements; struct timeval total; struct timeval current_start; + struct timeval last_statement_start; bool query_end; char *buf; int sql_index; + int latency_index; size_t max_sql_size; } TPM_SESSION; @@ -310,10 +314,12 @@ newSession(MXS_FILTER *instance, MXS_SESSION *session) { atomic_add(&my_instance->sessions, 1); + my_session->latency = (char*)MXS_CALLOC(latency_buf_size, sizeof(char)); my_session->max_sql_size = default_sql_size; // default max query size of 4k. my_session->sql = (char*)MXS_CALLOC(my_session->max_sql_size, sizeof(char)); memset(my_session->sql, 0x00, my_session->max_sql_size); my_session->sql_index = 0; + my_session->latency_index = 0; my_session->n_statements = 0; my_session->total.tv_sec = 0; my_session->total.tv_usec = 0; @@ -384,6 +390,7 @@ freeSession(MXS_FILTER *instance, MXS_FILTER_SESSION *session) MXS_FREE(my_session->clientHost); MXS_FREE(my_session->userName); MXS_FREE(my_session->sql); + MXS_FREE(my_session->latency); MXS_FREE(session); return; } @@ -508,6 +515,7 @@ routeQuery(MXS_FILTER *instance, MXS_FILTER_SESSION *session, GWBUF *queue) /* set new pointer for the buffer */ my_session->sql_index += (my_instance->query_delimiter_size + strlen(ptr)); } + gettimeofday(&my_session->last_statement_start, NULL); } } } @@ -528,6 +536,28 @@ clientReply(MXS_FILTER *instance, MXS_FILTER_SESSION *session, GWBUF *reply) struct timeval tv, diff; int i, inserted; + /* records latency of the SQL statement. */ + if (my_session->sql_index > 0) + { + gettimeofday(&tv, NULL); + timersub(&tv, &(my_session->last_statement_start), &diff); + + /* get latency */ + double millis = (diff.tv_sec * 1000 + diff.tv_usec / 1000.0); + + int written = sprintf(my_session->latency + my_session->latency_index, "%.3f", millis); + my_session->latency_index += written; + if (!my_session->query_end) + { + written = sprintf(my_session->latency + my_session->latency_index, "%s", my_instance->query_delimiter); + my_session->latency_index += written; + } + if (my_session->latency_index > latency_buf_size) + { + MXS_ERROR("Latency buffer overflow."); + } + } + /* found 'commit' and sql statements exist. */ if (my_session->query_end && my_session->sql_index > 0) { @@ -544,8 +574,8 @@ clientReply(MXS_FILTER *instance, MXS_FILTER_SESSION *session, GWBUF *reply) /* print to log. */ if (my_instance->log_enabled) { - /* this prints "timestamp | server_name | user_name | latency | sql_statements" */ - fprintf(my_instance->fp, "%ld%s%s%s%s%s%ld%s%s\n", + /* this prints "timestamp | server_name | user_name | latency of entire transaction | latencies of individual statements | sql_statements" */ + fprintf(my_instance->fp, "%ld%s%s%s%s%s%ld%s%s%s%s\n", timestamp, my_instance->delimiter, reply->server->unique_name, @@ -554,10 +584,13 @@ clientReply(MXS_FILTER *instance, MXS_FILTER_SESSION *session, GWBUF *reply) my_instance->delimiter, millis, my_instance->delimiter, + my_session->latency, + my_instance->delimiter, my_session->sql); } my_session->sql_index = 0; + my_session->latency_index = 0; } /* Pass the result upstream */ From 0ddc876567f3c4fe5fcb14d00f4053adf0b363ff Mon Sep 17 00:00:00 2001 From: Dong Young Yoon Date: Tue, 10 Jan 2017 17:28:20 -0500 Subject: [PATCH 45/55] Updated the documentation for TPM filter. --- .../Transaction-Performance-Monitoring-Filter.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Documentation/Filters/Transaction-Performance-Monitoring-Filter.md b/Documentation/Filters/Transaction-Performance-Monitoring-Filter.md index 21070c40d..a49602180 100644 --- a/Documentation/Filters/Transaction-Performance-Monitoring-Filter.md +++ b/Documentation/Filters/Transaction-Performance-Monitoring-Filter.md @@ -104,6 +104,11 @@ Similarly, the following command disables the logging: $ echo '0' > /tmp/tpmfilter +## Log Output Format + +For each transaction, the TPM filter prints its log in the following format: + +\ | \ | \ | \ | \ (delimited by 'query_delimiter') | \ ## Examples @@ -137,9 +142,10 @@ is an example log that is generated from the above TPM filter with the above con ``` -1453751768:::server1:::root:::3:::UPDATE WAREHOUSE SET W_YTD = W_YTD + 900.86 WHERE W_ID = 2 @@@SELECT W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP, W_NAME FROM WAREHOUSE WHERE W_ID = 2@@@UPDATE DISTRICT SET D_YTD = D_YTD + 900.86 WHERE D_W_ID = 2 AND D_ID = 5@@@SELECT D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP, D_NAME FROM DISTRICT WHERE D_W_ID = 2 AND D_ID = 5@@@SELECT C_FIRST, C_MIDDLE, C_ID, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_SINCE FROM CUSTOMER WHERE C_W_ID = 2 AND C_D_ID = 5 AND C_LAST = 'CALLYCALLYATION' ORDER BY C_FIRST@@@UPDATE CUSTOMER SET C_BALANCE = -90026.89, C_YTD_PAYMENT = 93507.06, C_PAYMENT_CNT = 38 WHERE C_W_ID = 2 AND C_D_ID = 5 AND C_ID = 779@@@INSERT INTO HISTORY (H_C_D_ID, H_C_W_ID, H_C_ID, H_D_ID, H_W_ID, H_DATE, H_AMOUNT, H_DATA) VALUES (5,2,779,5,2,'2016-01-25 14:56:08',900.86,'gqfla adopdon') -1453751768:::server1:::root:::5:::UPDATE WAREHOUSE SET W_YTD = W_YTD + 3679.75 WHERE W_ID = 2 @@@SELECT W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP, W_NAME FROM WAREHOUSE WHERE W_ID = 2@@@UPDATE DISTRICT SET D_YTD = D_YTD + 3679.75 WHERE D_W_ID = 2 AND D_ID = 1@@@SELECT D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP, D_NAME FROM DISTRICT WHERE D_W_ID = 2 AND D_ID = 1@@@SELECT C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_SINCE FROM CUSTOMER WHERE C_W_ID = 2 AND C_D_ID = 1 AND C_ID = 203@@@UPDATE CUSTOMER SET C_BALANCE = 1600482.5, C_YTD_PAYMENT = 1192789.8, C_PAYMENT_CNT = 485 WHERE C_W_ID = 2 AND C_D_ID = 1 AND C_ID = 203@@@INSERT INTO HISTORY (H_C_D_ID, H_C_W_ID, H_C_ID, H_D_ID, H_W_ID, H_DATE, H_AMOUNT, H_DATA) VALUES (1,2,203,1,2,'2016-01-25 14:56:08',3679.75,'gqfla uquslfu') +1484086477::::server1::::root::::3::::0.165@@@@0.108@@@@0.102@@@@0.092@@@@0.121@@@@0.122@@@@0.110@@@@2.081::::UPDATE WAREHOUSE SET W_YTD = W_YTD + 3630.48 WHERE W_ID = 2 @@@@SELECT W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP, W_NAME FROM WAREHOUSE WHERE W_ID = 2@@@@UPDATE DISTRICT SET D_YTD = D_YTD + 3630.48 WHERE D_W_ID = 2 AND D_ID = 9@@@@SELECT D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP, D_NAME FROM DISTRICT WHERE D_W_ID = 2 AND D_ID = 9@@@@SELECT C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_SINCE FROM CUSTOMER WHERE C_W_ID = 2 AND C_D_ID = 9 AND C_ID = 1025@@@@UPDATE CUSTOMER SET C_BALANCE = 1007749.25, C_YTD_PAYMENT = 465215.47, C_PAYMENT_CNT = 203 WHERE C_W_ID = 2 AND C_D_ID = 9 AND C_ID = 1025@@@@INSERT INTO HISTORY (H_C_D_ID, H_C_W_ID, H_C_ID, H_D_ID, H_W_ID, H_DATE, H_AMOUNT, H_DATA) VALUES (9,2,1025,9,2,'2017-01-10 17:14:37',3630.48,'locfljbe xtnfqn') +1484086477::::server1::::root::::6::::0.123@@@@0.087@@@@0.091@@@@0.098@@@@0.078@@@@0.106@@@@0.094@@@@0.074@@@@0.089@@@@0.073@@@@0.098@@@@0.073@@@@0.088@@@@0.072@@@@0.087@@@@0.071@@@@0.085@@@@0.078@@@@0.088@@@@0.098@@@@0.081@@@@0.076@@@@0.082@@@@0.073@@@@0.077@@@@0.070@@@@0.105@@@@0.093@@@@0.088@@@@0.089@@@@0.087@@@@0.087@@@@0.086@@@@1.883::::SELECT C_DISCOUNT, C_LAST, C_CREDIT, W_TAX FROM CUSTOMER, WAREHOUSE WHERE W_ID = 2 AND C_W_ID = 2 AND C_D_ID = 10 AND C_ID = 1267@@@@SELECT D_NEXT_O_ID, D_TAX FROM DISTRICT WHERE D_W_ID = 2 AND D_ID = 10 FOR UPDATE@@@@UPDATE DISTRICT SET D_NEXT_O_ID = D_NEXT_O_ID + 1 WHERE D_W_ID = 2 AND D_ID = 10@@@@INSERT INTO OORDER (O_ID, O_D_ID, O_W_ID, O_C_ID, O_ENTRY_D, O_OL_CNT, O_ALL_LOCAL) VALUES (286871, 10, 2, 1267, '2017-01-10 17:14:37', 7, 1)@@@@INSERT INTO NEW_ORDER (NO_O_ID, NO_D_ID, NO_W_ID) VALUES ( 286871, 10, 2)@@@@SELECT I_PRICE, I_NAME , I_DATA FROM ITEM WHERE I_ID = 24167@@@@SELECT S_QUANTITY, S_DATA, S_DIST_01, S_DIST_02, S_DIST_03, S_DIST_04, S_DIST_05, S_DIST_06, S_DIST_07, S_DIST_08, S_DIST_09, S_DIST_10 FROM STOCK WHERE S_I_ID = 24167 AND S_W_ID = 2 FOR UPDATE@@@@SELECT I_PRICE, I_NAME , I_DATA FROM ITEM WHERE I_ID = 96982@@@@SELECT S_QUANTITY, S_DATA, S_DIST_01, S_DIST_02, S_DIST_03, S_DIST_04, S_DIST_05, S_DIST_06, S_DIST_07, S_DIST_08, S_DIST_09, S_DIST_10 FROM STOCK WHERE S_I_ID = 96982 AND S_W_ID = 2 FOR UPDATE@@@@SELECT I_PRICE, I_NAME , I_DATA FROM ITEM WHERE I_ID = 40679@@@@SELECT S_QUANTITY, S_DATA, S_DIST_01, S_DIST_02, S_DIST_03, S_DIST_04, S_DIST_05, S_DIST_06, S_DIST_07, S_DIST_08, S_DIST_09, S_DIST_10 FROM STOCK WHERE S_I_ID = 40679 AND S_W_ID = 2 FOR UPDATE@@@@SELECT I_PRICE, I_NAME , I_DATA FROM ITEM WHERE I_ID = 31459@@@@SELECT S_QUANTITY, S_DATA, S_DIST_01, S_DIST_02, S_DIST_03, S_DIST_04, S_DIST_05, S_DIST_06, S_DIST_07, S_DIST_08, S_DIST_09, S_DIST_10 FROM STOCK WHERE S_I_ID = 31459 AND S_W_ID = 2 FOR UPDATE@@@@SELECT I_PRICE, I_NAME , I_DATA FROM ITEM WHERE I_ID = 6143@@@@SELECT S_QUANTITY, S_DATA, S_DIST_01, S_DIST_02, S_DIST_03, S_DIST_04, S_DIST_05, S_DIST_06, S_DIST_07, S_DIST_08, S_DIST_09, S_DIST_10 FROM STOCK WHERE S_I_ID = 6143 AND S_W_ID = 2 FOR UPDATE@@@@SELECT I_PRICE, I_NAME , I_DATA FROM ITEM WHERE I_ID = 12001@@@@SELECT S_QUANTITY, S_DATA, S_DIST_01, S_DIST_02, S_DIST_03, S_DIST_04, S_DIST_05, S_DIST_06, S_DIST_07, S_DIST_08, S_DIST_09, S_DIST_10 FROM STOCK WHERE S_I_ID = 12001 AND S_W_ID = 2 FOR UPDATE@@@@SELECT I_PRICE, I_NAME , I_DATA FROM ITEM WHERE I_ID = 40407@@@@SELECT S_QUANTITY, S_DATA, S_DIST_01, S_DIST_02, S_DIST_03, S_DIST_04, S_DIST_05, S_DIST_06, S_DIST_07, S_DIST_08, S_DIST_09, S_DIST_10 FROM STOCK WHERE S_I_ID = 40407 AND S_W_ID = 2 FOR UPDATE@@@@INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, OL_QUANTITY, OL_AMOUNT, OL_DIST_INFO) VALUES (286871,10,2,1,24167,2,7,348.31998,'btdyjesowlpzjwnmxdcsion')@@@@INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, OL_QUANTITY, OL_AMOUNT, OL_DIST_INFO) VALUES (286871,10,2,2,96982,2,1,4.46,'kudpnktydxbrbxibbsyvdiw')@@@@INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, OL_QUANTITY, OL_AMOUNT, OL_DIST_INFO) VALUES (286871,10,2,3,40679,2,7,528.43,'nhcixumgmosxlwgabvsrcnu')@@@@INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, OL_QUANTITY, OL_AMOUNT, OL_DIST_INFO) VALUES (286871,10,2,4,31459,2,9,341.82,'qbglbdleljyfzdpfbyziiea')@@@@INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, OL_QUANTITY, OL_AMOUNT, OL_DIST_INFO) VALUES (286871,10,2,5,6143,2,3,152.67,'tmtnuupaviimdmnvmetmcrc')@@@@INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, OL_QUANTITY, OL_AMOUNT, OL_DIST_INFO) VALUES (286871,10,2,6,12001,2,5,304.3,'ufytqwvkqxtmalhenrssfon')@@@@INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, OL_QUANTITY, OL_AMOUNT, OL_DIST_INFO) VALUES (286871,10,2,7,40407,2,1,30.32,'hvclpfnblxchbyluumetcqn')@@@@UPDATE STOCK SET S_QUANTITY = 65 , S_YTD = S_YTD + 7, S_ORDER_CNT = S_ORDER_CNT + 1, S_REMOTE_CNT = S_REMOTE_CNT + 0 WHERE S_I_ID = 24167 AND S_W_ID = 2@@@@UPDATE STOCK SET S_QUANTITY = 97 , S_YTD = S_YTD + 1, S_ORDER_CNT = S_ORDER_CNT + 1, S_REMOTE_CNT = S_REMOTE_CNT + 0 WHERE S_I_ID = 96982 AND S_W_ID = 2@@@@UPDATE STOCK SET S_QUANTITY = 58 , S_YTD = S_YTD + 7, S_ORDER_CNT = S_ORDER_CNT + 1, S_REMOTE_CNT = S_REMOTE_CNT + 0 WHERE S_I_ID = 40679 AND S_W_ID = 2@@@@UPDATE STOCK SET S_QUANTITY = 28 , S_YTD = S_YTD + 9, S_ORDER_CNT = S_ORDER_CNT + 1, S_REMOTE_CNT = S_REMOTE_CNT + 0 WHERE S_I_ID = 31459 AND S_W_ID = 2@@@@UPDATE STOCK SET S_QUANTITY = 86 , S_YTD = S_YTD + 3, S_ORDER_CNT = S_ORDER_CNT + 1, S_REMOTE_CNT = S_REMOTE_CNT + 0 WHERE S_I_ID = 6143 AND S_W_ID = 2@@@@UPDATE STOCK SET S_QUANTITY = 13 , S_YTD = S_YTD + 5, S_ORDER_CNT = S_ORDER_CNT + 1, S_REMOTE_CNT = S_REMOTE_CNT + 0 WHERE S_I_ID = 12001 AND S_W_ID = 2@@@@UPDATE STOCK SET S_QUANTITY = 44 , S_YTD = S_YTD + 1, S_ORDER_CNT = S_ORDER_CNT + 1, S_REMOTE_CNT = S_REMOTE_CNT + 0 WHERE S_I_ID = 40407 AND S_W_ID = 2 ... ``` -Note that 3 and 5 are latencies of each transaction in milliseconds. + +Note that 3 and 6 are latencies of each transaction in milliseconds, while 0.165 and 0.123 are latencies of the first statement of each transaction in milliseconds. From 9c12e78cceb2a512de3aa7355d3a8a111ae13b9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 9 May 2017 20:51:35 +0300 Subject: [PATCH 46/55] Remove trailing whitespace in tpmfilter documentation The document had some leftover whitespace. --- .../Filters/Transaction-Performance-Monitoring-Filter.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/Filters/Transaction-Performance-Monitoring-Filter.md b/Documentation/Filters/Transaction-Performance-Monitoring-Filter.md index a49602180..87d5d1550 100644 --- a/Documentation/Filters/Transaction-Performance-Monitoring-Filter.md +++ b/Documentation/Filters/Transaction-Performance-Monitoring-Filter.md @@ -47,7 +47,7 @@ filename=/tmp/SqlQueryLog ### Source -The optional `source` parameter defines an address that is used +The optional `source` parameter defines an address that is used to match against the address from which the client connection to MaxScale originates. Only sessions that originate from this address will be logged. @@ -95,11 +95,11 @@ disabled when the router receives the character '0' from this named pipe. The default named pipe is **`/tmp/tpmfilter`** and logging is **disabled** by default. named_pipe=/tmp/tpmfilter - + For example, the following command enables the logging: $ echo '1' > /tmp/tpmfilter - + Similarly, the following command disables the logging: $ echo '0' > /tmp/tpmfilter From aebe839990fa429ed85ad0ff80b0f1383f4f5c55 Mon Sep 17 00:00:00 2001 From: Esa Korhonen Date: Tue, 9 May 2017 16:09:10 +0300 Subject: [PATCH 47/55] Change session id to 64bit The server internal session id may be larger than 4 bytes (MariaDB uses 8) but only 4 are sent in the handshake. The full value can be queried from the server, but this query is not supported by MaxScale yet. In any case, both the protocol and MXS_SESSION now have 64 bit counters. Only the low 32 bits are sent in the handshake, similar to server. --- include/maxscale/dcb.h | 2 +- include/maxscale/protocol/mysql.h | 2 +- include/maxscale/session.h | 8 +++---- server/core/dcb.cc | 4 ++-- server/core/session.cc | 24 +++++++++---------- .../protocol/MySQL/MySQLClient/mysql_client.c | 9 +++---- server/modules/protocol/MySQL/mysql_common.c | 7 +++--- .../schemarouter/schemaroutersession.cc | 4 ++-- 8 files changed, 31 insertions(+), 29 deletions(-) diff --git a/include/maxscale/dcb.h b/include/maxscale/dcb.h index fa551ce3a..27cf49228 100644 --- a/include/maxscale/dcb.h +++ b/include/maxscale/dcb.h @@ -313,7 +313,7 @@ int dcb_isvalid(DCB *); /* Check the DCB is in the linked li int dcb_count_by_usage(DCB_USAGE); /* Return counts of DCBs */ int dcb_persistent_clean_count(DCB *, int, bool); /* Clean persistent and return count */ void dcb_hangup_foreach (struct server* server); -uint32_t dcb_get_session_id(DCB* dcb); +uint64_t dcb_get_session_id(DCB* dcb); char *dcb_role_name(DCB *); /* Return the name of a role */ int dcb_accept_SSL(DCB* dcb); int dcb_connect_SSL(DCB* dcb); diff --git a/include/maxscale/protocol/mysql.h b/include/maxscale/protocol/mysql.h index a07262dae..438655eef 100644 --- a/include/maxscale/protocol/mysql.h +++ b/include/maxscale/protocol/mysql.h @@ -320,7 +320,7 @@ typedef struct uint32_t server_capabilities; /*< server capabilities, created or received */ uint32_t client_capabilities; /*< client capabilities, created or received */ uint32_t extra_capabilities; /*< MariaDB 10.2 capabilities */ - uint32_t tid; /*< MySQL Thread ID, in handshake */ + uint64_t thread_id; /*< MySQL Thread ID. Send only 32bits in handshake. */ unsigned int charset; /*< MySQL character set at connect time */ int ignore_replies; /*< How many replies should be discarded */ GWBUF* stored_query; /*< Temporarily stored queries */ diff --git a/include/maxscale/session.h b/include/maxscale/session.h index 03f5f4963..240d5990e 100644 --- a/include/maxscale/session.h +++ b/include/maxscale/session.h @@ -134,7 +134,7 @@ typedef struct session { skygw_chk_t ses_chk_top; mxs_session_state_t state; /*< Current descriptor state */ - uint32_t ses_id; /*< Unique session identifier */ + uint64_t ses_id; /*< Unique session identifier */ struct dcb *client_dcb; /*< The client connection */ struct mxs_router_session *router_session; /*< The router instance data */ MXS_SESSION_STATS stats; /*< Session statistics */ @@ -194,7 +194,7 @@ MXS_SESSION *session_alloc(struct service *, struct dcb *); * @param id Id for the new session. * @return The newly created session or NULL if an error occurred */ -MXS_SESSION *session_alloc_with_id(struct service *, struct dcb *, uint32_t); +MXS_SESSION *session_alloc_with_id(struct service *, struct dcb *, uint64_t); MXS_SESSION *session_set_dummy(struct dcb *); @@ -352,14 +352,14 @@ static inline bool session_set_autocommit(MXS_SESSION* ses, bool autocommit) * * @note The caller must free the session reference by calling session_put_ref */ -MXS_SESSION* session_get_by_id(uint32_t id); +MXS_SESSION* session_get_by_id(uint64_t id); /** * Get the next available unique (assuming no overflow) session id number. * * @return An unused session id. */ -uint32_t session_get_next_id(); +uint64_t session_get_next_id(); /** * @brief Close a session diff --git a/server/core/dcb.cc b/server/core/dcb.cc index af83c99f7..b302b9579 100644 --- a/server/core/dcb.cc +++ b/server/core/dcb.cc @@ -163,7 +163,7 @@ static uint32_t dcb_process_poll_events(DCB *dcb, int thread_id, uint32_t ev); static void dcb_process_fake_events(DCB *dcb, int thread_id); static bool dcb_session_check(DCB *dcb, const char *); -uint32_t dcb_get_session_id(DCB *dcb) +uint64_t dcb_get_session_id(DCB *dcb) { return (dcb && dcb->session) ? dcb->session->ses_id : 0; } @@ -1673,7 +1673,7 @@ dprintDCB(DCB *pdcb, DCB *dcb) if (dcb->session && dcb->session->state != SESSION_STATE_DUMMY) { - dcb_printf(pdcb, "\tOwning Session: %" PRIu32 "\n", dcb->session->ses_id); + dcb_printf(pdcb, "\tOwning Session: %" PRIu64 "\n", dcb->session->ses_id); } if (dcb->writeq) diff --git a/server/core/session.cc b/server/core/session.cc index 0ed98a34b..61d48bbe5 100644 --- a/server/core/session.cc +++ b/server/core/session.cc @@ -61,7 +61,7 @@ using std::stringstream; /** Global session id counter. Must be updated atomically. Value 0 is reserved for * dummy/unused sessions. */ -static uint32_t next_session_id = 1; +static uint64_t next_session_id = 1; static struct session session_dummy_struct; @@ -162,7 +162,7 @@ MXS_SESSION* session_alloc(SERVICE *service, DCB *client_dcb) return session_alloc_body(service, client_dcb, session); } -MXS_SESSION* session_alloc_with_id(SERVICE *service, DCB *client_dcb, uint32_t id) +MXS_SESSION* session_alloc_with_id(SERVICE *service, DCB *client_dcb, uint64_t id) { MXS_SESSION *session = (MXS_SESSION *)(MXS_MALLOC(sizeof(*session))); if (session == NULL) @@ -263,13 +263,13 @@ static MXS_SESSION* session_alloc_body(SERVICE* service, DCB* client_dcb, if (session->client_dcb->user == NULL) { - MXS_INFO("Started session [%" PRIu32 "] for %s service ", + MXS_INFO("Started session [%" PRIu64 "] for %s service ", session->ses_id, service->name); } else { - MXS_INFO("Started %s client session [%" PRIu32 "] for '%s' from %s", + MXS_INFO("Started %s client session [%" PRIu64 "] for '%s' from %s", service->name, session->ses_id, session->client_dcb->user, @@ -278,7 +278,7 @@ static MXS_SESSION* session_alloc_body(SERVICE* service, DCB* client_dcb, } else { - MXS_INFO("Start %s client session [%" PRIu32 "] for '%s' from %s failed, will be " + MXS_INFO("Start %s client session [%" PRIu64 "] for '%s' from %s failed, will be " "closed as soon as all related DCBs have been closed.", service->name, session->ses_id, @@ -448,7 +448,7 @@ static void session_free(MXS_SESSION *session) MXS_FREE(session->filters); } - MXS_INFO("Stopped %s client session [%" PRIu32 "]", session->service->name, session->ses_id); + MXS_INFO("Stopped %s client session [%" PRIu64 "]", session->service->name, session->ses_id); /** If session doesn't have parent referencing to it, it can be freed */ if (!session->ses_is_child) @@ -560,7 +560,7 @@ dprintSession(DCB *dcb, MXS_SESSION *print_session) char buf[30]; int i; - dcb_printf(dcb, "Session %" PRIu32 "\n", print_session->ses_id); + dcb_printf(dcb, "Session %" PRIu64 "\n", print_session->ses_id); dcb_printf(dcb, "\tState: %s\n", session_state(print_session->state)); dcb_printf(dcb, "\tService: %s\n", print_session->service->name); @@ -600,7 +600,7 @@ bool dListSessions_cb(DCB *dcb, void *data) { DCB *out_dcb = (DCB*)data; MXS_SESSION *session = dcb->session; - dcb_printf(out_dcb, "%-16" PRIu32 " | %-15s | %-14s | %s\n", session->ses_id, + dcb_printf(out_dcb, "%-16" PRIu64 " | %-15s | %-14s | %s\n", session->ses_id, session->client_dcb && session->client_dcb->remote ? session->client_dcb->remote : "", session->service && session->service->name ? @@ -964,7 +964,7 @@ static bool ses_find_id(DCB *dcb, void *data) { void **params = (void**)data; MXS_SESSION **ses = (MXS_SESSION**)params[0]; - uint32_t *id = (uint32_t*)params[1]; + uint64_t *id = (uint64_t*)params[1]; bool rval = true; if (dcb->session->ses_id == *id) @@ -976,7 +976,7 @@ static bool ses_find_id(DCB *dcb, void *data) return rval; } -MXS_SESSION* session_get_by_id(uint32_t id) +MXS_SESSION* session_get_by_id(uint64_t id) { MXS_SESSION *session = NULL; void *params[] = {&session, &id}; @@ -1048,9 +1048,9 @@ void session_clear_stmt(MXS_SESSION *session) session->stmt.target = NULL; } -uint32_t session_get_next_id() +uint64_t session_get_next_id() { - return atomic_add_uint32(&next_session_id, 1); + return atomic_add_uint64(&next_session_id, 1); } void session_broadcast_kill_command(MXS_SESSION* issuer, uint64_t target_id) diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index fa684b277..0fca02133 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -241,9 +241,10 @@ int MySQLSendHandshake(DCB* dcb) memcpy(mysql_filler_ten + 6, &new_flags, sizeof(new_flags)); } - // Get the equivalent of the server process id. - protocol->tid = session_get_next_id(); - gw_mysql_set_byte4(mysql_thread_id_num, protocol->tid); + // Get the equivalent of the server thread id. + protocol->thread_id = session_get_next_id(); + // Send only the low 32bits in the handshake. + gw_mysql_set_byte4(mysql_thread_id_num, (uint32_t)(protocol->thread_id)); memcpy(mysql_scramble_buf, server_scramble, 8); memcpy(mysql_plugin_data, server_scramble + 8, 12); @@ -668,7 +669,7 @@ gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) * normal data handling function instead of this one. */ MXS_SESSION *session = - session_alloc_with_id(dcb->service, dcb, protocol->tid); + session_alloc_with_id(dcb->service, dcb, protocol->thread_id); if (session != NULL) { diff --git a/server/modules/protocol/MySQL/mysql_common.c b/server/modules/protocol/MySQL/mysql_common.c index 83e530db1..5e592f6d3 100644 --- a/server/modules/protocol/MySQL/mysql_common.c +++ b/server/modules/protocol/MySQL/mysql_common.c @@ -1410,7 +1410,6 @@ gw_decode_mysql_server_handshake(MySQLProtocol *conn, uint8_t *payload) uint8_t *server_version_end = NULL; uint16_t mysql_server_capabilities_one = 0; uint16_t mysql_server_capabilities_two = 0; - unsigned long tid = 0; uint8_t scramble_data_1[GW_SCRAMBLE_LENGTH_323] = ""; uint8_t scramble_data_2[GW_MYSQL_SCRAMBLE_SIZE - GW_SCRAMBLE_LENGTH_323] = ""; uint8_t capab_ptr[4] = ""; @@ -1433,8 +1432,10 @@ gw_decode_mysql_server_handshake(MySQLProtocol *conn, uint8_t *payload) payload = server_version_end + 1; // get ThreadID: 4 bytes - tid = gw_mysql_get_byte4(payload); - memcpy(&conn->tid, &tid, 4); + uint32_t tid = gw_mysql_get_byte4(payload); + /* TODO: Correct value of thread id could be queried later from backend if + * there is any worry it might be larger than 32bit allows. */ + conn->thread_id = tid; payload += 4; diff --git a/server/modules/routing/schemarouter/schemaroutersession.cc b/server/modules/routing/schemarouter/schemaroutersession.cc index 64a10bd61..0bad5ecde 100644 --- a/server/modules/routing/schemarouter/schemaroutersession.cc +++ b/server/modules/routing/schemarouter/schemaroutersession.cc @@ -363,7 +363,7 @@ int32_t SchemaRouterSession::routeQuery(GWBUF* pPacket) if (m_config->debug) { sprintf(errbuf + strlen(errbuf), - " ([%" PRIu32 "]: DB change failed)", + " ([%" PRIu64 "]: DB change failed)", m_client->session->ses_id); } @@ -991,7 +991,7 @@ bool SchemaRouterSession::handle_default_db() sprintf(errmsg, "Unknown database '%s'", m_connect_db.c_str()); if (m_config->debug) { - sprintf(errmsg + strlen(errmsg), " ([%" PRIu32 "]: DB not found on connect)", + sprintf(errmsg + strlen(errmsg), " ([%" PRIu64 "]: DB not found on connect)", m_client->session->ses_id); } write_error_to_client(m_client, From a0cd067a036366b1525a99863b25e53d3ade56d1 Mon Sep 17 00:00:00 2001 From: Esa Korhonen Date: Thu, 4 May 2017 17:51:10 +0300 Subject: [PATCH 48/55] KILL [CONNECTION | QUERY] support, part3 The text-version of "KILL CONNECTION" command is now supported. To keep the overhead low, only minimal parsing is done on the query. The query needs to start in the beginning of the mysql-packet, have no comments and have limited whitespace as the total length of the query is limited. Both "KILL 123" and "KILL CONNECTION 123" are accepted. "KILL QUERY 123" is also accepted but not acted on, as it requires larger changes. --- .../protocol/MySQL/MySQLClient/mysql_client.c | 295 ++++++++++++++++-- 1 file changed, 269 insertions(+), 26 deletions(-) diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index 0fca02133..51b08458c 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -14,6 +14,11 @@ #define MXS_MODULE_NAME "MySQLClient" +#include +#include +#include +#include + #include #include #include @@ -21,14 +26,33 @@ #include #include #include -#include #include -#include #include #include #include #include +/** Return type of process_special_commands() */ +typedef enum spec_com_res_t +{ + RES_CONTINUE, // No special command detected, proceed as normal. + RES_END, // Query handling completed, do not send to filters/router. + RES_MORE_DATA // Possible special command, but not enough data to be sure. Must + // wait for more data. +} spec_com_res_t; + +/* Type of the kill-command sent by client. */ +typedef enum kill_type +{ + KT_CONNECTION, + KT_QUERY +} kill_type_t; + +/* Limits on the length of the queries in which "KILL" is searched for. Reducing + * LONGEST_KILL will reduce overhead but also limit the range of accepted queries. */ +const int SHORTEST_KILL = sizeof("KILL 1") - 1; +const int LONGEST_KILL = sizeof("KILL CONNECTION 12345678901234567890 ;"); + static int process_init(void); static void process_finish(void); static int thread_init(void); @@ -52,11 +76,10 @@ static int gw_read_normal_data(DCB *dcb, GWBUF *read_buffer, int nbytes_read); static int gw_read_finish_processing(DCB *dcb, GWBUF *read_buffer, uint64_t capabilities); static bool ensure_complete_packet(DCB *dcb, GWBUF **read_buffer, int nbytes_read); static void gw_process_one_new_client(DCB *client_dcb); -static bool process_special_commands(DCB* client_dcb, GWBUF *read_buffer, int nbytes_read); - -/* - * The "module object" for the mysqld client protocol module. - */ +static spec_com_res_t process_special_commands(DCB *client_dcb, GWBUF *read_buffer, int nbytes_read); +static spec_com_res_t handle_query_kill(DCB* dcb, GWBUF* read_buffer, spec_com_res_t current, + bool is_complete, unsigned int packet_len); +static uint64_t parse_kill_query(char *query, kill_type_t *kt_out); /** * The module entry point routine. It is this routine that @@ -908,12 +931,29 @@ gw_read_normal_data(DCB *dcb, GWBUF *read_buffer, int nbytes_read) } } - if (!process_special_commands(dcb, read_buffer, nbytes_read)) + spec_com_res_t res = process_special_commands(dcb, read_buffer, nbytes_read); + int rval = 1; + switch (res) { - return 0; - } + case RES_MORE_DATA: + dcb->dcb_readqueue = read_buffer; + rval = 0; + break; - return gw_read_finish_processing(dcb, read_buffer, capabilities); + case RES_END: + // Do not send this packet for routing + gwbuf_free(read_buffer); + rval = 0; + break; + + case RES_CONTINUE: + rval = gw_read_finish_processing(dcb, read_buffer, capabilities); + break; + + default: + ss_dassert(!true); + } + return rval; } /** @@ -1484,14 +1524,23 @@ static bool ensure_complete_packet(DCB *dcb, GWBUF **read_buffer, int nbytes_rea /** * Some SQL commands/queries need to be detected and handled by the protocol * and MaxScale instead of being routed forward as is. + * * @param dcb Client dcb * @param read_buffer the current read buffer * @param nbytes_read How many bytes were read - * @return true if read buffer should be sent forward to routing, false if more - * data is required or processing is complete + * @return see @c spec_com_res_t */ -static bool process_special_commands(DCB* dcb, GWBUF *read_buffer, int nbytes_read) +static spec_com_res_t process_special_commands(DCB *dcb, GWBUF *read_buffer, int nbytes_read) { + spec_com_res_t rval = RES_CONTINUE; + bool is_complete = false; + unsigned int packet_len = + MYSQL_GET_PAYLOAD_LEN((uint8_t *)GWBUF_DATA(read_buffer)) + MYSQL_HEADER_LEN; + if (gwbuf_length(read_buffer) == packet_len) + { + is_complete = true; + } + /** * Handle COM_SET_OPTION. This seems to be only used by some versions of PHP. * @@ -1516,32 +1565,226 @@ static bool process_special_commands(DCB* dcb, GWBUF *read_buffer, int nbytes_re /** * Handle COM_PROCESS_KILL */ - else if ((proto->current_command == MYSQL_COM_PROCESS_KILL)) + else if (proto->current_command == MYSQL_COM_PROCESS_KILL) { /* Make sure we have a complete SQL packet before trying to read the * process id. If not, try again next time. */ - unsigned int expected_len = - MYSQL_GET_PAYLOAD_LEN((uint8_t *)GWBUF_DATA(read_buffer)) + MYSQL_HEADER_LEN; - if (gwbuf_length(read_buffer) < expected_len) + if (!is_complete) { - dcb->dcb_readqueue = read_buffer; - return false; + rval = RES_MORE_DATA; } else { uint8_t bytes[4]; - if (gwbuf_copy_data(read_buffer, MYSQL_HEADER_LEN + 1, sizeof(bytes), (uint8_t*)bytes) + if (gwbuf_copy_data(read_buffer, MYSQL_HEADER_LEN + 1, sizeof(bytes), bytes) == sizeof(bytes)) { uint64_t process_id = gw_mysql_get_byte4(bytes); - // Do not send this packet for routing - gwbuf_free(read_buffer); session_broadcast_kill_command(dcb->session, process_id); // Even if id not found, send ok. TODO: send a correct response to client mxs_mysql_send_ok(dcb, 1, 0, NULL); - return false; + rval = RES_END; } } } - return true; -} \ No newline at end of file + else if (proto->current_command == MYSQL_COM_QUERY) + { + /* Is length within limits for a kill-type query? */ + if (packet_len >= (MYSQL_HEADER_LEN + 1 + SHORTEST_KILL) && + packet_len <= (MYSQL_HEADER_LEN + 1 + LONGEST_KILL)) + { + rval = handle_query_kill(dcb, read_buffer, rval, is_complete, packet_len); + } + } + return rval; +} + +/** + * Handle text version of KILL [CONNECTION | QUERY] . Only detects + * commands in the beginning of the packet and with no comments. + * Increased parsing would slow down the handling of every single query. + * + * @param dcb Client dcb + * @param read_buffer Input buffer + * @param current Latest value of rval in calling function + * @param is_complete Is read_buffer a complete sql packet + * @param packet_len Read from sql header + * @return Updated (or old) value of rval + */ +spec_com_res_t handle_query_kill(DCB* dcb, GWBUF* read_buffer, spec_com_res_t current, + bool is_complete, unsigned int packet_len) +{ + spec_com_res_t rval = current; + /* First, we need to detect the text "KILL" (ignorecase) in the start + * of the packet. Copy just enough characters. */ + const char KILL_BEGIN[] = "KILL"; + const size_t KILL_BEGIN_LEN = sizeof(KILL_BEGIN) - 1; + char startbuf[KILL_BEGIN_LEN]; // Not 0-terminated, careful... + size_t copied_len = gwbuf_copy_data(read_buffer, MYSQL_HEADER_LEN + 1, + KILL_BEGIN_LEN, (uint8_t*)startbuf); + if (is_complete) + { + if (strncasecmp(KILL_BEGIN, startbuf, KILL_BEGIN_LEN) == 0) + { + /* Good chance that the query is a KILL-query. Copy the entire + * buffer (skip the "KILL ") and process. */ + size_t buffer_len = packet_len - (MYSQL_HEADER_LEN + 1) - KILL_BEGIN_LEN; + char querybuf[buffer_len + 1]; // 0-terminated + copied_len = gwbuf_copy_data(read_buffer, + MYSQL_HEADER_LEN + 1 + KILL_BEGIN_LEN, + buffer_len, + (uint8_t*)querybuf); + querybuf[copied_len] = '\0'; + kill_type_t kt = KT_CONNECTION; + uint64_t thread_id = parse_kill_query(querybuf, &kt); + + if (thread_id) + { + switch (kt) + { + case KT_CONNECTION: + session_broadcast_kill_command(dcb->session, thread_id); + // Even if id not found, send ok. TODO: send a correct response to client + mxs_mysql_send_ok(dcb, 1, 0, NULL); + rval = RES_END; + break; + + case KT_QUERY: + // TODO: Implement this + MXS_WARNING("Received 'KILL QUERY %" PRIu64 "' from " + "the client. This feature is not supported.", thread_id); + mysql_send_custom_error(dcb, 1, 0, "'KILL QUERY ' " + "is not supported."); + rval = RES_END; + break; + + default: + ss_dassert(!true); + } + } + } + } + else + { + /* Look at the start of the query and see if it might contain "KILL" */ + if (strncasecmp(KILL_BEGIN, startbuf, copied_len) == 0) + { + rval = RES_MORE_DATA; + } + } + return rval; +} + +/** + * Parse and process a "KILL [CONNECTION | QUERY] " query. Will modify + * the argument string even if not successful. + * + * @param query The query string + * @param kt_out The kill command type output + * @return Zero on error, a valid ID otherwise + */ +static uint64_t parse_kill_query(char *query, kill_type_t *kt_out) +{ + const char WORD_CONNECTION[] = "CONNECTION"; + const char WORD_QUERY[] = "QUERY"; + const char DELIM[] = " \n\t"; + + kill_type_t kill_type = KT_CONNECTION; + unsigned long long int thread_id = 0; + + enum kill_parse_state_t + { + CONN_QUERY, + ID, + SEMICOLON, + DONE + } state = CONN_QUERY; + + char *saveptr = NULL; + char *token = strtok_r(query, DELIM, &saveptr); + bool error = false; + + while (token && !error && state != DONE) + { + bool get_next = false; + switch (state) + { + case CONN_QUERY: + { + if (strncasecmp(token, WORD_QUERY, sizeof(WORD_QUERY) - 1) == 0) + { + kill_type = KT_QUERY; + get_next = true; + } + else if (strncasecmp(token, WORD_CONNECTION, sizeof(WORD_CONNECTION) - 1) == 0) + { + get_next = true; + } + /* Move to next state regardless of comparison result. The current + * part is optional and the process id may already be in the token. */ + state = ID; + } + break; + + case ID: + { + char *endptr_id; + thread_id = strtoull(token, &endptr_id, 0); + /* Zero is an error value, also MaxScale session id:s start at 1. */ + if (thread_id == 0 || (thread_id == ULLONG_MAX && errno == ERANGE)) + { + error = true; + errno = 0; + } + else if (*endptr_id == '\0') // Can be real end or written by strtok + { + state = SEMICOLON; // In case we have space before ; + get_next = true; + } + else if (*endptr_id == ';') + { + token = endptr_id; + state = SEMICOLON; + } + else + { + error = true; + } + } + break; + + case SEMICOLON: + { + if (strncmp(token, ";", 1) == 0) + { + state = DONE; + } + else + { + error = true; + } + } + break; + + default: + { + error = true; + } + break; + } + if (get_next) + { + token = strtok_r(NULL, DELIM, &saveptr); + } + } + + if (error || (state != DONE && state != SEMICOLON)) + { + return 0; + } + else + { + *kt_out = kill_type; + return thread_id; + } +} From c2879cc52f0264b5f7fba6f6099b93c7b1ffc29b Mon Sep 17 00:00:00 2001 From: Esa Korhonen Date: Wed, 10 May 2017 13:23:55 +0300 Subject: [PATCH 49/55] Update release notes and limitations regarding the KILL-command support --- Documentation/About/Limitations.md | 13 ++++++++++++- .../Release-Notes/MaxScale-2.2.0-Release-Notes.md | 7 +++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Documentation/About/Limitations.md b/Documentation/About/Limitations.md index 286d2454b..ff16f4963 100644 --- a/Documentation/About/Limitations.md +++ b/Documentation/About/Limitations.md @@ -47,7 +47,18 @@ transaction or change the autocommit mode using a prepared statement. ### Limitations with MySQL Protocol support (MySQLClient) -Compression is not included in the MySQL server handshake. +* Compression is not included in the MySQL server handshake. + +* MariaDB MaxScale will intercept `KILL ` statements which are of the +form `KILL 3`, `KILL CONNECTION 321` and `KILL QUERY 8`. These queries are not +routed to backends because the `` sent by the client does not equal a +backend id. MaxScale reacts to a thread kill command by killing the session with +the given id if the user and host of the issuing session and the target session +match. Query kill command is not supported and results in an error message. For +MaxScale to recognize the *KILL* statement, the statement must start right after +the command byte, have no comments and have minimal whitespace. These +limitations are in place to limit the parsing MaxScale needs to do to every +query. ## Authenticator limitations diff --git a/Documentation/Release-Notes/MaxScale-2.2.0-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.2.0-Release-Notes.md index e315436d4..781f7776f 100644 --- a/Documentation/Release-Notes/MaxScale-2.2.0-Release-Notes.md +++ b/Documentation/Release-Notes/MaxScale-2.2.0-Release-Notes.md @@ -68,6 +68,13 @@ The MySQL backend protocol module now supports sending a proxy protocol header to the server. For more information, see the server section in the [Configuration guide](../Getting-Started/Configuration-Guide.md). +### KILL command support + +The MySQL client protocol now detects `KILL ` statements (binary and +query forms) and kills the MaxScale session with the given id. This feature has +some limitations, see [Limitations](../About/Limitations.md) for more +information. + ## Bug fixes [Here is a list of bugs fixed since the release of MaxScale 2.1.X.]() From 4add7a14fcae177200443ad89f2157528cd8ca38 Mon Sep 17 00:00:00 2001 From: Esa Korhonen Date: Wed, 10 May 2017 15:50:47 +0300 Subject: [PATCH 50/55] parse_kill_query() now also expects the "KILL" This was originally removed, since it was checking the same word twice. However, the parsing is clearer with it and the cost is only paid when the KILL is detected, which is very rare. Also, fix some incorrect parsing. --- .../protocol/MySQL/MySQLClient/mysql_client.c | 113 +++++++++++------- 1 file changed, 68 insertions(+), 45 deletions(-) diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index 51b08458c..22f412a94 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -48,10 +48,7 @@ typedef enum kill_type KT_QUERY } kill_type_t; -/* Limits on the length of the queries in which "KILL" is searched for. Reducing - * LONGEST_KILL will reduce overhead but also limit the range of accepted queries. */ -const int SHORTEST_KILL = sizeof("KILL 1") - 1; -const int LONGEST_KILL = sizeof("KILL CONNECTION 12345678901234567890 ;"); +const char WORD_KILL[] = "KILL"; static int process_init(void); static void process_finish(void); @@ -79,7 +76,7 @@ static void gw_process_one_new_client(DCB *client_dcb); static spec_com_res_t process_special_commands(DCB *client_dcb, GWBUF *read_buffer, int nbytes_read); static spec_com_res_t handle_query_kill(DCB* dcb, GWBUF* read_buffer, spec_com_res_t current, bool is_complete, unsigned int packet_len); -static uint64_t parse_kill_query(char *query, kill_type_t *kt_out); +static bool parse_kill_query(char *query, uint64_t *thread_id_out, kill_type_t *kt_out); /** * The module entry point routine. It is this routine that @@ -1589,6 +1586,10 @@ static spec_com_res_t process_special_commands(DCB *dcb, GWBUF *read_buffer, int } else if (proto->current_command == MYSQL_COM_QUERY) { + /* Limits on the length of the queries in which "KILL" is searched for. Reducing + * LONGEST_KILL will reduce overhead but also limit the range of accepted queries. */ + const int SHORTEST_KILL = sizeof("KILL 1") - 1; + const int LONGEST_KILL = sizeof("KILL CONNECTION 12345678901234567890 ;"); /* Is length within limits for a kill-type query? */ if (packet_len >= (MYSQL_HEADER_LEN + 1 + SHORTEST_KILL) && packet_len <= (MYSQL_HEADER_LEN + 1 + LONGEST_KILL)) @@ -1617,28 +1618,28 @@ spec_com_res_t handle_query_kill(DCB* dcb, GWBUF* read_buffer, spec_com_res_t cu spec_com_res_t rval = current; /* First, we need to detect the text "KILL" (ignorecase) in the start * of the packet. Copy just enough characters. */ - const char KILL_BEGIN[] = "KILL"; - const size_t KILL_BEGIN_LEN = sizeof(KILL_BEGIN) - 1; + const size_t KILL_BEGIN_LEN = sizeof(WORD_KILL) - 1; char startbuf[KILL_BEGIN_LEN]; // Not 0-terminated, careful... size_t copied_len = gwbuf_copy_data(read_buffer, MYSQL_HEADER_LEN + 1, KILL_BEGIN_LEN, (uint8_t*)startbuf); if (is_complete) { - if (strncasecmp(KILL_BEGIN, startbuf, KILL_BEGIN_LEN) == 0) + if (strncasecmp(WORD_KILL, startbuf, KILL_BEGIN_LEN) == 0) { /* Good chance that the query is a KILL-query. Copy the entire - * buffer (skip the "KILL ") and process. */ - size_t buffer_len = packet_len - (MYSQL_HEADER_LEN + 1) - KILL_BEGIN_LEN; + * buffer and process. */ + size_t buffer_len = packet_len - (MYSQL_HEADER_LEN + 1); char querybuf[buffer_len + 1]; // 0-terminated copied_len = gwbuf_copy_data(read_buffer, - MYSQL_HEADER_LEN + 1 + KILL_BEGIN_LEN, + MYSQL_HEADER_LEN + 1, buffer_len, (uint8_t*)querybuf); querybuf[copied_len] = '\0'; kill_type_t kt = KT_CONNECTION; - uint64_t thread_id = parse_kill_query(querybuf, &kt); + uint64_t thread_id = 0; + bool parsed = parse_kill_query(querybuf, &thread_id, &kt); - if (thread_id) + if (parsed && (thread_id > 0)) // MaxScale session counter starts at 1 { switch (kt) { @@ -1667,7 +1668,7 @@ spec_com_res_t handle_query_kill(DCB* dcb, GWBUF* read_buffer, spec_com_res_t cu else { /* Look at the start of the query and see if it might contain "KILL" */ - if (strncasecmp(KILL_BEGIN, startbuf, copied_len) == 0) + if (strncasecmp(WORD_KILL, startbuf, copied_len) == 0) { rval = RES_MORE_DATA; } @@ -1676,14 +1677,15 @@ spec_com_res_t handle_query_kill(DCB* dcb, GWBUF* read_buffer, spec_com_res_t cu } /** - * Parse and process a "KILL [CONNECTION | QUERY] " query. Will modify - * the argument string even if not successful. + * Parse a "KILL [CONNECTION | QUERY] " query. Will modify + * the argument string even if unsuccessful. * - * @param query The query string - * @param kt_out The kill command type output - * @return Zero on error, a valid ID otherwise + * @param query Query string to parse + * @paran thread_id_out Thread id output + * @param kt_out Kill command type output + * @return true on success, false on error */ -static uint64_t parse_kill_query(char *query, kill_type_t *kt_out) +static bool parse_kill_query(char *query, uint64_t *thread_id_out, kill_type_t *kt_out) { const char WORD_CONNECTION[] = "CONNECTION"; const char WORD_QUERY[] = "QUERY"; @@ -1694,48 +1696,68 @@ static uint64_t parse_kill_query(char *query, kill_type_t *kt_out) enum kill_parse_state_t { + KILL, CONN_QUERY, ID, SEMICOLON, DONE - } state = CONN_QUERY; - + } state = KILL; char *saveptr = NULL; - char *token = strtok_r(query, DELIM, &saveptr); bool error = false; - while (token && !error && state != DONE) + char *token = strtok_r(query, DELIM, &saveptr); + + while (token && !error) { bool get_next = false; switch (state) { - case CONN_QUERY: + case KILL: + if (strncasecmp(token, WORD_KILL, sizeof(WORD_KILL) - 1) == 0) { - if (strncasecmp(token, WORD_QUERY, sizeof(WORD_QUERY) - 1) == 0) - { - kill_type = KT_QUERY; - get_next = true; - } - else if (strncasecmp(token, WORD_CONNECTION, sizeof(WORD_CONNECTION) - 1) == 0) - { - get_next = true; - } - /* Move to next state regardless of comparison result. The current - * part is optional and the process id may already be in the token. */ - state = ID; + state = CONN_QUERY; + get_next = true; } + else + { + error = true; + } + break; + + case CONN_QUERY: + if (strncasecmp(token, WORD_QUERY, sizeof(WORD_QUERY) - 1) == 0) + { + kill_type = KT_QUERY; + get_next = true; + } + else if (strncasecmp(token, WORD_CONNECTION, sizeof(WORD_CONNECTION) - 1) == 0) + { + get_next = true; + } + /* Move to next state regardless of comparison result. The current + * part is optional and the process id may already be in the token. */ + state = ID; break; case ID: { - char *endptr_id; + /* strtoull() accepts negative numbers, so check for '-' here */ + if (*token == '-') + { + error = true; + break; + } + char *endptr_id = NULL; thread_id = strtoull(token, &endptr_id, 0); - /* Zero is an error value, also MaxScale session id:s start at 1. */ - if (thread_id == 0 || (thread_id == ULLONG_MAX && errno == ERANGE)) + if ((thread_id == ULLONG_MAX) && (errno == ERANGE)) { error = true; errno = 0; } + else if (endptr_id == token) + { + error = true; // No digits were read + } else if (*endptr_id == '\0') // Can be real end or written by strtok { state = SEMICOLON; // In case we have space before ; @@ -1758,6 +1780,7 @@ static uint64_t parse_kill_query(char *query, kill_type_t *kt_out) if (strncmp(token, ";", 1) == 0) { state = DONE; + get_next = true; } else { @@ -1767,11 +1790,10 @@ static uint64_t parse_kill_query(char *query, kill_type_t *kt_out) break; default: - { - error = true; - } + error = true; break; } + if (get_next) { token = strtok_r(NULL, DELIM, &saveptr); @@ -1780,11 +1802,12 @@ static uint64_t parse_kill_query(char *query, kill_type_t *kt_out) if (error || (state != DONE && state != SEMICOLON)) { - return 0; + return false; } else { + *thread_id_out = thread_id; *kt_out = kill_type; - return thread_id; + return true; } } From 39da11763b267641ab665d5d7d1917563f8d37e6 Mon Sep 17 00:00:00 2001 From: Esa Korhonen Date: Wed, 10 May 2017 14:01:01 +0300 Subject: [PATCH 51/55] Add test for parse_kill_query() --- server/modules/protocol/MySQL/CMakeLists.txt | 1 + .../protocol/MySQL/test/CMakeLists.txt | 4 + .../protocol/MySQL/test/test_parse_kill.c | 89 +++++++++++++++++++ 3 files changed, 94 insertions(+) create mode 100644 server/modules/protocol/MySQL/test/CMakeLists.txt create mode 100644 server/modules/protocol/MySQL/test/test_parse_kill.c diff --git a/server/modules/protocol/MySQL/CMakeLists.txt b/server/modules/protocol/MySQL/CMakeLists.txt index c8e967bf3..5669ead0f 100644 --- a/server/modules/protocol/MySQL/CMakeLists.txt +++ b/server/modules/protocol/MySQL/CMakeLists.txt @@ -5,3 +5,4 @@ install_module(MySQLCommon core) add_subdirectory(MySQLBackend) add_subdirectory(MySQLClient) +add_subdirectory(test) diff --git a/server/modules/protocol/MySQL/test/CMakeLists.txt b/server/modules/protocol/MySQL/test/CMakeLists.txt new file mode 100644 index 000000000..5da15e4b3 --- /dev/null +++ b/server/modules/protocol/MySQL/test/CMakeLists.txt @@ -0,0 +1,4 @@ +add_executable(test_parse_kill test_parse_kill.c) +target_link_libraries(test_parse_kill maxscale-common MySQLCommon) +add_test(test_parse_kill test_parse_kill) + diff --git a/server/modules/protocol/MySQL/test/test_parse_kill.c b/server/modules/protocol/MySQL/test/test_parse_kill.c new file mode 100644 index 000000000..075ba7e09 --- /dev/null +++ b/server/modules/protocol/MySQL/test/test_parse_kill.c @@ -0,0 +1,89 @@ +#include + +#include "../MySQLClient/mysql_client.c" + +int test_one_query(char *query, bool should_succeed, uint64_t expected_tid, + kill_type_t expected_kt) +{ + char *query_copy = MXS_STRDUP_A(query); + uint64_t result_tid = 1111111; + kill_type_t result_kt = KT_QUERY; + + /* If the parse fails, these should remain unchanged */ + if (!should_succeed) + { + result_tid = expected_tid; + result_kt = expected_kt; + } + bool success = parse_kill_query(query_copy, &result_tid, &result_kt); + MXS_FREE(query_copy); + + if ((success == should_succeed) && (result_tid == expected_tid) && + (result_kt == expected_kt)) + { + return 0; + } + else + { + printf("Result wrong on query: '%s'.\n", query); + if (success != should_succeed) + { + printf("Expected success '%d', got '%d'.\n", should_succeed, success); + } + if (result_tid != expected_tid) + { + printf("Expected thread id '%" PRIu64 "', got '%" PRIu64 "'.\n", + expected_tid, result_tid); + } + if (result_kt != expected_kt) + { + printf("Expected kill type '%u', got '%u'.\n", + expected_kt, result_kt); + } + printf("\n"); + return 1; + } +} +typedef struct test_t +{ + char *query; + bool should_succeed; + uint64_t correct_id; + kill_type_t correct_kt; +} test_t; + +int main(int argc, char **argv) +{ + test_t tests[] = + { + {" kill ConNectioN 123 ", true, 123, KT_CONNECTION}, + {"kIlL coNNectioN 987654321 ;", true, 987654321, KT_CONNECTION}, + {" Ki5L CoNNectioN 987654321 ", false, 0, KT_CONNECTION}, + {"1", false, 0, KT_CONNECTION}, + {"kILL 1", true, 1, KT_CONNECTION}, + {"\n\t kill \nQueRy 456", true, 456, KT_QUERY}, + {" A kill 1; ", false, 0, KT_CONNECTION}, + {" kill connection 1A", false, 0, KT_CONNECTION}, + {" kill connection 1 A ", false, 0, KT_CONNECTION}, + {"kill query 7 ; select * ", false, 0, KT_CONNECTION}, + { + "KIll query \t \n \t 12345678901234567890 \n \t ", + true, 12345678901234567890ULL, KT_QUERY + }, + {"KIll query \t \n \t 21 \n \t ", true, 21, KT_QUERY}, + {"KIll \t \n \t -6 \n \t ", false, 0, KT_CONNECTION}, + {"KIll 12345678901234567890123456 \n \t ", false, 0, KT_CONNECTION}, + {"kill ;", false, 0, KT_QUERY} + }; + int result = 0; + int arr_size = sizeof(tests) / sizeof(test_t); + for (int i = 0; i < arr_size; i++) + { + char *query = tests[i].query; + bool should_succeed = tests[i].should_succeed; + uint64_t expected_tid = tests[i].correct_id; + kill_type_t expected_kt = tests[i].correct_kt; + result += test_one_query(query, should_succeed, expected_tid, expected_kt); + } + return result; +} From ca7b24f6fa7fc23932f5db582f4fec9cd482718d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 11 May 2017 12:04:36 +0300 Subject: [PATCH 52/55] MXS-1220: Allow admin interface to be disabled Allowing the admin interface to be disabled completely makes it possible to remove any security concerns that could arise from its use. --- .../Getting-Started/Configuration-Guide.md | 7 ++++++- include/maxscale/config.h | 2 ++ server/core/config.cc | 6 ++++++ server/core/gateway.cc | 21 +++++++++++-------- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/Documentation/Getting-Started/Configuration-Guide.md b/Documentation/Getting-Started/Configuration-Guide.md index 086a939dc..dbe271171 100644 --- a/Documentation/Getting-Started/Configuration-Guide.md +++ b/Documentation/Getting-Started/Configuration-Guide.md @@ -542,7 +542,7 @@ The port where the HTTP admin interface listens on. The default value is port Enable HTTP admin interface authentication using HTTP Basic Access authentication. This is not a secure method of authentication but it does add a -small layer of security. This option id disabled by default. +small layer of security. This option is disabled by default. #### `admin_user` @@ -573,6 +573,11 @@ documentation for more details. The path to the TLS CA certificate in PEM format. See `admin_ssl_key` documentation for more details. +#### `admin_enabled` + +Enable or disable the admin interface. This allows the admin interface to +be completely disabled to prevent access to it. + ### Service A service represents the database service that MariaDB MaxScale offers to the diff --git a/include/maxscale/config.h b/include/maxscale/config.h index 2afffa84d..a84d520fd 100644 --- a/include/maxscale/config.h +++ b/include/maxscale/config.h @@ -69,6 +69,7 @@ MXS_BEGIN_DECLS */ extern const char CN_ADDRESS[]; extern const char CN_ADMIN_AUTH[]; +extern const char CN_ADMIN_ENABLED[]; extern const char CN_ADMIN_HOST[]; extern const char CN_ADMIN_PASSWORD[]; extern const char CN_ADMIN_PORT[]; @@ -196,6 +197,7 @@ typedef struct char admin_host[MAX_ADMIN_HOST_LEN]; /**< Admin interface host */ uint16_t admin_port; /**< Admin interface port */ bool admin_auth; /**< Admin interface authentication */ + bool admin_enabled; /**< Admin interface is enabled */ char admin_ssl_key[PATH_MAX]; /**< Admin SSL key */ char admin_ssl_cert[PATH_MAX]; /**< Admin SSL cert */ char admin_ssl_ca_cert[PATH_MAX]; /**< Admin SSL CA cert */ diff --git a/server/core/config.cc b/server/core/config.cc index 58fce45a9..06408c03c 100644 --- a/server/core/config.cc +++ b/server/core/config.cc @@ -54,6 +54,7 @@ using std::string; const char CN_ADDRESS[] = "address"; const char CN_ADMIN_AUTH[] = "admin_auth"; +const char CN_ADMIN_ENABLED[] = "admin_enabled"; const char CN_ADMIN_HOST[] = "admin_host"; const char CN_ADMIN_PASSWORD[] = "admin_password"; const char CN_ADMIN_PORT[] = "admin_port"; @@ -1552,6 +1553,10 @@ handle_global_item(const char *name, const char *value) { gateway.admin_auth = config_truth_value(value); } + else if (strcmp(name, CN_ADMIN_ENABLED) == 0) + { + gateway.admin_enabled = config_truth_value(value); + } else { for (i = 0; lognames[i].name; i++) @@ -1774,6 +1779,7 @@ global_defaults() gateway.skip_permission_checks = false; gateway.admin_port = DEFAULT_ADMIN_HTTP_PORT; gateway.admin_auth = false; + gateway.admin_enabled = true; strcpy(gateway.admin_host, DEFAULT_ADMIN_HOST); strcpy(gateway.admin_user, INET_DEFAULT_USERNAME); strcpy(gateway.admin_password, INET_DEFAULT_PASSWORD); diff --git a/server/core/gateway.cc b/server/core/gateway.cc index 47425c8cd..2cbf3329f 100644 --- a/server/core/gateway.cc +++ b/server/core/gateway.cc @@ -1982,16 +1982,19 @@ int main(int argc, char **argv) } } - if (mxs_admin_init()) + if (cnf->admin_enabled) { - MXS_NOTICE("Started REST API on [%s]:%u", cnf->admin_host, cnf->admin_port); - } - else - { - const char* logerr = "Failed to initialize admin interface"; - print_log_n_stderr(true, true, logerr, logerr, 0); - rc = MAXSCALE_INTERNALERROR; - goto return_main; + if (mxs_admin_init()) + { + MXS_NOTICE("Started REST API on [%s]:%u", cnf->admin_host, cnf->admin_port); + } + else + { + const char* logerr = "Failed to initialize admin interface"; + print_log_n_stderr(true, true, logerr, logerr, 0); + rc = MAXSCALE_INTERNALERROR; + goto return_main; + } } MXS_NOTICE("MaxScale started with %d server threads.", config_threadcount()); From ffc6dba72088835bebc26e414d7f5e9ca3646a1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 11 May 2017 13:20:27 +0300 Subject: [PATCH 53/55] MXS-1220: Fix error on PUT request with no parameters Whena server was modified and no parameters were given, the operation was reported as a failure even though it was successful. --- server/core/config_runtime.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/core/config_runtime.cc b/server/core/config_runtime.cc index 0a6e7e840..e51d8b45a 100644 --- a/server/core/config_runtime.cc +++ b/server/core/config_runtime.cc @@ -955,6 +955,7 @@ bool runtime_alter_server_from_json(SERVER* server, json_t* new_json) if (server_to_object_relations(server, old_json.get(), new_json)) { + rval = true; json_t* parameters = mxs_json_pointer(new_json, MXS_JSON_PTR_PARAMETERS); json_t* old_parameters = mxs_json_pointer(old_json.get(), MXS_JSON_PTR_PARAMETERS); @@ -962,7 +963,6 @@ bool runtime_alter_server_from_json(SERVER* server, json_t* new_json) if (parameters) { - rval = true; const char* key; json_t* value; From b317f66335403cbbd3ab8592d9887863918cf668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 11 May 2017 13:23:41 +0300 Subject: [PATCH 54/55] MXS-1220: Update server resource documentation Updated documentation with latest resource layouts. Expanded explanations on how to modify the servers. Use the correct resource endpoints. --- .../Getting-Started/Configuration-Guide.md | 12 + Documentation/REST-API/API.md | 205 +++---- Documentation/REST-API/Resources-Server.md | 577 +++++++++++------- 3 files changed, 429 insertions(+), 365 deletions(-) diff --git a/Documentation/Getting-Started/Configuration-Guide.md b/Documentation/Getting-Started/Configuration-Guide.md index dbe271171..976e150f4 100644 --- a/Documentation/Getting-Started/Configuration-Guide.md +++ b/Documentation/Getting-Started/Configuration-Guide.md @@ -1047,6 +1047,18 @@ refuse these due to the lack of the header. To bypass this restriction, the server monitor needs to be disabled and the service listener needs to be configured to disregard authentication errors (`skip_authentication=true`). +#### `authenticator` + +The authenticator module to use. Each protocol module defines a default +authentication module which is used if no `authenticator` parameter is found +from the configuration. + +#### `authenticator_options` + +Option string given to the authenticator module. The value of this parameter +should be a comma-separated list of key-value pairs. See authenticator specific +documentation for more details. + ### Server and SSL This section describes configuration parameters for servers that control the diff --git a/Documentation/REST-API/API.md b/Documentation/REST-API/API.md index 1a297126d..2bc7980df 100644 --- a/Documentation/REST-API/API.md +++ b/Documentation/REST-API/API.md @@ -4,6 +4,8 @@ This document describes the version 1 of the MaxScale REST API. ## Table of Contents +- [Resources](#resources) +- [Common Request Parameter](#common-request-parameters) - [HTTP Headers](#http-headers) - [Request Headers](#request-headers) - [Response Headers](#response-headers) @@ -12,8 +14,6 @@ This document describes the version 1 of the MaxScale REST API. - [3xx Redirection](#3xx-redirection) - [4xx Client Error](#4xx-client-error) - [5xx Server Error](#5xx-server-error) -- [Resources](#resources) -- [Common Request Parameter](#common-request-parameters) ## Note About Syntax @@ -21,6 +21,78 @@ Although JSON does not define a syntax for comments, some of the JSON examples have C-style inline comments in them. These comments use `//` to mark the start of the comment and extend to the end of the current line. +## Resources + +The MaxScale REST API provides the following resources. All resources conform to +the [JSON API](http://jsonapi.org/format/) specification. + +- [/maxscale](Resources-MaxScale.md) +- [/services](Resources-Service.md) +- [/servers](Resources-Server.md) +- [/filters](Resources-Filter.md) +- [/monitors](Resources-Monitor.md) +- [/sessions](Resources-Session.md) +- [/users](Resources-User.md) + +### Resource Relationships + +All resources return complete JSON objects. The returned objects can have a +_relationships_ field that represents any relations the object has to other +objects. This closely resembles the JSON API definition of links. + +In the _relationships_ objects, all resources have a _self_ link that points to +the resource itself. This allows for easier updating of resources as the reply +URL is included in the response itself. + +The following lists the resources and the types of links each resource can have +in addition to the _self_ link. + +- `services` - Service resource + + - `servers` + + List of servers used by the service + + - `filters` + + List of filters used by the service + +- `monitors` - Monitor resource + + - `servers` + + List of servers used by the monitor + +- `filters` - Filter resource + + - `services` + + List of services that use this filter + +- `servers` - Server resource + + - `services` + + List of services that use this server + + - `monitors` + + List of monitors that use this server + +## Common Request Parameters + +Most of the resources that support GET also support the following +parameters. See the resource documentation for a list of supported request +parameters. + +- `pretty` + + - Pretty-print output. + + If this parameter is set to `true` then the returned objects are + formatted in a more human readable format. All resources support this + parameter. + ## HTTP Headers ### Request Headers @@ -41,19 +113,9 @@ Credentials for authentication. All PUT and POST requests must use the `Content-Type: application/json` media type and the request body must be a valid JSON representation of a resource. All -PATCH requests must use the `Content-Type: application/json-patch` media type -and the request body must be a valid JSON Patch document which is applied to the -resource. Curently, only _add_, _remove_, _replace_ and _test_ operations are -supported. - -Read the [JSON Patch](https://tools.ietf.org/html/draft-ietf-appsawg-json-patch-08) -draft for more details on how to use it with PATCH. - -#### Date - -This header is required and should be in the RFC 1123 standard form, e.g. Mon, -18 Nov 2013 08:14:29 -0600. Please note that the date must be in English. It -will be checked by the API for being close to the current date and time. +PATCH requests must use the `Content-Type: application/json` media type and the +request body must be a JSON document containing a partial definition of the +original resource. #### Host @@ -101,8 +163,6 @@ the intended method in the `X-HTTP-Method-Override` header, a client can perform a POST, PATCH or DELETE request with the PUT method (e.g. `X-HTTP-Method-Override: PATCH`). -_TODO: Add API version header?_ - ### Response Headers #### Allow @@ -294,21 +354,6 @@ contains a more detailed version of the error message. The server failed to fulfill an apparently valid request. -Response status codes beginning with the digit "5" indicate cases in which the -server is aware that it has encountered an error or is otherwise incapable of -performing the request. Except when responding to a HEAD request, the server -includes an entity containing an explanation of the error situation. - -``` -{ - "error": "Log rotation failed", - "description": "Failed to rotate log files: 13, Permission denied" -} -``` - -The _error_ field contains a short error description and the _description_ field -contains a more detailed version of the error message. - - 500 Internal Server Error - A generic error message, given when an unexpected condition was encountered @@ -414,99 +459,3 @@ API could return them. - The user has sent too many requests in a given amount of time. Intended for use with rate-limiting schemes. - -## Resources - -The MaxScale REST API provides the following resources. - -- [/maxscale](Resources-MaxScale.md) -- [/services](Resources-Service.md) -- [/servers](Resources-Server.md) -- [/filters](Resources-Filter.md) -- [/monitors](Resources-Monitor.md) -- [/sessions](Resources-Session.md) -- [/users](Resources-User.md) - -### Resource Relationships - -All resources return complete JSON objects. The returned objects can have a -_relationships_ field that represents any relations the object has to other -objects. This closely resembles the JSON API definition of links. - -In the _relationships_ objects, all resources have a _self_ link that points to -the resource itself. This allows for easier updating of resources as the reply -URL is included in the response itself. - -The following lists the resources and the types of links each resource can have -in addition to the _self_ link. - -- `services` - Service resource - - - `servers` - - List of servers used by the service - - - `filters` - - List of filters used by the service - -- `monitors` - Monitor resource - - - `servers` - - List of servers used by the monitor - -- `filters` - Filter resource - - - `services` - - List of services that use this filter - -- `servers` - Server resource - - - `services` - - List of services that use this server - - - `monitors` - - List of monitors that use this server - -## Common Request Parameters - -Most of the resources that support GET also support the following -parameters. See the resource documentation for a list of supported request -parameters. - -- `pretty` - - - Pretty-print output. - - If this parameter is set to `true` then the returned objects are - formatted in a more human readable format. All resources support this - parameter. - -- `fields` - - - A list of fields to return. - - This allows the returned object to be filtered so that only needed - parts are returned. The value of this parameter is a comma separated - list of fields to return. - - For example, the parameter `?fields=id,name` would return object which - would only contain the _id_ and _name_ fields. - -- `range` - - - Return a subset of the object array. - - The value of this parameter is the range of objects to return given as - a inclusive range separated by a hyphen. If the size of the array is - less than the end of the range, only the objects between the requested - start of the range and the actual end of the array are returned. This - means that - - For example, the parameter `?range=10-20` would return objects 10 - through 20 from the object array if the actual size of the original - array is greater than or equal to 20. diff --git a/Documentation/REST-API/Resources-Server.md b/Documentation/REST-API/Resources-Server.md index b2bc34569..9309f0cad 100644 --- a/Documentation/REST-API/Resources-Server.md +++ b/Documentation/REST-API/Resources-Server.md @@ -7,7 +7,7 @@ A server resource represents a backend database server. ### Get a server ``` -GET /servers/:name +GET /v1/servers/:name ``` Get a single server. The _:name_ in the URI must be a valid server name with all @@ -20,38 +20,67 @@ whitespace replaced with hyphens. The server names are case-insensitive. ``` Status: 200 OK +``` +```javascript { - "name": "server1", - "parameters": { - "address": "127.0.0.1", - "port": 3000, - "protocol": "MySQLBackend", - "monitoruser": "maxuser", - "monitorpw": "maxpwd" + "links": { + "self": "http://localhost:8989/v1/servers/server1" }, - "status": "Master, Running", - "version": "10.1.22-MariaDB", - "node_id": 3000, - "master_id": -1, - "replication_depth": 0, - "slaves": [ - 3001 - ], - "statictics": { - "connections": 0, - "total_connections": 0, - "active_operations": 0 - }, - "relationships": { - "self": "http://localhost:8989/servers/server1", - "services": [ - "http://localhost:8989/services/RW-Split-Router", - "http://localhost:8989/services/Read-Connection-Router" - ], - "monitors": [ - "http://localhost:8989/monitors/MySQL-Monitor" - ] + "data": { + "id": "server1", // Resource identifier + "type": "servers", // Resource type + "relationships": { // Resource relationships to other resources + "services": { // Services that use this server + "links": { + "self": "http://localhost:8989/v1/services/" + }, + "data": [ // References to service resources + { + "id": "RW-Split-Router", + "type": "services" + }, + { + "id": "Read-Connection-Router", + "type": "services" + } + ] + }, + "monitors": { // The monitor that is monitoring this server + "links": { + "self": "http://localhost:8989/v1/monitors/" + }, + "data": [ + { + "id": "MySQL-Monitor", + "type": "monitors" + } + ] + } + }, + "attributes": { // Resource attributes + "parameters": { // Server parameters + "address": "127.0.0.1", + "port": 3000, + "protocol": "MySQLBackend" + }, + "status": "Master, Running", // Server status string + "version_string": "10.1.22-MariaDB", // Server version + "node_id": 3000, // Server node ID i.e. value of @@server_id + "master_id": -1, + "replication_depth": 0, + "slaves": [ // List of slave server IDs + 3001 + ], + "statictics": { // Server statistics + "connections": 0, + "total_connections": 0, + "active_operations": 0 + } + }, + "links": { // Link to the server itself + "self": "http://localhost:8989/v1/servers/server1" + } } } ``` @@ -69,80 +98,129 @@ Status: 404 Not Found ### Get all servers ``` -GET /servers +GET /v1/servers ``` #### Response -Response contains an array of all servers. +Response contains a resource collection with all servers. ``` Status: 200 OK +``` -[ - { - "name": "server1", - "parameters": { - "address": "127.0.0.1", - "port": 3000, - "protocol": "MySQLBackend", - "monitoruser": "maxuser", - "monitorpw": "maxpwd" - }, - "status": "Master, Running", - "version": "10.1.22-MariaDB", - "node_id": 3000, - "master_id": -1, - "replication_depth": 0, - "slaves": [ - 3001 - ], - "statictics": { - "connections": 0, - "total_connections": 0, - "active_operations": 0 - }, - "relationships": { - "self": "http://localhost:8989/servers/server1", - "services": [ - "http://localhost:8989/services/RW-Split-Router", - "http://localhost:8989/services/Read-Connection-Router" - ], - "monitors": [ - "http://localhost:8989/monitors/MySQL-Monitor" - ] - } +```javascript +{ + "links": { + "self": "http://localhost:8989/v1/servers/" }, - { - "name": "server2", - "parameters": { - "address": "127.0.0.1", - "port": 3001, - "protocol": "MySQLBackend", - "my-weighting-parameter": "3" + "data": [ // List of server resouces + { + "id": "server1", + "type": "servers", + "relationships": { + "services": { + "links": { + "self": "http://localhost:8989/v1/services/" + }, + "data": [ + { + "id": "RW-Split-Router", + "type": "services" + }, + { + "id": "Read-Connection-Router", + "type": "services" + } + ] + }, + "monitors": { + "links": { + "self": "http://localhost:8989/v1/monitors/" + }, + "data": [ + { + "id": "MySQL-Monitor", + "type": "monitors" + } + ] + } + }, + "attributes": { + "parameters": { + "address": "127.0.0.1", + "port": 3000, + "protocol": "MySQLBackend" + }, + "status": "Master, Running", + "version_string": "10.1.22-MariaDB", + "node_id": 3000, + "master_id": -1, + "replication_depth": 0, + "slaves": [ + 3001 + ], + "statictics": { + "connections": 0, + "total_connections": 0, + "active_operations": 0 + } + }, + "links": { + "self": "http://localhost:8989/v1/servers/server1" + } }, - "status": "Slave, Running", - "version": "10.1.22-MariaDB", - "node_id": 3001, - "master_id": 3000, - "replication_depth": 1, - "slaves": [], - "statictics": { - "connections": 0, - "total_connections": 0, - "active_operations": 0 - }, - "relationships": { - "self": "http://localhost:8989/servers/server2", - "services": [ - "http://localhost:8989/services/RW-Split-Router" - ], - "monitors": [ - "http://localhost:8989/monitors/MySQL-Monitor" - ] + { + "id": "server2", + "type": "servers", + "relationships": { + "services": { + "links": { + "self": "http://localhost:8989/v1/services/" + }, + "data": [ + { + "id": "RW-Split-Router", + "type": "services" + } + ] + }, + "monitors": { + "links": { + "self": "http://localhost:8989/v1/monitors/" + }, + "data": [ + { + "id": "MySQL-Monitor", + "type": "monitors" + } + ] + } + }, + "attributes": { + "parameters": { + "address": "127.0.0.1", + "port": 3001, + "protocol": "MySQLBackend" + }, + "status": "Slave, Running", + "version_string": "10.1.22-MariaDB", + "node_id": 3001, + "master_id": 3000, + "replication_depth": 1, + "slaves": [], + "statictics": { + "connections": 0, + "total_connections": 0, + "active_operations": 0 + } + }, + "links": { + "self": "http://localhost:8989/v1/servers/server2" + } } - } -] + ] +} ``` #### Supported Request Parameter @@ -152,7 +230,7 @@ Status: 200 OK ### Create a server ``` -POST /servers +POST /v1/servers ``` Create a new server by defining the resource. The posted object must define the @@ -160,60 +238,89 @@ _name_ field with the name of the server and the _parameters_ field with JSON object containing values for the _address_ and _port_ parameters. The following is the minimal required JSON object for defining a new server. -``` +```javascript { - "name": "test-server", - "parameters": { - "address": "127.0.0.1", - "port": 3003 + "data": { + "id": "server3", + "type": "servers", + "attributes": { + "parameters": { + "address": "127.0.0.1", + "port": 3003, + "protocol": "MySQLBackend" + } + } } } ``` +The relationships of a server can also be defined at creation time. This allows +new servers to be created and immediately taken into use. + +```javascript +{ + "data": { + "id": "server4", + "type": "servers", + "attributes": { + "parameters": { + "address": "127.0.0.1", + "port": 3002, + "protocol": "MySQLBackend" + } + }, + "relationships": { + "services": { + "data": [ + { + "id": "RW-Split-Router", + "type": "services" + }, + { + "id": "Read-Connection-Router", + "type": "services" + } + ] + }, + "monitors": { + "data": [ + { + "id": "MySQL-Monitor", + "type": "monitors" + } + ] + } + } + } +} +``` + +The following parameters can be defined when a server is being created. + +- [address](../Getting-Started/Configuration-Guide.md#address) +- [port](../Getting-Started/Configuration-Guide.md#port) +- [protocol](../Getting-Started/Configuration-Guide.md#protocol) +- [authenticator](../Getting-Started/Configuration-Guide.md#authenticator) +- [authenticator_options](../Getting-Started/Configuration-Guide.md#authenticator-options) + #### Response -Response contains the created resource. +Server created: ``` -Status: 200 OK - -{ - "name": "test-server", - "parameters": { - "address": "127.0.0.1", - "port": 3003, - "protocol": "MySQLBackend" - }, - "status": "Running", - "node_id": -1, - "master_id": -1, - "replication_depth": -1, - "slaves": [], - "statictics": { - "connections": 0, - "total_connections": 0, - "active_operations": 0 - }, - "relationships": { - "self": "http://localhost:8989/servers/test-server" - } -} +Status: 204 No Content ``` Invalid JSON body: ``` -Status: 400 Bad Request +Status: 403 Forbidden ``` -#### Supported Request Parameter - -- `pretty` - ### Update a server ``` -PUT /servers/:name +PUT /v1/servers/:name ``` The _:name_ in the URI must map to a server name with all whitespace replaced @@ -249,81 +356,114 @@ _server1_ from the service _RW-Split-Router_. Removing a service from a server is analogous to removing the server from the service. Both unlink the two objects from each other. -``` +Response to `GET /v1/server/server1`: + +```javascript { - "name": "server1", - "parameters": { - "address": "127.0.0.1", - "port": 3000, - "protocol": "MySQLBackend", - "monitoruser": "maxuser", - "monitorpw": "maxpwd" + "links": { + "self": "http://localhost:8989/v1/servers/server1" }, - "status": "Master, Running", - "version": "10.1.22-MariaDB", - "node_id": 3000, - "master_id": -1, - "replication_depth": 0, - "slaves": [ - 3001 - ], - "statictics": { - "connections": 0, - "total_connections": 0, - "active_operations": 0 - }, - "relationships": { - "self": "http://localhost:8989/servers/server1", - "services": [ - "http://localhost:8989/services/RW-Split-Router", // This value is removed - "http://localhost:8989/services/Read-Connection-Router" - ], - "monitors": [ - "http://localhost:8989/monitors/MySQL-Monitor" - ] + "data": { + "id": "server1", + "type": "servers", + "relationships": { + "services": { + "links": { + "self": "http://localhost:8989/v1/services/" + }, + "data": [ + { + "id": "RW-Split-Router", // We'll remove this service + "type": "services" + }, + { + "id": "Read-Connection-Router", + "type": "services" + } + ] + }, + "monitors": { + "links": { + "self": "http://localhost:8989/v1/monitors/" + }, + "data": [ + { + "id": "MySQL-Monitor", + "type": "monitors" + } + ] + } + }, + "attributes": { + "parameters": { + "address": "127.0.0.1", + "port": 3000, + "protocol": "MySQLBackend" + }, + "status": "Master, Running", + "version_string": "10.1.22-MariaDB", + "node_id": 3000, + "master_id": -1, + "replication_depth": 0, + "slaves": [ + 3001, + 3002 + ], + "statictics": { + "connections": 0, + "total_connections": 0, + "active_operations": 0 + } + }, + "links": { + "self": "http://localhost:8989/v1/servers/server1" + } } } ``` +Request for `PUT /v1/server/server1`: + +```javascript +{ + "data": { + "id": "server1", + "type": "servers", + "relationships": { + "services": { + "data": [ + { + "id": "Read-Connection-Router", + "type": "services" + } + ] + }, + "monitors": { + "data": [ + { + "id": "MySQL-Monitor", + "type": "monitors" + } + ] + } + } + } +} +``` + +The current implementation accepts both PUT and PATCH requests with partially +defined resources as request body. If parts of the resource are not defined +(e.g. the `attributes` field in the above example), those parts of the resource +are not modified. All parts that are defined are interpreted as the new +definition of those part of the resource. In the above example, the +`relationships` of the resource are completely redefined. + #### Response -Response contains the modified resource. +Server modified: ``` -Status: 200 OK - -{ - "name": "server1", - "parameters": { - "address": "127.0.0.1", - "port": 3000, - "protocol": "MySQLBackend", - "monitoruser": "maxuser", - "monitorpw": "maxpwd" - }, - "status": "Master, Running", - "version": "10.1.22-MariaDB", - "node_id": 3000, - "master_id": -1, - "replication_depth": 0, - "slaves": [ - 3001 - ], - "statictics": { - "connections": 0, - "total_connections": 0, - "active_operations": 0 - }, - "relationships": { - "self": "http://localhost:8989/servers/server1", - "services": [ - "http://localhost:8989/services/Read-Connection-Router" - ], - "monitors": [ - "http://localhost:8989/monitors/MySQL-Monitor" - ] - } -} +Status: 204 No Content ``` Server not found: @@ -335,7 +475,7 @@ Status: 404 Not Found Invalid JSON body: ``` -Status: 400 Bad Request +Status: 403 Forbidden ``` #### Supported Request Parameter @@ -345,14 +485,13 @@ Status: 400 Bad Request ### Destroy a server ``` -DELETE /servers/:name +DELETE /v1/servers/:name ``` The _:name_ in the URI must map to a server name with all whitespace replaced with hyphens. -A server can only be deleted if the only relations in the _relationships_ object -is the _self_ link. +A server can only be deleted if it is not used by any services or monitors. #### Response @@ -371,7 +510,7 @@ Status: 404 Not Found Server is in use: ``` -Status: 400 Bad Request +Status: 403 Forbidden ``` # **TODO:** Implement the following features @@ -381,56 +520,20 @@ Status: 400 Bad Request Get all connections that are connected to a server. ``` -GET /servers/:name/connections +GET /v1/servers/:name/connections ``` #### Response -``` -Status: 200 OK - -[ - { - "state": "DCB in the polling loop", - "role": "Backend Request Handler", - "server": "/servers/db-serv-01", - "service": "/services/my-service", - "statistics": { - "reads": 2197 - "writes": 1562 - "buffered_writes": 0 - "high_water_events": 0 - "low_water_events": 0 - } - }, - { - "state": "DCB in the polling loop", - "role": "Backend Request Handler", - "server": "/servers/db-serv-01", - "service": "/services/my-second-service" - "statistics": { - "reads": 0 - "writes": 0 - "buffered_writes": 0 - "high_water_events": 0 - "low_water_events": 0 - } - } -] -``` - #### Supported Request Parameter -- `fields` -- `range` - ### Close all connections to a server Close all connections to a particular server. This will forcefully close all backend connections. ``` -DELETE /servers/:name/connections +DELETE /v1/servers/:name/connections ``` #### Response From 18c71a8ebd02f7ab03d9585033e912fb6f45c098 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 11 May 2017 15:18:51 +0300 Subject: [PATCH 55/55] MXS-1220: Fix build failure with older GCC Older GCC versions seem to have a bug where `struct sockaddr_in` pointers cannot be accessed directly and need to be dereferenced first. --- server/core/admin.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/core/admin.cc b/server/core/admin.cc index 7f2f4ca11..bd97bdd94 100644 --- a/server/core/admin.cc +++ b/server/core/admin.cc @@ -229,12 +229,12 @@ static bool host_to_sockaddr(const char* host, uint16_t port, struct sockaddr_st if (addr->ss_family == AF_INET) { struct sockaddr_in *ip = (struct sockaddr_in*)addr; - ip->sin_port = htons(port); + (*ip).sin_port = htons(port); } else if (addr->ss_family == AF_INET6) { struct sockaddr_in6 *ip = (struct sockaddr_in6*)addr; - ip->sin6_port = htons(port); + (*ip).sin6_port = htons(port); } }