From 5ab5e914e79b2ca48aed23938128a61cfbcafc89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 28 Jun 2019 09:13:11 +0300 Subject: [PATCH 01/14] MXS-2582: Deep-copy PS buffers in RWBackend::write Deep-copying prevents subsequent modifications done by the caller from affecting the data that can be potentially stored in the write queue of the backend's DCB. --- server/modules/protocol/MySQL/rwbackend.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/modules/protocol/MySQL/rwbackend.cc b/server/modules/protocol/MySQL/rwbackend.cc index 1b460e59a..0840c6cb6 100644 --- a/server/modules/protocol/MySQL/rwbackend.cc +++ b/server/modules/protocol/MySQL/rwbackend.cc @@ -88,6 +88,14 @@ bool RWBackend::write(GWBUF* buffer, response_type type) if (mxs_mysql_is_ps_command(cmd)) { + // We need to completely separate the buffer this backend owns and the one that the caller owns to + // prevent any modifications from affecting the one that was written through this backend. If the + // buffer gets placed into the write queue of the DCB, subsequent modifications to the original buffer + // would be propagated to the one this backend owns. + GWBUF* tmp = gwbuf_deep_clone(buffer); + gwbuf_free(buffer); + buffer = tmp; + uint32_t id = mxs_mysql_extract_ps_id(buffer); BackendHandleMap::iterator it = m_ps_handles.find(id); From 74d15cee0980bff77f1809e3d0a61eb41435cc1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 27 Jun 2019 18:32:27 +0300 Subject: [PATCH 02/14] MXS-2562: Always send errors with sequence number 1 As is explained in MDEV-19893: Client reads from socket, gets the packet from 1. with seqno=0, which it does not expect, since seqno is supposed to be incremented. Client complains, throws tantrums and exceptions. To cater for clients that do not expect out-of-bound messages (i.e. server-initiated packets with seqno 0), all messages generated by MaxScale should use at least sequence number 1. --- server/modules/protocol/MySQL/mariadbclient/mysql_client.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc b/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc index 67be3016a..dc1526d55 100644 --- a/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc +++ b/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc @@ -1594,7 +1594,7 @@ static int gw_client_hangup_event(DCB* dcb) errmsg += ": " + extra; } - modutil_send_mysql_err_packet(dcb, 0, 0, 1927, "08S01", errmsg.c_str()); + modutil_send_mysql_err_packet(dcb, 1, 0, 1927, "08S01", errmsg.c_str()); } dcb_close(dcb); } From 3b6387c95258eb1fffacf5669ac6411c3cc96eb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 27 Jun 2019 23:18:33 +0300 Subject: [PATCH 03/14] MXS-2562: Stop immediately on mid-resultset failure If a server fails mid-resultset, there's not a lot we can do to recover the situation. A few cases could be handled (e.g. generate an ERR if the resultset has proceeded to the row processing stage) but these fall outside the scope of the original issue. --- include/maxscale/protocol/rwbackend.hh | 10 ++++++++++ .../modules/routing/readwritesplit/rwsplitsession.cc | 12 ++++++++++++ 2 files changed, 22 insertions(+) diff --git a/include/maxscale/protocol/rwbackend.hh b/include/maxscale/protocol/rwbackend.hh index bbf8f0c2a..cb89d855d 100644 --- a/include/maxscale/protocol/rwbackend.hh +++ b/include/maxscale/protocol/rwbackend.hh @@ -141,6 +141,16 @@ public: return m_reply_state == REPLY_STATE_DONE; } + /** + * Check if a partial response has been received from the backend + * + * @return True if some parts of the reply have been received + */ + bool reply_has_started() const + { + return m_reply_state != REPLY_STATE_START && m_reply_state != REPLY_STATE_DONE; + } + void process_packets(GWBUF* buffer); void process_reply_start(mxs::Buffer::iterator it); diff --git a/server/modules/routing/readwritesplit/rwsplitsession.cc b/server/modules/routing/readwritesplit/rwsplitsession.cc index b9259df2d..bbdb45297 100644 --- a/server/modules/routing/readwritesplit/rwsplitsession.cc +++ b/server/modules/routing/readwritesplit/rwsplitsession.cc @@ -947,6 +947,18 @@ void RWSplitSession::handleError(GWBUF* errmsgbuf, SRWBackend& backend = get_backend_from_dcb(problem_dcb); mxb_assert(backend->in_use()); + if (backend->reply_has_started()) + { + MXS_ERROR("Server '%s' was lost in the middle of a resultset, cannot continue the session: %s", + backend->name(), extract_error(errmsgbuf).c_str()); + + // This effectively causes an instant termination of the client connection and prevents any errors + // from being sent to the client (MXS-2562). + dcb_close(m_client); + *succp = true; + return; + } + switch (action) { case ERRACT_NEW_CONNECTION: From 40f35618adbc881fdfeb0869fbf4444a68fd1e63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 27 Jun 2019 23:28:46 +0300 Subject: [PATCH 04/14] MXS-2562: Fix out-of-order error during COM_CHANGE_USER If an error is generated while a COM_CHANGE_USER is being done, it would always use the sequence number 1. To properly handle this case and send the correct sequence number, the COM_CHANGE_USER progress needs to be tracked at the session level. The information needs to be shared between the backend and client protocols as the final OK to the COM_CHANGE_USER, with the sequence number 3, is the one that the backend server returns. Only after this response has been received and routed to the client can the COM_CHANGE_USER processing stop. --- include/maxscale/protocol/mysql.h | 17 +++++++++-------- .../MySQL/mariadbbackend/mysql_backend.cc | 3 +++ .../MySQL/mariadbclient/mysql_client.cc | 14 +++++++++++++- server/modules/protocol/MySQL/mysql_common.cc | 2 +- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/include/maxscale/protocol/mysql.h b/include/maxscale/protocol/mysql.h index 350efe07d..e185d1404 100644 --- a/include/maxscale/protocol/mysql.h +++ b/include/maxscale/protocol/mysql.h @@ -142,14 +142,15 @@ typedef enum */ typedef struct mysql_session { - uint8_t client_sha1[MYSQL_SCRAMBLE_LEN]; /*< SHA1(password) */ - char user[MYSQL_USER_MAXLEN + 1]; /*< username */ - char db[MYSQL_DATABASE_MAXLEN + 1]; /*< database */ - int auth_token_len; /*< token length */ - uint8_t* auth_token; /*< token */ - bool correct_authenticator; /*< is session using mysql_native_password? */ - uint8_t next_sequence; /*< Next packet sequence */ - bool auth_switch_sent; /*< Expecting a response to AuthSwitchRequest? */ + uint8_t client_sha1[MYSQL_SCRAMBLE_LEN]; /*< SHA1(password) */ + char user[MYSQL_USER_MAXLEN + 1]; /*< username */ + char db[MYSQL_DATABASE_MAXLEN + 1]; /*< database */ + int auth_token_len; /*< token length */ + uint8_t* auth_token; /*< token */ + bool correct_authenticator; /*< is session using mysql_native_password? */ + uint8_t next_sequence; /*< Next packet sequence */ + bool auth_switch_sent; /*< Expecting a response to AuthSwitchRequest? */ + bool changing_user; /*< True if a COM_CHANGE_USER is in progress */ } MYSQL_session; /** Protocol packing macros. */ diff --git a/server/modules/protocol/MySQL/mariadbbackend/mysql_backend.cc b/server/modules/protocol/MySQL/mariadbbackend/mysql_backend.cc index 9de1843c1..40203f61b 100644 --- a/server/modules/protocol/MySQL/mariadbbackend/mysql_backend.cc +++ b/server/modules/protocol/MySQL/mariadbbackend/mysql_backend.cc @@ -916,6 +916,9 @@ static int gw_read_and_write(DCB* dcb) */ GWBUF_DATA(read_buffer)[3] = 0x3; proto->changing_user = false; + + auto s = (MYSQL_session*)session->client_dcb->data; + s->changing_user = false; } } diff --git a/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc b/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc index dc1526d55..c6fe58f24 100644 --- a/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc +++ b/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc @@ -1594,7 +1594,15 @@ static int gw_client_hangup_event(DCB* dcb) errmsg += ": " + extra; } - modutil_send_mysql_err_packet(dcb, 1, 0, 1927, "08S01", errmsg.c_str()); + int seqno = 1; + + if (dcb->data && ((MYSQL_session*)dcb->data)->changing_user) + { + // In case a COM_CHANGE_USER is in progress, we need to send the error with the seqno 3 + seqno = 3; + } + + modutil_send_mysql_err_packet(dcb, seqno, 0, 1927, "08S01", errmsg.c_str()); } dcb_close(dcb); } @@ -1797,6 +1805,10 @@ static int route_by_statement(MXS_SESSION* session, uint64_t capabilities, GWBUF if (!proto->changing_user && proto->current_command == MXS_COM_CHANGE_USER) { + // Track the COM_CHANGE_USER progress at the session level + auto s = (MYSQL_session*)session->client_dcb->data; + s->changing_user = true; + changed_user = true; send_auth_switch_request_packet(session->client_dcb); diff --git a/server/modules/protocol/MySQL/mysql_common.cc b/server/modules/protocol/MySQL/mysql_common.cc index 4efded961..9f0b7a3b9 100644 --- a/server/modules/protocol/MySQL/mysql_common.cc +++ b/server/modules/protocol/MySQL/mysql_common.cc @@ -38,7 +38,7 @@ uint8_t null_client_sha1[MYSQL_SCRAMBLE_LEN] = ""; MYSQL_session* mysql_session_alloc() { MYSQL_session* ses = (MYSQL_session*)MXS_CALLOC(1, sizeof(MYSQL_session)); - + ses->changing_user = false; return ses; } From 0f7c1aa418052f7c1f679213043a4c782b819d17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 1 Jul 2019 10:18:58 +0300 Subject: [PATCH 05/14] Improve mxs1776_ps_exec_hang Syncing the slaves should prevent replication lag from affecting the test. The added logging will help determine what the error was that caused the failure. --- maxscale-system-test/mxs1776_ps_exec_hang.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/maxscale-system-test/mxs1776_ps_exec_hang.cpp b/maxscale-system-test/mxs1776_ps_exec_hang.cpp index e8d0c0d9e..5dabb3cdf 100644 --- a/maxscale-system-test/mxs1776_ps_exec_hang.cpp +++ b/maxscale-system-test/mxs1776_ps_exec_hang.cpp @@ -56,7 +56,8 @@ void run_test(TestConnections& test, TestCase test_case) mysql_stmt_close(stmt); - test.expect(mysql_query(test.maxscales->conn_rwsplit[0], "SELECT 1") == 0, "Normal queries should work"); + test.expect(mysql_query(test.maxscales->conn_rwsplit[0], "SELECT 1") == 0, + "Normal queries should work: %s", mysql_error(test.maxscales->conn_rwsplit[0])); test.maxscales->disconnect(); } @@ -79,6 +80,7 @@ int main(int argc, char* argv[]) test.try_query(test.maxscales->conn_rwsplit[0], "COMMIT"); test.maxscales->disconnect(); + test.repl->sync_slaves(); vector tests = { From 4954c7f6b74f44fd3208541358b04c62c0cfbd43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 1 Jul 2019 10:25:41 +0300 Subject: [PATCH 06/14] Fix sending of unknown PS error The error was only generated for COM_STMT_EXECUTE commands when all PS commands should trigger it. In addition, large packets would get sent two errors upon the arrival of the trailing end. --- .../routing/readwritesplit/rwsplit_route_stmt.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc b/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc index d88ae1ca7..4a62c693e 100644 --- a/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc +++ b/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc @@ -202,12 +202,7 @@ bool RWSplitSession::route_single_stmt(GWBUF* querybuf) SRWBackend target; - if (command == MXS_COM_STMT_EXECUTE && stmt_id == 0) - { - // Unknown prepared statement ID - succp = send_unknown_ps_error(extract_binary_ps_id(querybuf)); - } - else if (TARGET_IS_ALL(route_target)) + if (TARGET_IS_ALL(route_target)) { succp = handle_target_is_all(route_target, querybuf, command, qtype); } @@ -249,6 +244,11 @@ bool RWSplitSession::route_single_stmt(GWBUF* querybuf) target = m_prev_target; succp = true; } + else if (mxs_mysql_is_ps_command(command) && stmt_id == 0) + { + // Unknown prepared statement ID + succp = send_unknown_ps_error(extract_binary_ps_id(querybuf)); + } else if (TARGET_IS_NAMED_SERVER(route_target) || TARGET_IS_RLAG_MAX(route_target)) { if ((target = handle_hinted_target(querybuf, route_target))) From b2019ea18e063e152814877ce57270454d3cc964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 1 Jul 2019 10:33:18 +0300 Subject: [PATCH 07/14] Correctly reset PS continuation state The state was modified only by PS commands. --- server/core/queryclassifier.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/core/queryclassifier.cc b/server/core/queryclassifier.cc index 84ccd41d4..87b78704d 100644 --- a/server/core/queryclassifier.cc +++ b/server/core/queryclassifier.cc @@ -994,6 +994,9 @@ QueryClassifier::RouteInfo QueryClassifier::update_route_info( uint32_t type_mask = QUERY_TYPE_UNKNOWN; uint32_t stmt_id = 0; + // Reset for every classification + m_ps_continuation = false; + // TODO: It may be sufficient to simply check whether we are in a read-only // TODO: transaction. bool in_read_only_trx = From ce2d3778d5ebefa536c34cf7f3704dd0757b529e Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Mon, 1 Jul 2019 10:42:48 +0300 Subject: [PATCH 08/14] change upgrade_test.sh according changes in MDBCI install_product command The logic of MDBCI 'install_product' command was changed: now it works in the same way as product installation during initial VM start (using Chef). Everything moved to Chef recipe and there is no need in 'setup_repo' command --- BUILD/mdbci/upgrade_test.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/BUILD/mdbci/upgrade_test.sh b/BUILD/mdbci/upgrade_test.sh index 24695b5b0..32d19a579 100755 --- a/BUILD/mdbci/upgrade_test.sh +++ b/BUILD/mdbci/upgrade_test.sh @@ -51,8 +51,7 @@ export sshopt="$scpopt $sshuser@$IP" old_version=`ssh $sshopt "maxscale --version" ` -${mdbci_dir}/mdbci setup_repo --product maxscale_ci --product-version ${target} $name/maxscale -${mdbci_dir}/mdbci install_product --product maxscale_ci $name/maxscale +${mdbci_dir}/mdbci install_product --product maxscale_ci --product-version ${target} $name/maxscale res=$? From fa13ec8c3824629864a0cb90c046c440121ae44c Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Mon, 1 Jul 2019 15:12:04 +0300 Subject: [PATCH 09/14] Apply MDBCI install_product chnages also to reinstall_maxscale() --- maxscale-system-test/testconnections.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/maxscale-system-test/testconnections.cpp b/maxscale-system-test/testconnections.cpp index ea397b7e2..cae23fa88 100644 --- a/maxscale-system-test/testconnections.cpp +++ b/maxscale-system-test/testconnections.cpp @@ -2213,12 +2213,6 @@ int TestConnections::reinstall_maxscales() maxscales->ssh_node(i, "yum remove maxscale -y", true); maxscales->ssh_node(i, "yum clean all", true); - sprintf(sys, "mdbci setup_repo --product maxscale_ci --product-version %s %s/%s_%03d", - target, mdbci_config_name, maxscales->prefix, i); - if (system(sys)) - { - return 1; - } sprintf(sys, "mdbci install_product --product maxscale_ci --product-version %s %s/%s_%03d", target, mdbci_config_name, maxscales->prefix, i); if (system(sys)) From 418a1f5210fc0e9af6f0eea6c824a425f9678659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 1 Jul 2019 16:37:55 +0300 Subject: [PATCH 10/14] MXS-2584: Assert that workers are initialized The initialization must always be done before a call to RoutingWorker::get is done. --- server/core/routingworker.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/core/routingworker.cc b/server/core/routingworker.cc index 86d8cd9a5..b76c1eaf9 100644 --- a/server/core/routingworker.cc +++ b/server/core/routingworker.cc @@ -446,6 +446,8 @@ bool mxs_worker_should_shutdown(MXB_WORKER* pWorker) RoutingWorker* RoutingWorker::get(int worker_id) { + mxb_assert(this_unit.initialized); + if (worker_id == MAIN) { worker_id = this_unit.id_main_worker; From 0b18826fc1fd9d97e67780aa51d5b346a6e7b9a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Wed, 22 May 2019 19:45:55 +0300 Subject: [PATCH 11/14] Fix no_password The test used freed memory. --- maxscale-system-test/no_password.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maxscale-system-test/no_password.cpp b/maxscale-system-test/no_password.cpp index 7557625dd..325a889b9 100644 --- a/maxscale-system-test/no_password.cpp +++ b/maxscale-system-test/no_password.cpp @@ -16,7 +16,7 @@ int main(int argc, char** argv) test.tprintf("MySQL error: %s", mysql_error(mysql)); mysql_close(mysql); - open_conn(test.maxscales->rwsplit_port[0], test.maxscales->IP[0], "testuser", "testpassword", false); + mysql = open_conn(test.maxscales->rwsplit_port[0], test.maxscales->IP[0], "testuser", "testpassword", false); test.add_result(mysql_errno(mysql) == 0, "Connecting to MaxScale should fail"); test.add_result(strstr(mysql_error(mysql), "using password: YES") == NULL, "Missing (using password: YES) error message, got this instead: %s", From b3eb5ccc81a47d98a6fe4a38750ac714a8b95fc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 1 Jul 2019 17:13:09 +0300 Subject: [PATCH 12/14] Fix run_ctrl_c.sh on newer systems Systems that no longer support SysV init scripts and the service command need to use systemctl. --- maxscale-system-test/test_ctrl_c/test_ctrl_c.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maxscale-system-test/test_ctrl_c/test_ctrl_c.sh b/maxscale-system-test/test_ctrl_c/test_ctrl_c.sh index fafff6de3..0eb2d64d8 100755 --- a/maxscale-system-test/test_ctrl_c/test_ctrl_c.sh +++ b/maxscale-system-test/test_ctrl_c/test_ctrl_c.sh @@ -1,6 +1,6 @@ #!/bin/bash -sudo service maxscale stop +sudo systemctl stop maxscale || sudo service maxscale stop hm=`pwd` $hm/start_killer.sh & From 25d134df9c50a797636f16ade652173fda8ba7c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 1 Jul 2019 17:18:22 +0300 Subject: [PATCH 13/14] Too slow a system isn't a test failure If testing cannot be reliably performed, the test should be skipped to minimize unnecessary work that non-deterministic failures cause. --- maxscale-system-test/mxs173_throttle_filter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maxscale-system-test/mxs173_throttle_filter.cpp b/maxscale-system-test/mxs173_throttle_filter.cpp index 871c2c716..1ce5dfd1b 100644 --- a/maxscale-system-test/mxs173_throttle_filter.cpp +++ b/maxscale-system-test/mxs173_throttle_filter.cpp @@ -132,7 +132,7 @@ void gauge_raw_speed(TestConnections& test) std::ostringstream os; os << "The raw speed is too slow, " << rs.qps << "qps, compared to max_qps = " << max_qps << "qps for accurate testing."; - test.add_result(1, "%s", os.str().c_str()); + exit(0); } } From 6b110676902d13102903b5e3d4336624b194cd22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 1 Jul 2019 17:26:17 +0300 Subject: [PATCH 14/14] Lower throttling limit in mxs173_throttle_filter The test appears to fail when the throttling is unable to keep the QPS high enough for the test to pass. To reduce the likelihood of this, lower the limit to 500 QPS. In theory, the minimum delay of one millisecond in the delayed_call limits the filter to a maximum QPS of 1000 as each query would wait for at least a millisecond before being routed. This is yet to be proven but it would explain why the tests are having a hard time approaching that level of QPS. --- .../cnf/maxscale.cnf.template.mxs173_throttle_filter | 2 +- maxscale-system-test/mxs173_throttle_filter.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.mxs173_throttle_filter b/maxscale-system-test/cnf/maxscale.cnf.template.mxs173_throttle_filter index 0ac5943e0..8903c666c 100644 --- a/maxscale-system-test/cnf/maxscale.cnf.template.mxs173_throttle_filter +++ b/maxscale-system-test/cnf/maxscale.cnf.template.mxs173_throttle_filter @@ -56,7 +56,7 @@ port = 4009 [throttle] type = filter module = throttlefilter -max_qps = 1000 +max_qps = 500 throttling_duration = 10000 sampling_duration = 250 continuous_duration = 2000 diff --git a/maxscale-system-test/mxs173_throttle_filter.cpp b/maxscale-system-test/mxs173_throttle_filter.cpp index 1ce5dfd1b..6215b0be7 100644 --- a/maxscale-system-test/mxs173_throttle_filter.cpp +++ b/maxscale-system-test/mxs173_throttle_filter.cpp @@ -15,7 +15,7 @@ DEFINE_EXCEPTION(Whoopsy); // TODO these should be read from maxscale.cnf. Maybe the test-lib should replace // any "###ENV_VAR###", with environment variables so that code and conf can share. -constexpr int max_qps = 1000; +constexpr int max_qps = 500; constexpr float throttling_duration = 10000 / 1000.0; constexpr float sampling_duration = 250 / 1000.0; constexpr float continuous_duration = 2000 / 1000.0;