From b41934864e3f4e31acbc4d433a7eba7411d2e83f Mon Sep 17 00:00:00 2001 From: Zhengguo Yang Date: Fri, 6 Jan 2023 09:15:08 +0800 Subject: [PATCH] [enhancement](frontendservice) add retry when create connection to frontend service (#15635) --- be/src/runtime/client_cache.h | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/be/src/runtime/client_cache.h b/be/src/runtime/client_cache.h index cbef1631cc..a8d1f47b33 100644 --- a/be/src/runtime/client_cache.h +++ b/be/src/runtime/client_cache.h @@ -149,22 +149,25 @@ template class ClientConnection { public: ClientConnection(ClientCache* client_cache, const TNetworkAddress& address, Status* status) - : _client_cache(client_cache), _client(nullptr) { - *status = _client_cache->get_client(address, &_client, 0); - - if (status->ok()) { - DCHECK(_client != nullptr); - } - } + : ClientConnection(client_cache, address, 0, status, 3) {} ClientConnection(ClientCache* client_cache, const TNetworkAddress& address, int timeout_ms, - Status* status) + Status* status, int max_retries = 3) : _client_cache(client_cache), _client(nullptr) { - *status = _client_cache->get_client(address, &_client, timeout_ms); - - if (status->ok()) { - DCHECK(_client != nullptr); - } + int num_retries = 0; + do { + *status = _client_cache->get_client(address, &_client, timeout_ms); + if (status->ok()) { + DCHECK(_client != nullptr); + break; + } + if (num_retries++ < max_retries) { + // exponential backoff retry with starting delay of 500ms + usleep(500000 * (1 << num_retries)); + LOG(INFO) << "Failed to get client from cache: " << status->to_string() + << ", retrying[" << num_retries << "]..."; + } + } while (num_retries < max_retries); } ~ClientConnection() {