[Improvement](rpc) set grpc channel's keepAliveTime and remove proxy … (#38381)

…on InterruptedExcep… (#37304)

## Proposed changes
1. set grpc channel's keepAliveTime
2. remove proxy on InterruptedException/TimeoutException to avoid
channel unavailable
pick from #37304
## Proposed changes

Issue Number: close #xxx

<!--Describe your changes.-->

fe/fe-common/src/main/java/org/apache/doris/common/Config.java

@@ -1464,6 +1464,14 @@ public class Config extends ConfigBase {
     @ConfField
     public static int grpc_threadmgr_threads_nums = 4096;
 
+    /**
+     * sets the time without read activity before sending a keepalive ping
+     * the smaller the value, the sooner the channel is unavailable, but it will increase network io
+     */
+    @ConfField(description = { "设置grpc连接发送 keepalive ping 之前没有数据传输的时间。",
+            "The time without grpc read activity before sending a keepalive ping" })
+    public static int grpc_keep_alive_second = 10;
+
     /**
      * Used to set minimal number of replication per tablet.
      */

fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java

@@ -1219,6 +1219,7 @@ public class Coordinator implements CoordInterface {
             } catch (InterruptedException e) {
                 exception = e;
                 code = TStatusCode.INTERNAL_ERROR;
+                triple.getMiddle().removeProxy(triple.getLeft().brpcAddr);
             } catch (TimeoutException e) {
                 exception = e;
                 errMsg = String.format(
@@ -1226,6 +1227,7 @@ public class Coordinator implements CoordInterface {
                                             operation, queryOptions.getExecutionTimeout(), timeoutMs / 1000);
                 LOG.warn("Query {} {}", DebugUtil.printId(queryId), errMsg);
                 code = TStatusCode.TIMEOUT;
+                triple.getMiddle().removeProxy(triple.getLeft().brpcAddr);
             }
 
             if (code != TStatusCode.OK) {

fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java

@@ -46,7 +46,7 @@ public class BackendServiceClient {
     public BackendServiceClient(TNetworkAddress address, Executor executor) {
         this.address = address;
         channel = NettyChannelBuilder.forAddress(address.getHostname(), address.getPort())
-                .executor(executor)
+                .executor(executor).keepAliveTime(Config.grpc_keep_alive_second, TimeUnit.SECONDS)
                 .flowControlWindow(Config.grpc_max_message_size_bytes)
                 .keepAliveWithoutCalls(true)
                 .maxInboundMessageSize(Config.grpc_max_message_size_bytes).enableRetry().maxRetryAttempts(MAX_RETRY_NUM)