From bdaef84a105b064daec9da4a487427167760e71b Mon Sep 17 00:00:00 2001 From: Lijia Liu Date: Sat, 1 Aug 2020 17:59:01 +0800 Subject: [PATCH] [FE] [HttpServer] Config netty param in HttpServer (#4225) Now, if the length of URL is longer than 4096 bytes, netty will refuse. The case can be reproduced by constructing a very long URL(longer than 4096bytes) Add 2 http server params: 1. http_max_line_length 2. http_max_header_size --- .../administrator-guide/config/fe_config.md | 10 +++++++++ .../administrator-guide/config/fe_config.md | 10 +++++++++ .../main/java/org/apache/doris/PaloFe.java | 7 +++++- .../java/org/apache/doris/common/Config.java | 10 +++++++++ .../org/apache/doris/http/HttpServer.java | 22 +++++++++++++++++-- 5 files changed, 56 insertions(+), 3 deletions(-) diff --git a/docs/en/administrator-guide/config/fe_config.md b/docs/en/administrator-guide/config/fe_config.md index b62e3f1fad..c03355ccb1 100644 --- a/docs/en/administrator-guide/config/fe_config.md +++ b/docs/en/administrator-guide/config/fe_config.md @@ -333,6 +333,16 @@ This variable is a dynamic configuration, and users can modify the configuration ### `http_port` +HTTP bind port. Defaults to 8030. + +### `http_max_line_length` + +The max length of an HTTP URL. The unit of this configuration is BYTE. Defaults to 4096. + +### `http_max_header_size` + +The max size of allowed HTTP headers. The unit of this configuration is BYTE. Defaults to 8192. + ### `ignore_meta_check` ### `init_connect` diff --git a/docs/zh-CN/administrator-guide/config/fe_config.md b/docs/zh-CN/administrator-guide/config/fe_config.md index e1ad951ce4..ed91bb7f11 100644 --- a/docs/zh-CN/administrator-guide/config/fe_config.md +++ b/docs/zh-CN/administrator-guide/config/fe_config.md @@ -331,6 +331,16 @@ FE 的配置项有两种方式进行配置： ### `http_port` +HTTP服务监听的端口号，默认为8030 + +### `http_max_line_length` + +HTTP服务允许接收请求的URL的最大长度，单位为比特，默认是4096 + +### `http_max_header_size` + +HTTP服务允许接收请求的Header的最大长度，单位为比特，默认是8192 + ### `ignore_meta_check` ### `init_connect` diff --git a/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java b/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java index 2fbae1ffc7..5458b88269 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java +++ b/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java @@ -111,7 +111,12 @@ public class PaloFe { // 3. HttpServer for HTTP Server QeService qeService = new QeService(Config.query_port, Config.mysql_service_nio_enabled, ExecuteEnv.getInstance().getScheduler()); FeServer feServer = new FeServer(Config.rpc_port); - HttpServer httpServer = new HttpServer(Config.http_port); + HttpServer httpServer = new HttpServer( + Config.http_port, + Config.http_max_line_length, + Config.http_max_header_size, + Config.http_max_chunk_size + ); httpServer.setup(); feServer.start(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java index bb16fee6d8..a505b8a285 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java @@ -18,6 +18,7 @@ package org.apache.doris.common; import org.apache.doris.PaloFe; +import org.apache.doris.http.HttpServer; public class Config extends ConfigBase { @@ -284,6 +285,15 @@ public class Config extends ConfigBase { */ @ConfField public static int http_port = 8030; + /* + * Netty http param + */ + @ConfField public static int http_max_line_length = HttpServer.DEFAULT_MAX_LINE_LENGTH; + + @ConfField public static int http_max_header_size = HttpServer.DEFAULT_MAX_HEADER_SIZE; + + @ConfField public static int http_max_chunk_size = HttpServer.DEFAULT_MAX_CHUNK_SIZE; + /** * The backlog_num for netty http server * When you enlarge this backlog_num, you should ensure it's value larger than diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/HttpServer.java b/fe/fe-core/src/main/java/org/apache/doris/http/HttpServer.java index 1d4dcc44d4..1b1df9cf26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/http/HttpServer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/http/HttpServer.java @@ -94,7 +94,18 @@ import io.netty.handler.stream.ChunkedWriteHandler; public class HttpServer { private static final Logger LOG = LogManager.getLogger(HttpServer.class); - private int port; + + /** + * The default netty param, witch is the same as `HttpServerCodec`. + */ + public static final int DEFAULT_MAX_LINE_LENGTH = 4096; + public static final int DEFAULT_MAX_HEADER_SIZE = 8192; + public static final int DEFAULT_MAX_CHUNK_SIZE = 8192; + + private final int port; + private final int maxInitialLineLength; + private final int maxHeaderSize; + private final int maxChunkSize; private ActionController controller; private Thread serverThread; @@ -102,7 +113,14 @@ public class HttpServer { private AtomicBoolean isStarted = new AtomicBoolean(false); public HttpServer(int port) { + this(port, DEFAULT_MAX_LINE_LENGTH, DEFAULT_MAX_HEADER_SIZE, DEFAULT_MAX_CHUNK_SIZE); + } + + public HttpServer(int port, int maxInitialLineLength, int maxHeaderSize, int maxChunkSize) { this.port = port; + this.maxInitialLineLength = maxInitialLineLength; + this.maxHeaderSize = maxHeaderSize; + this.maxChunkSize = maxChunkSize; controller = new ActionController(); } @@ -188,7 +206,7 @@ public class HttpServer { protected class PaloHttpServerInitializer extends ChannelInitializer { @Override protected void initChannel(SocketChannel ch) throws Exception { - ch.pipeline().addLast(new HttpServerCodec()); + ch.pipeline().addLast(new HttpServerCodec(maxInitialLineLength, maxHeaderSize, maxChunkSize)); ch.pipeline().addLast(new DorisHttpPostObjectAggregator(100 * 65536)); ch.pipeline().addLast(new ChunkedWriteHandler()); ch.pipeline().addLast(new HttpServerHandler(controller));