database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Enhancement](HttpServer) Support https interface (#16834)

Browse Source 
1. Organize http documents
2. Add http interface authentication for FE
3. **Support https interface for FE**
4. Provide authentication interface
5. Add http interface authentication for BE
6. Support https interface for BE
This commit is contained in:
yongjinhou
2023-04-03 14:18:17 +08:00
committed by GitHub
parent ecd3fd07f6
commit aff260c06f
26 changed files with 375 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
fe/fe-core/src/main/java/org/apache/doris/DorisFE.java
View File

@ -158,10 +158,16 @@ public class DorisFE {
if (options.enableHttpServer) {
HttpServer httpServer = new HttpServer();
httpServer.setPort(Config.http_port);
httpServer.setHttpsPort(Config.https_port);
httpServer.setMaxHttpPostSize(Config.jetty_server_max_http_post_size);
httpServer.setAcceptors(Config.jetty_server_acceptors);
httpServer.setSelectors(Config.jetty_server_selectors);
httpServer.setWorkers(Config.jetty_server_workers);
httpServer.setKeyStorePath(Config.key_store_path);
httpServer.setKeyStorePassword(Config.key_store_password);
httpServer.setKeyStoreType(Config.key_store_type);
httpServer.setKeyStoreAlias(Config.key_store_alias);
httpServer.setEnableHttps(Config.enable_https);
httpServer.setMaxThreads(Config.jetty_threadPool_maxThreads);
httpServer.setMinThreads(Config.jetty_threadPool_minThreads);
httpServer.setMaxHttpHeaderSize(Config.jetty_server_max_http_header_size);
@ -192,6 +198,10 @@ public class DorisFE {
"Http port", NetUtils.HTTP_PORT_SUGGESTION)) {
throw new IOException("port " + Config.http_port + " already in use");
}
if (Config.enable_https && !NetUtils.isPortAvailable(FrontendOptions.getLocalHostAddress(),
Config.https_port, "Https port", NetUtils.HTTPS_PORT_SUGGESTION)) {
throw new IOException("port " + Config.https_port + " already in use");
}
if (!NetUtils.isPortAvailable(FrontendOptions.getLocalHostAddress(), Config.query_port,
"Query port", NetUtils.QUERY_PORT_SUGGESTION)) {
throw new IOException("port " + Config.query_port + " already in use");

2
fe/fe-core/src/main/java/org/apache/doris/common/util/NetUtils.java
View File

@ -45,6 +45,8 @@ public class NetUtils {
public static final String QUERY_PORT_SUGGESTION = "Please change the 'query_port' in fe.conf and try again.";
public static final String HTTP_PORT_SUGGESTION = "Please change the 'http_port' in fe.conf and try again. "
+ "But you need to make sure that ALL FEs http_port are same.";
public static final String HTTPS_PORT_SUGGESTION = "Please change the 'https_port' in fe.conf and try again. "
+ "But you need to make sure that ALL FEs https_port are same.";
public static final String RPC_PORT_SUGGESTION = "Please change the 'rpc_port' in fe.conf and try again.";
// Target format is "host:port"

50
fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java
View File

@ -36,13 +36,19 @@ import java.util.Map;
@EnableConfigurationProperties
@ServletComponentScan
public class HttpServer extends SpringBootServletInitializer {
private int port;
private int httpsPort;
private int acceptors;
private int selectors;
private int maxHttpPostSize;
private int workers;
private String keyStorePath;
private String keyStorePassword;
private String keyStoreType;
private String keyStoreAlias;
private boolean enableHttps;
private int minThreads;
private int maxThreads;
private int maxHttpHeaderSize;
@ -91,6 +97,30 @@ public class HttpServer extends SpringBootServletInitializer {
this.port = port;
}
public void setHttpsPort(int httpsPort) {
this.httpsPort = httpsPort;
}
public void setKeyStorePath(String keyStorePath) {
this.keyStorePath = keyStorePath;
}
public void setKeyStorePassword(String keyStorePassword) {
this.keyStorePassword = keyStorePassword;
}
public void setKeyStoreType(String keyStoreType) {
this.keyStoreType = keyStoreType;
}
public void setKeyStoreAlias(String keyStoreAlias) {
this.keyStoreAlias = keyStoreAlias;
}
public void setEnableHttps(boolean enableHttps) {
this.enableHttps = enableHttps;
}
@Override
protected SpringApplicationBuilder configure(SpringApplicationBuilder application) {
return application.sources(HttpServer.class);
@ -98,7 +128,19 @@ public class HttpServer extends SpringBootServletInitializer {
public void start() {
Map<String, Object> properties = new HashMap<>();
properties.put("server.port", port);
if (enableHttps) {
properties.put("server.http.port", port);
properties.put("server.port", httpsPort);
// ssl config
properties.put("server.ssl.key-store", keyStorePath);
properties.put("server.ssl.key-store-password", keyStorePassword);
properties.put("server.ssl.key-store-type", keyStoreType);
properties.put("server.ssl.keyalias", keyStoreAlias);
properties.put("server.ssl.enabled", enableHttps);
} else {
properties.put("server.port", port);
properties.put("server.ssl.enabled", enableHttps);
}
if (FrontendOptions.isBindIPV6()) {
properties.put("server.address", "::0");
} else {
@ -109,14 +151,14 @@ public class HttpServer extends SpringBootServletInitializer {
properties.put("spring.http.encoding.charset", "UTF-8");
properties.put("spring.http.encoding.enabled", true);
properties.put("spring.http.encoding.force", true);
//enable jetty config
// enable jetty config
properties.put("server.jetty.acceptors", this.acceptors);
properties.put("server.jetty.max-http-post-size", this.maxHttpPostSize);
properties.put("server.jetty.selectors", this.selectors);
properties.put("server.jetty.threadPool.maxThreads", this.maxThreads);
properties.put("server.jetty.threadPool.minThreads", this.minThreads);
properties.put("server.max-http-header-size", this.maxHttpHeaderSize);
//Worker thread pool is not set by default, set according to your needs
// Worker thread pool is not set by default, set according to your needs
if (this.workers > 0) {
properties.put("server.jetty.workers", this.workers);
}

54
fe/fe-core/src/main/java/org/apache/doris/httpv2/config/HttpToHttpsJettyConfig.java Normal file
View File

@ -0,0 +1,54 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.httpv2.config;
import org.eclipse.jetty.security.ConstraintMapping;
import org.eclipse.jetty.security.ConstraintSecurityHandler;
import org.eclipse.jetty.util.security.Constraint;
import org.eclipse.jetty.webapp.AbstractConfiguration;
import org.eclipse.jetty.webapp.WebAppContext;
public class HttpToHttpsJettyConfig extends AbstractConfiguration {
@Override
public void configure(WebAppContext context) throws Exception {
Constraint constraint = new Constraint();
constraint.setDataConstraint(Constraint.DC_CONFIDENTIAL);
ConstraintSecurityHandler handler = new ConstraintSecurityHandler();
ConstraintMapping mappingGet = new ConstraintMapping();
mappingGet.setConstraint(constraint);
mappingGet.setPathSpec("/*");
mappingGet.setMethod("GET");
handler.addConstraintMapping(mappingGet);
ConstraintMapping mappingDel = new ConstraintMapping();
mappingDel.setConstraint(constraint);
mappingDel.setPathSpec("/*");
mappingDel.setMethod("DELETE");
handler.addConstraintMapping(mappingDel);
ConstraintMapping mappingRest = new ConstraintMapping();
mappingRest.setConstraint(constraint);
mappingRest.setPathSpec("/rest/*");
mappingRest.setMethod("POST");
handler.addConstraintMapping(mappingRest);
context.setSecurityHandler(handler);
}
}

56
fe/fe-core/src/main/java/org/apache/doris/httpv2/config/WebServerFactoryCustomizerConfig.java Normal file
View File

@ -0,0 +1,56 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.httpv2.config;
import org.apache.doris.common.Config;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.ServerConnector;
import org.springframework.boot.web.embedded.jetty.ConfigurableJettyWebServerFactory;
import org.springframework.boot.web.embedded.jetty.JettyServletWebServerFactory;
import org.springframework.boot.web.server.WebServerFactoryCustomizer;
import org.springframework.context.annotation.Configuration;
import java.util.Collections;
@Configuration
public class WebServerFactoryCustomizerConfig implements WebServerFactoryCustomizer<ConfigurableJettyWebServerFactory> {
@Override
public void customize(ConfigurableJettyWebServerFactory factory) {
if (Config.enable_https) {
((JettyServletWebServerFactory) factory).setConfigurations(
Collections.singleton(new HttpToHttpsJettyConfig())
);
factory.addServerCustomizers(
server -> {
HttpConfiguration httpConfiguration = new HttpConfiguration();
httpConfiguration.setSecurePort(Config.https_port);
httpConfiguration.setSecureScheme("https");
ServerConnector connector = new ServerConnector(server);
connector.addConnectionFactory(new HttpConnectionFactory(httpConfiguration));
connector.setPort(Config.http_port);
server.addConnector(connector);
}
);
}
}
}

7
fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/BaseController.java
View File

@ -296,6 +296,11 @@ public class BaseController {
}
protected String getCurrentFrontendURL() {
return "http://" + FrontendOptions.getLocalHostAddress() + ":" + Config.http_port;
if (Config.enable_https) {
// this could be the result of redirection.
return "https://" + FrontendOptions.getLocalHostAddress() + ":" + Config.https_port;
} else {
return "http://" + FrontendOptions.getLocalHostAddress() + ":" + Config.http_port;
}
}
}

8
fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/ColocateMetaService.java
View File

@ -103,6 +103,10 @@ public class ColocateMetaService extends RestBaseController {
@RequestMapping(path = "/api/colocate/group_stable", method = {RequestMethod.POST, RequestMethod.DELETE})
public Object group_stable(HttpServletRequest request, HttpServletResponse response)
throws DdlException {
if (needRedirect(request.getScheme())) {
return redirectToHttps(request);
}
executeWithoutPassword(request, response);
GroupId groupId = checkAndGetGroupId(request);
@ -118,6 +122,10 @@ public class ColocateMetaService extends RestBaseController {
@RequestMapping(path = "/api/colocate/bucketseq", method = RequestMethod.POST)
public Object bucketseq(HttpServletRequest request, HttpServletResponse response, @RequestBody String meta)
throws DdlException {
if (needRedirect(request.getScheme())) {
return redirectToHttps(request);
}
executeWithoutPassword(request, response);
final String clusterName = ConnectContext.get().getClusterName();
GroupId groupId = checkAndGetGroupId(request);

5
fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/CancelLoadAction.java
View File

@ -45,8 +45,11 @@ public class CancelLoadAction extends RestBaseController {
@RequestMapping(path = "/api/{" + DB_KEY + "}/_cancel", method = RequestMethod.POST)
public Object execute(@PathVariable(value = DB_KEY) final String dbName,
HttpServletRequest request, HttpServletResponse response) {
executeCheckPassword(request, response);
if (needRedirect(request.getScheme())) {
return redirectToHttps(request);
}
executeCheckPassword(request, response);
RedirectView redirectView = redirectToMaster(request, response);
if (redirectView != null) {
return redirectView;

16
fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java
View File

@ -59,6 +59,10 @@ public class LoadAction extends RestBaseController {
@RequestMapping(path = "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_load", method = RequestMethod.PUT)
public Object load(HttpServletRequest request, HttpServletResponse response,
@PathVariable(value = DB_KEY) String db, @PathVariable(value = TABLE_KEY) String table) {
if (needRedirect(request.getScheme())) {
return redirectToHttps(request);
}
if (Config.disable_mini_load) {
ResponseEntity entity = ResponseEntityBuilder.notFound("The mini load operation has been"
+ " disabled by default, if you need to add disable_mini_load=false in fe.conf.");
@ -73,6 +77,10 @@ public class LoadAction extends RestBaseController {
public Object streamLoad(HttpServletRequest request,
HttpServletResponse response,
@PathVariable(value = DB_KEY) String db, @PathVariable(value = TABLE_KEY) String table) {
if (needRedirect(request.getScheme())) {
return redirectToHttps(request);
}
executeCheckPassword(request, response);
return executeWithoutPassword(request, response, db, table, true);
}
@ -81,6 +89,10 @@ public class LoadAction extends RestBaseController {
public Object streamLoad2PC(HttpServletRequest request,
HttpServletResponse response,
@PathVariable(value = DB_KEY) String db) {
if (needRedirect(request.getScheme())) {
return redirectToHttps(request);
}
executeCheckPassword(request, response);
return executeStreamLoad2PC(request, db);
}
@ -90,6 +102,10 @@ public class LoadAction extends RestBaseController {
HttpServletResponse response,
@PathVariable(value = DB_KEY) String db,
@PathVariable(value = TABLE_KEY) String table) {
if (needRedirect(request.getScheme())) {
return redirectToHttps(request);
}
executeCheckPassword(request, response);
return executeStreamLoad2PC(request, db);
}

24
fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java
View File

@ -52,6 +52,10 @@ public class MultiAction extends RestBaseController {
public Object multi_desc(
@PathVariable(value = DB_KEY) final String dbName,
HttpServletRequest request, HttpServletResponse response) {
if (needRedirect(request.getScheme())) {
return redirectToHttps(request);
}
try {
executeCheckPassword(request, response);
@ -84,6 +88,10 @@ public class MultiAction extends RestBaseController {
@PathVariable(value = DB_KEY) final String dbName,
HttpServletRequest request, HttpServletResponse response)
throws DdlException {
if (needRedirect(request.getScheme())) {
return redirectToHttps(request);
}
try {
executeCheckPassword(request, response);
execEnv = ExecuteEnv.getInstance();
@ -110,6 +118,10 @@ public class MultiAction extends RestBaseController {
@PathVariable(value = DB_KEY) final String dbName,
HttpServletRequest request, HttpServletResponse response)
throws DdlException {
if (needRedirect(request.getScheme())) {
return redirectToHttps(request);
}
try {
executeCheckPassword(request, response);
execEnv = ExecuteEnv.getInstance();
@ -154,6 +166,10 @@ public class MultiAction extends RestBaseController {
public Object multi_unload(
@PathVariable(value = DB_KEY) final String dbName,
HttpServletRequest request, HttpServletResponse response) {
if (needRedirect(request.getScheme())) {
return redirectToHttps(request);
}
try {
executeCheckPassword(request, response);
execEnv = ExecuteEnv.getInstance();
@ -188,6 +204,10 @@ public class MultiAction extends RestBaseController {
@PathVariable(value = DB_KEY) final String dbName,
HttpServletRequest request, HttpServletResponse response)
throws DdlException {
if (needRedirect(request.getScheme())) {
return redirectToHttps(request);
}
try {
executeCheckPassword(request, response);
execEnv = ExecuteEnv.getInstance();
@ -222,6 +242,10 @@ public class MultiAction extends RestBaseController {
@PathVariable(value = DB_KEY) final String dbName,
HttpServletRequest request, HttpServletResponse response)
throws DdlException {
if (needRedirect(request.getScheme())) {
return redirectToHttps(request);
}
try {
executeCheckPassword(request, response);
execEnv = ExecuteEnv.getInstance();

18
fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.httpv2.rest;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.Env;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.Config;
import org.apache.doris.httpv2.controller.BaseController;
import org.apache.doris.httpv2.exception.UnauthorizedException;
import org.apache.doris.qe.ConnectContext;
@ -30,6 +31,7 @@ import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.http.HttpStatus;
import org.springframework.web.servlet.view.RedirectView;
import java.io.BufferedInputStream;
@ -172,4 +174,20 @@ public class RestBaseController extends BaseController {
}
return fullDbName;
}
public boolean needRedirect(String scheme) {
return Config.enable_https && "http".equalsIgnoreCase(scheme);
}
public Object redirectToHttps(HttpServletRequest request) {
String serverName = request.getServerName();
String uri = request.getRequestURI();
String query = request.getQueryString();
query = query == null ? "" : query;
String newUrl = "https://" + serverName + ":" + Config.https_port + uri + "?" + query;
LOG.info("redirect to new url: {}", newUrl);
RedirectView redirectView = new RedirectView(newUrl);
redirectView.setStatusCode(HttpStatus.TEMPORARY_REDIRECT);
return redirectView;
}
}

10
fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java
View File

@ -84,6 +84,10 @@ public class StmtExecutionAction extends RestBaseController {
@RequestMapping(path = "/api/query/{" + NS_KEY + "}/{" + DB_KEY + "}", method = {RequestMethod.POST})
public Object executeSQL(@PathVariable(value = NS_KEY) String ns, @PathVariable(value = DB_KEY) String dbName,
HttpServletRequest request, HttpServletResponse response, @RequestBody String body) {
if (needRedirect(request.getScheme())) {
return redirectToHttps(request);
}
ActionAuthorizationInfo authInfo = checkWithCookie(request, response, false);
String fullDbName = getFullDbName(dbName);
if (Config.enable_all_http_auth) {
@ -125,8 +129,12 @@ public class StmtExecutionAction extends RestBaseController {
* @return plain text of create table stmts
*/
@RequestMapping(path = "/api/query_schema/{" + NS_KEY + "}/{" + DB_KEY + "}", method = {RequestMethod.POST})
public String querySchema(@PathVariable(value = NS_KEY) String ns, @PathVariable(value = DB_KEY) String dbName,
public Object querySchema(@PathVariable(value = NS_KEY) String ns, @PathVariable(value = DB_KEY) String dbName,
HttpServletRequest request, HttpServletResponse response, @RequestBody String sql) {
if (needRedirect(request.getScheme())) {
return redirectToHttps(request);
}
checkWithCookie(request, response, false);
if (ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) {

4
fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java
View File

@ -84,6 +84,10 @@ public class TableQueryPlanAction extends RestBaseController {
@PathVariable(value = DB_KEY) final String dbName,
@PathVariable(value = TABLE_KEY) final String tblName,
HttpServletRequest request, HttpServletResponse response) {
if (needRedirect(request.getScheme())) {
return redirectToHttps(request);
}
executeCheckPassword(request, response);
// just allocate 2 slot for top holder map
Map<String, Object> resultMap = new HashMap<>(4);

3
fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/UploadAction.java
View File

@ -75,6 +75,9 @@ public class UploadAction extends RestBaseController {
@PathVariable(value = TABLE_KEY) String tblName,
@RequestParam("file") MultipartFile file,
HttpServletRequest request, HttpServletResponse response) {
if (needRedirect(request.getScheme())) {
return redirectToHttps(request);
}
checkWithCookie(request, response, false);

4
fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/ImportAction.java
View File

@ -76,6 +76,10 @@ public class ImportAction extends RestBaseController {
@RequestMapping(path = "/api/import/file_review", method = RequestMethod.POST)
public Object fileReview(@RequestBody FileReviewRequestVo body,
HttpServletRequest request, HttpServletResponse response) {
if (needRedirect(request.getScheme())) {
return redirectToHttps(request);
}
if (Config.enable_all_http_auth) {
executeCheckPassword(request, response);
}