From c3538ca8047db340004564d95b9eaaecf8315083 Mon Sep 17 00:00:00 2001 From: yongjinhou <109586248+yongjinhou@users.noreply.github.com> Date: Fri, 24 Feb 2023 10:59:33 +0800 Subject: [PATCH] [Enhancement](HttpServer) Add http interface authentication (#16571) 1. Organize http documents 2. Add http interface authentication for FE 3. Support https interface for FE 4. Provide authentication interface 5. Add http interface authentication for BE 6. Support https interface for BE --- .../main/java/org/apache/doris/common/Config.java | 5 +++++ .../org/apache/doris/httpv2/meta/MetaService.java | 4 ++++ .../apache/doris/httpv2/rest/BackendsAction.java | 5 +++++ .../doris/httpv2/rest/BootstrapFinishAction.java | 6 +++++- .../doris/httpv2/rest/ExtraBasepathAction.java | 6 +++++- .../doris/httpv2/rest/GetSmallFileAction.java | 8 ++++++++ .../org/apache/doris/httpv2/rest/HealthAction.java | 9 ++++++++- .../apache/doris/httpv2/rest/MetaInfoAction.java | 10 +++++++--- .../apache/doris/httpv2/rest/MetricsAction.java | 7 ++++++- .../org/apache/doris/httpv2/rest/ShowAction.java | 14 ++++++++++++++ .../doris/httpv2/rest/StmtExecutionAction.java | 6 ++++++ .../apache/doris/httpv2/restv2/ImportAction.java | 8 +++++++- .../doris/httpv2/restv2/StatisticAction.java | 5 +++++ 13 files changed, 85 insertions(+), 8 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 251f7ad2d0..826ae98543 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -344,6 +344,11 @@ public class Config extends ConfigBase { */ @ConfField public static int http_port = 8030; + /** + * Whether to enable all http interface authentication + */ + @ConfField public static boolean enable_all_http_auth = false; + /** * Jetty container default configuration * Jetty's thread architecture model is very simple, divided into three thread pools: diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java index c8f3d2306d..15dab7d2e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java @@ -225,6 +225,10 @@ public class MetaService extends RestBaseController { @RequestMapping(value = "/dump", method = RequestMethod.GET) public Object dump(HttpServletRequest request, HttpServletResponse response) throws DdlException { + if (Config.enable_all_http_auth) { + executeCheckPassword(request, response); + } + /* * Before dump, we acquired the catalog read lock and all databases' read lock and all * the jobs' read lock. This will guarantee the consistency of database and job queues. diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/BackendsAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/BackendsAction.java index 0bfdd49147..986a807280 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/BackendsAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/BackendsAction.java @@ -66,6 +66,11 @@ public class BackendsAction extends RestBaseController { @RequestMapping(path = "/api/backends", method = {RequestMethod.GET}) public Object getBackends(HttpServletRequest request, HttpServletResponse response) { + /** + * As required, the interface should require user have GlobalAuth-PrivPredicate.ADMIN permission. + * However, a user who uses spark-doris-connector/flink-doris-connector does not have corresponding permission. + * To ensure that the connector works properly, we do not verify the permission of the interface. + */ executeCheckPassword(request, response); boolean needAlive = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/BootstrapFinishAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/BootstrapFinishAction.java index 45d28ab19d..b2878522ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/BootstrapFinishAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/BootstrapFinishAction.java @@ -46,7 +46,7 @@ import javax.servlet.http.HttpServletResponse; * } */ @RestController -public class BootstrapFinishAction { +public class BootstrapFinishAction extends RestBaseController { private static final String CLUSTER_ID = "cluster_id"; private static final String TOKEN = "token"; @@ -58,6 +58,10 @@ public class BootstrapFinishAction { @RequestMapping(path = "/api/bootstrap", method = RequestMethod.GET) public ResponseEntity execute(HttpServletRequest request, HttpServletResponse response) { + if (Config.enable_all_http_auth) { + executeCheckPassword(request, response); + } + boolean isReady = Env.getCurrentEnv().isReady(); // to json response diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ExtraBasepathAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ExtraBasepathAction.java index e6c7af71e2..5a04c8a4ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ExtraBasepathAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ExtraBasepathAction.java @@ -36,9 +36,13 @@ import javax.servlet.http.HttpServletResponse; * This Api will return the path configured in Config.http_api_extra_base_path. */ @RestController -public class ExtraBasepathAction { +public class ExtraBasepathAction extends RestBaseController { @RequestMapping(path = "/api/basepath", method = RequestMethod.GET) public ResponseEntity execute(HttpServletRequest request, HttpServletResponse response) { + if (Config.enable_all_http_auth) { + executeCheckPassword(request, response); + } + BasepathResponse resp = new BasepathResponse(); resp.path = Config.http_api_extra_base_path; if (Strings.isNullOrEmpty(Config.http_api_extra_base_path)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetSmallFileAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetSmallFileAction.java index 465301a37f..714bd6aed0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetSmallFileAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetSmallFileAction.java @@ -18,8 +18,11 @@ package org.apache.doris.httpv2.rest; import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; import org.apache.doris.common.util.SmallFileMgr; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; import com.google.common.base.Strings; import org.apache.logging.log4j.LogManager; @@ -38,6 +41,11 @@ public class GetSmallFileAction extends RestBaseController { @RequestMapping(path = "/api/get_small_file", method = RequestMethod.GET) public Object execute(HttpServletRequest request, HttpServletResponse response) { + if (Config.enable_all_http_auth) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + } + String token = request.getParameter("token"); String fileIdStr = request.getParameter("file_id"); // check param empty diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/HealthAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/HealthAction.java index 930e304a22..e7ed496487 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/HealthAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/HealthAction.java @@ -18,6 +18,7 @@ package org.apache.doris.httpv2.rest; import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; import org.springframework.web.bind.annotation.RequestMapping; @@ -26,12 +27,18 @@ import org.springframework.web.bind.annotation.RestController; import java.util.HashMap; import java.util.Map; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; @RestController public class HealthAction extends RestBaseController { @RequestMapping(path = "/api/health", method = RequestMethod.GET) - public Object execute() { + public Object execute(HttpServletRequest request, HttpServletResponse response) { + if (Config.enable_all_http_auth) { + executeCheckPassword(request, response); + } + Map result = new HashMap<>(); result.put("total_backend_num", Env.getCurrentSystemInfo().getBackendIds(false).size()); result.put("online_backend_num", Env.getCurrentSystemInfo().getBackendIds(true).size()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetaInfoAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetaInfoAction.java index 05e740722f..236f0fc351 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetaInfoAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetaInfoAction.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.cluster.ClusterNamespace; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; @@ -86,7 +87,8 @@ public class MetaInfoAction extends RestBaseController { public Object getAllDatabases( @PathVariable(value = NS_KEY) String ns, HttpServletRequest request, HttpServletResponse response) { - checkWithCookie(request, response, false); + boolean checkAuth = Config.enable_all_http_auth ? true : false; + checkWithCookie(request, response, checkAuth); // use NS_KEY as catalog, but NS_KEY's default value is 'default_cluster'. if (ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { @@ -133,7 +135,8 @@ public class MetaInfoAction extends RestBaseController { public Object getTables( @PathVariable(value = NS_KEY) String ns, @PathVariable(value = DB_KEY) String dbName, HttpServletRequest request, HttpServletResponse response) { - checkWithCookie(request, response, false); + boolean checkAuth = Config.enable_all_http_auth ? true : false; + checkWithCookie(request, response, checkAuth); if (!ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { return ResponseEntityBuilder.badRequest("Only support 'default_cluster' now"); @@ -209,7 +212,8 @@ public class MetaInfoAction extends RestBaseController { @PathVariable(value = NS_KEY) String ns, @PathVariable(value = DB_KEY) String dbName, @PathVariable(value = TABLE_KEY) String tblName, HttpServletRequest request, HttpServletResponse response) throws UserException { - checkWithCookie(request, response, false); + boolean checkAuth = Config.enable_all_http_auth ? true : false; + checkWithCookie(request, response, checkAuth); if (!ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { return ResponseEntityBuilder.badRequest("Only support 'default_cluster' now"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetricsAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetricsAction.java index 6173f1aa20..0026717c7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetricsAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetricsAction.java @@ -17,6 +17,7 @@ package org.apache.doris.httpv2.rest; +import org.apache.doris.common.Config; import org.apache.doris.metric.JsonMetricVisitor; import org.apache.doris.metric.MetricRepo; import org.apache.doris.metric.MetricVisitor; @@ -34,12 +35,16 @@ import javax.servlet.http.HttpServletResponse; //fehost:port/metrics //fehost:port/metrics?type=core @RestController -public class MetricsAction { +public class MetricsAction extends RestBaseController { private static final String TYPE_PARAM = "type"; @RequestMapping(path = "/metrics") public void execute(HttpServletRequest request, HttpServletResponse response) { + if (Config.enable_all_http_auth) { + executeCheckPassword(request, response); + } + String type = request.getParameter(TYPE_PARAM); MetricVisitor visitor = null; if (!Strings.isNullOrEmpty(type) && type.equalsIgnoreCase("core")) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java index dd3fbf48f8..ac9748c90e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java @@ -75,6 +75,11 @@ public class ShowAction extends RestBaseController { @RequestMapping(path = "/api/show_meta_info", method = RequestMethod.GET) public Object show_meta_info(HttpServletRequest request, HttpServletResponse response) { + if (Config.enable_all_http_auth) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + } + String action = request.getParameter("action"); if (Strings.isNullOrEmpty(action)) { return ResponseEntityBuilder.badRequest("Missing action parameter"); @@ -143,6 +148,11 @@ public class ShowAction extends RestBaseController { @RequestMapping(path = "/api/show_runtime_info", method = RequestMethod.GET) public Object show_runtime_info(HttpServletRequest request, HttpServletResponse response) { + if (Config.enable_all_http_auth) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + } + HashMap feInfo = new HashMap(); // Get memory info @@ -164,6 +174,10 @@ public class ShowAction extends RestBaseController { @RequestMapping(path = "/api/show_data", method = RequestMethod.GET) public Object show_data(HttpServletRequest request, HttpServletResponse response) { + if (Config.enable_all_http_auth) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + } Map oneEntry = Maps.newHashMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java index 76f2c50447..190c5cf82e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java @@ -24,11 +24,13 @@ import org.apache.doris.analysis.SqlScanner; import org.apache.doris.analysis.StatementBase; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.Config; import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; import org.apache.doris.httpv2.util.ExecutionResultSet; import org.apache.doris.httpv2.util.StatementSubmitter; +import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; import org.apache.doris.system.SystemInfoService; @@ -83,6 +85,10 @@ public class StmtExecutionAction extends RestBaseController { public Object executeSQL(@PathVariable(value = NS_KEY) String ns, @PathVariable(value = DB_KEY) String dbName, HttpServletRequest request, HttpServletResponse response, @RequestBody String body) { ActionAuthorizationInfo authInfo = checkWithCookie(request, response, false); + String fullDbName = getFullDbName(dbName); + if (Config.enable_all_http_auth) { + checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.ADMIN); + } if (ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { ns = InternalCatalog.INTERNAL_CATALOG_NAME; diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/ImportAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/ImportAction.java index a82cf4aa37..11f66a906c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/ImportAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/ImportAction.java @@ -18,10 +18,12 @@ package org.apache.doris.httpv2.restv2; import org.apache.doris.analysis.BrokerDesc; +import org.apache.doris.common.Config; import org.apache.doris.common.UserException; import org.apache.doris.common.parquet.ParquetReader; import org.apache.doris.common.util.BrokerUtil; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.httpv2.rest.RestBaseController; import org.apache.doris.thrift.TBrokerFileStatus; import com.google.common.collect.Lists; @@ -42,7 +44,7 @@ import javax.servlet.http.HttpServletResponse; @RestController @RequestMapping("/rest/v2") -public class ImportAction { +public class ImportAction extends RestBaseController { private static final Logger LOG = LogManager.getLogger(ImportAction.class); @@ -74,6 +76,10 @@ public class ImportAction { @RequestMapping(path = "/api/import/file_review", method = RequestMethod.POST) public Object fileReview(@RequestBody FileReviewRequestVo body, HttpServletRequest request, HttpServletResponse response) { + if (Config.enable_all_http_auth) { + executeCheckPassword(request, response); + } + FileInfo fileInfo = body.getFileInfo(); ConnectInfo connectInfo = body.getConnectInfo(); BrokerDesc brokerDesc = new BrokerDesc(connectInfo.getBrokerName(), connectInfo.getBrokerProps()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/StatisticAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/StatisticAction.java index 54793cef73..368c870f31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/StatisticAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/StatisticAction.java @@ -18,6 +18,7 @@ package org.apache.doris.httpv2.restv2; import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; import org.apache.doris.httpv2.rest.RestBaseController; @@ -45,6 +46,10 @@ public class StatisticAction extends RestBaseController { @RequestMapping(path = "/api/cluster_overview", method = RequestMethod.GET) public Object clusterOverview(HttpServletRequest request, HttpServletResponse response) { + if (Config.enable_all_http_auth) { + executeCheckPassword(request, response); + } + if (!Env.getCurrentEnv().isMaster()) { return redirectToMaster(request, response); }