From 17a8b57018ac76eb5bf19c35e0c3fce7f5bbdd26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AE=B6=E9=94=8B?= Date: Wed, 16 Sep 2020 15:02:59 +0800 Subject: [PATCH] [UI Part 3] New implemented HTTP RESTful API of Frontend (#4596) Use spring mvc rest to replace the original netty http rest Created a new package `org/apache/doris/httpv2`, and the origin implementations under `org/apache/doris/http` remain unchanged. This part of the code will not be used at present, so it will not affect existing functions. API document can be found in #4584 Proposal #4308 --- .../main/java/org/apache/doris/PaloFe.java | 2 +- .../org/apache/doris/common/Log4jConfig.java | 174 ++++----- .../apache/doris/httpv2/HttpAuthManager.java | 88 +++++ .../org/apache/doris/httpv2/HttpServer.java | 65 ++++ .../doris/httpv2/config/ReadEnvironment.java | 66 ++++ .../httpv2/config/SpringLog4j2Config.java | 58 +++ .../doris/httpv2/config/WebConfigurer.java | 68 ++++ .../httpv2/controller/BaseController.java | 296 ++++++++++++++++ .../httpv2/controller/ConfigController.java | 64 ++++ .../doris/httpv2/controller/HaController.java | 198 +++++++++++ .../controller/HardwareInfoController.java | 271 ++++++++++++++ .../httpv2/controller/HelpController.java | 154 ++++++++ .../httpv2/controller/LogController.java | 151 ++++++++ .../httpv2/controller/LoginController.java | 41 +++ .../httpv2/controller/LogoutController.java | 48 +++ .../controller/QueryProfileController.java | 99 ++++++ .../httpv2/controller/SessionController.java | 85 +++++ .../httpv2/controller/SystemController.java | 199 +++++++++++ .../doris/httpv2/entity/ResponseBody.java | 100 ++++++ .../httpv2/entity/ResponseEntityBuilder.java | 60 ++++ .../httpv2/exception/BadRequestException.java | 24 ++ .../exception/RestApiExceptionHandler.java | 64 ++++ .../exception/UnauthorizedException.java | 24 ++ .../httpv2/interceptor/AuthInterceptor.java | 66 ++++ .../httpv2/meta/ColocateMetaService.java | 168 +++++++++ .../httpv2/meta/InvalidClientException.java | 24 ++ .../doris/httpv2/meta/MetaBaseAction.java | 25 ++ .../apache/doris/httpv2/meta/MetaService.java | 245 +++++++++++++ .../httpv2/rest/BootstrapFinishAction.java | 135 +++++++ .../doris/httpv2/rest/CancelLoadAction.java | 87 +++++ .../httpv2/rest/CheckDecommissionAction.java | 94 +++++ .../doris/httpv2/rest/ConnectionAction.java | 85 +++++ .../doris/httpv2/rest/GetDdlStmtAction.java | 94 +++++ .../doris/httpv2/rest/GetLoadInfoAction.java | 91 +++++ .../doris/httpv2/rest/GetLogFileAction.java | 128 +++++++ .../doris/httpv2/rest/GetSmallFileAction.java | 78 ++++ .../doris/httpv2/rest/GetStreamLoadState.java | 63 ++++ .../doris/httpv2/rest/HealthAction.java | 40 +++ .../apache/doris/httpv2/rest/LoadAction.java | 153 ++++++++ .../doris/httpv2/rest/MetaInfoAction.java | 333 ++++++++++++++++++ .../httpv2/rest/MetaReplayerCheckAction.java | 51 +++ .../doris/httpv2/rest/MetricsAction.java | 62 ++++ .../apache/doris/httpv2/rest/MultiAction.java | 223 ++++++++++++ .../doris/httpv2/rest/ProfileAction.java | 66 ++++ .../doris/httpv2/rest/QueryDetailAction.java | 66 ++++ .../doris/httpv2/rest/RestApiStatusCode.java | 29 ++ .../doris/httpv2/rest/RestBaseController.java | 157 +++++++++ .../doris/httpv2/rest/RowCountAction.java | 110 ++++++ .../doris/httpv2/rest/SetConfigAction.java | 107 ++++++ .../apache/doris/httpv2/rest/ShowAction.java | 284 +++++++++++++++ .../httpv2/rest/StmtExecutionAction.java | 126 +++++++ .../httpv2/rest/StorageTypeCheckAction.java | 86 +++++ .../httpv2/rest/TableQueryPlanAction.java | 292 +++++++++++++++ .../httpv2/rest/TableRowCountAction.java | 92 +++++ .../doris/httpv2/rest/TableSchemaAction.java | 114 ++++++ .../doris/httpv2/rest/UploadAction.java | 306 ++++++++++++++++ .../doris/httpv2/util/ExecutionResultSet.java | 48 +++ .../apache/doris/httpv2/util/HttpUtil.java | 61 ++++ .../doris/httpv2/util/LoadSubmitter.java | 143 ++++++++ .../doris/httpv2/util/StatementSubmitter.java | 217 ++++++++++++ .../apache/doris/httpv2/util/TmpFileMgr.java | 306 ++++++++++++++++ 61 files changed, 7174 insertions(+), 80 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpAuthManager.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/config/ReadEnvironment.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/config/SpringLog4j2Config.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/config/WebConfigurer.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/BaseController.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/ConfigController.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/HaController.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/HardwareInfoController.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/HelpController.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LogController.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LoginController.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LogoutController.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/QueryProfileController.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/SessionController.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/SystemController.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseBody.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseEntityBuilder.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/exception/BadRequestException.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/exception/RestApiExceptionHandler.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/exception/UnauthorizedException.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/interceptor/AuthInterceptor.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/ColocateMetaService.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/InvalidClientException.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaBaseAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/BootstrapFinishAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/CancelLoadAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/CheckDecommissionAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ConnectionAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetDdlStmtAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLogFileAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetSmallFileAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetStreamLoadState.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/HealthAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetaInfoAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetaReplayerCheckAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetricsAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ProfileAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/QueryDetailAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestApiStatusCode.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RowCountAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/SetConfigAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StorageTypeCheckAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableRowCountAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/UploadAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/util/ExecutionResultSet.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/util/HttpUtil.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/util/LoadSubmitter.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/util/TmpFileMgr.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java b/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java index 5458b88269..76f320f9f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java +++ b/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java @@ -88,7 +88,7 @@ public class PaloFe { throw new IllegalArgumentException("Java version doesn't match"); } - Log4jConfig.initLogging(); + Log4jConfig.initLogging(dorisHomeDir + "/conf/"); // set dns cache ttl java.security.Security.setProperty("networkaddress.cache.ttl" , "60"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java index c3109bb743..9296bad1c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java @@ -17,7 +17,7 @@ package org.apache.doris.common; -import com.google.common.collect.Maps; +import org.apache.doris.httpv2.config.SpringLog4j2Config; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.LoggerContext; @@ -26,6 +26,8 @@ import org.apache.logging.log4j.core.config.xml.XmlConfiguration; import org.apache.logging.log4j.core.lookup.Interpolator; import org.apache.logging.log4j.core.lookup.StrSubstitutor; +import com.google.common.collect.Maps; + import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.Map; @@ -34,84 +36,91 @@ import java.util.Map; // don't use trace. use INFO, WARN, ERROR, FATAL // public class Log4jConfig extends XmlConfiguration { - private static final long serialVersionUID = 1L; - - private static String xmlConfTemplate = "\n" + - "\n" + - "\n" + - " \n" + - " \n" + - " \n" + + private static final long serialVersionUID = 1L; + + private static String xmlConfTemplate = "\n" + + "\n\n" + + "\n" + + " \n" + + " \n" + + " \n" + " %d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid) [%C{1}.%M():%L] %m%n\n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + - " \n" + - " \n" + - " \n" + + " \n" + + " \n" + + " \n" + " %d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid) [%C{1}.%M():%L] %m%n\n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + - " \n" + - " \n" + - " \n" + - " %d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}] %m%n\n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + + " \n" + + " \n" + + " \n" + + " %d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}] %m%n\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + ""; - + private static StrSubstitutor strSub; private static String sysLogLevel; private static String[] verboseModules; private static String[] auditModules; - + // save the generated xml conf template + private static String logXmlConfTemplate; + // dir of fe.conf + public static String confDir; + private static void reconfig() throws IOException { String newXmlConfTemplate = xmlConfTemplate; @@ -119,14 +128,14 @@ public class Log4jConfig extends XmlConfiguration { String sysLogDir = Config.sys_log_dir; String sysRollNum = String.valueOf(Config.sys_log_roll_num); String sysDeleteAge = String.valueOf(Config.sys_log_delete_age); - - if (!(sysLogLevel.equalsIgnoreCase("INFO") || + + if (!(sysLogLevel.equalsIgnoreCase("INFO") || sysLogLevel.equalsIgnoreCase("WARN") || - sysLogLevel.equalsIgnoreCase("ERROR") || + sysLogLevel.equalsIgnoreCase("ERROR") || sysLogLevel.equalsIgnoreCase("FATAL"))) { throw new IOException("sys_log_level config error"); } - + String sysLogRollPattern = "%d{yyyyMMdd}"; String sysRollMaxSize = String.valueOf(Config.log_roll_size_mb); if (Config.sys_log_roll_interval.equals("HOUR")) { @@ -136,7 +145,7 @@ public class Log4jConfig extends XmlConfiguration { } else { throw new IOException("sys_log_roll_interval config error: " + Config.sys_log_roll_interval); } - + // audit log config String auditLogDir = Config.audit_log_dir; String auditLogRollPattern = "%d{yyyyMMdd}"; @@ -150,7 +159,7 @@ public class Log4jConfig extends XmlConfiguration { } else { throw new IOException("audit_log_roll_interval config error: " + Config.audit_log_roll_interval); } - + // verbose modules and audit log modules StringBuilder sb = new StringBuilder(); for (String s : verboseModules) { @@ -160,8 +169,8 @@ public class Log4jConfig extends XmlConfiguration { sb.append(""); } newXmlConfTemplate = newXmlConfTemplate.replaceAll("", - sb.toString()); - + sb.toString()); + Map properties = Maps.newHashMap(); properties.put("sys_log_dir", sysLogDir); properties.put("sys_file_pattern", sysLogRollPattern); @@ -169,41 +178,47 @@ public class Log4jConfig extends XmlConfiguration { properties.put("sys_roll_num", sysRollNum); properties.put("sys_log_delete_age", sysDeleteAge); properties.put("sys_log_level", sysLogLevel); - + properties.put("audit_log_dir", auditLogDir); properties.put("audit_file_pattern", auditLogRollPattern); properties.put("audit_roll_maxsize", auditRollMaxSize); properties.put("audit_roll_num", auditRollNum); properties.put("audit_log_delete_age", auditDeleteAge); - + strSub = new StrSubstitutor(new Interpolator(properties)); newXmlConfTemplate = strSub.replace(newXmlConfTemplate); System.out.println("====="); System.out.println(newXmlConfTemplate); System.out.println("====="); + logXmlConfTemplate = newXmlConfTemplate; + SpringLog4j2Config.writeSpringLogConf(confDir); // new SimpleLog4jConfiguration with xmlConfTemplate ByteArrayInputStream bis = new ByteArrayInputStream(newXmlConfTemplate.getBytes("UTF-8")); ConfigurationSource source = new ConfigurationSource(bis); Log4jConfig config = new Log4jConfig(source); - - // LoggerContext.start(new Configuration) + LoggerContext context = (LoggerContext) LogManager.getContext(false); - context.start(config); + context.start(config); } - + + public static String getLogXmlConfTemplate() { + return logXmlConfTemplate; + } + public static class Tuple { - public final X x; - public final Y y; - public final Z z; - public Tuple(X x, Y y, Z z) { - this.x = x; - this.y = y; + public final X x; + public final Y y; + public final Z z; + + public Tuple(X x, Y y, Z z) { + this.x = x; + this.y = y; this.z = z; - } - } - + } + } + @Override public StrSubstitutor getStrSubstitutor() { return strSub; @@ -212,11 +227,12 @@ public class Log4jConfig extends XmlConfiguration { public Log4jConfig(final ConfigurationSource configSource) { super(LoggerContext.getContext(), configSource); } - - public synchronized static void initLogging() throws IOException { + + public synchronized static void initLogging(String dorisConfDir) throws IOException { sysLogLevel = Config.sys_log_level; verboseModules = Config.sys_log_verbose_modules; auditModules = Config.audit_log_modules; + confDir = dorisConfDir; reconfig(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpAuthManager.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpAuthManager.java new file mode 100644 index 0000000000..a0fbbaf1ab --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpAuthManager.java @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2; + +import org.apache.doris.analysis.UserIdentity; + +import com.google.common.base.Strings; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +// We simulate a simplified session here: only store user-name of clients who already logged in, +// and we only have a default admin user for now. +public final class HttpAuthManager { + private static final Logger LOG = LogManager.getLogger(HttpAuthManager.class); + + private static long SESSION_EXPIRE_TIME = 2; // hour + private static long SESSION_MAX_SIZE = 100; // avoid to store too many + + private static HttpAuthManager instance = new HttpAuthManager(); + + public static class SessionValue { + public UserIdentity currentUser; + public String password; + } + + // session_id => session value + private Cache authSessions = CacheBuilder.newBuilder() + .maximumSize(SESSION_MAX_SIZE) + .expireAfterAccess(SESSION_EXPIRE_TIME, TimeUnit.HOURS) + .build(); + + private HttpAuthManager() { + // do nothing + } + + public static HttpAuthManager getInstance() { + return instance; + } + + public SessionValue getSessionValue(List sessionIds) { + for (String sessionId : sessionIds) { + SessionValue sv = authSessions.getIfPresent(sessionId); + if (sv != null) { + LOG.debug("get session value {} by session id: {}, left size: {}", + sv == null ? null : sv.currentUser, sessionId, authSessions.size()); + return sv; + } + } + return null; + } + + public void removeSession(String sessionId){ + if (!Strings.isNullOrEmpty(sessionId)) { + authSessions.invalidate(sessionId); + LOG.debug("remove session id: {}, left size: {}", sessionId, authSessions.size()); + } + } + + public void addSessionValue(String key, SessionValue value) { + authSessions.put(key, value); + } + + public Cache getAuthSessions() { + return authSessions; + } +} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java new file mode 100644 index 0000000000..ae9e6911f7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2; + +import org.apache.doris.httpv2.config.SpringLog4j2Config; + +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.web.servlet.ServletComponentScan; +import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; + +import java.util.HashMap; +import java.util.Map; + +@SpringBootApplication +@EnableConfigurationProperties +@ServletComponentScan +public class HttpServer extends SpringBootServletInitializer { + + private int port; + + public void setPort(int port) { + this.port = port; + } + + @Override + protected SpringApplicationBuilder configure(SpringApplicationBuilder application) { + return application.sources(HttpServer.class); + } + + public void start(String dorisHome) { + Map properties = new HashMap<>(); + properties.put("server.port", port); + properties.put("server.servlet.context-path", "/"); + properties.put("spring.resources.static-locations", "classpath:/static"); + properties.put("spring.http.encoding.charset", "UTF-8"); + properties.put("spring.http.encoding.enabled", true); + properties.put("spring.http.encoding.force", true); + // properties.put("spring.http.multipart.maxFileSize", "100Mb"); + // properties.put("spring.http.multipart.maxRequestSize", "100Mb"); + properties.put("spring.servlet.multipart.max-file-size", "100MB"); + properties.put("spring.servlet.multipart.max-request-size", "100MB"); + properties.put("logging.config", dorisHome + "/conf/" + SpringLog4j2Config.SPRING_LOG_XML_FILE); + new SpringApplicationBuilder() + .sources(HttpServer.class) + .properties(properties) + .run(new String[]{}); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/ReadEnvironment.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/ReadEnvironment.java new file mode 100644 index 0000000000..a8dfd29a03 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/ReadEnvironment.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.config; + +import org.apache.doris.common.Log4jConfig; + +import org.springframework.beans.BeansException; +import org.springframework.boot.logging.LogFile; +import org.springframework.boot.logging.LoggingInitializationContext; +import org.springframework.boot.logging.LoggingSystem; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.stereotype.Component; +import org.springframework.util.ResourceUtils; + +import java.io.File; + + +@Component +public class ReadEnvironment implements ApplicationContextAware { + + private ApplicationContext applicationContext; + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; + } + + + public void reinitializeLoggingSystem() { + ConfigurableEnvironment environment = (ConfigurableEnvironment) this.applicationContext.getEnvironment(); + File file = new File(Log4jConfig.confDir + SpringLog4j2Config.SPRING_LOG_XML_FILE); + String logConfig = file.getAbsolutePath(); + LogFile logFile = LogFile.get(environment); + LoggingSystem system = LoggingSystem.get(LoggingSystem.class.getClassLoader()); + try { + ResourceUtils.getURL(logConfig).openStream().close(); + // Three step initialization that accounts for the clean up of the logging + // context before initialization. Spring Boot doesn't initialize a logging + // system that hasn't had this sequence applied (since 1.4.1). + system.cleanUp(); + system.beforeInitialize(); + system.initialize(new LoggingInitializationContext(environment), + logConfig, logFile); + } catch (Exception ex) { + ex.printStackTrace(); + } + + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/SpringLog4j2Config.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/SpringLog4j2Config.java new file mode 100644 index 0000000000..d18c2b50bc --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/SpringLog4j2Config.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.config; + +import org.apache.doris.common.Log4jConfig; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Writer; + +public class SpringLog4j2Config { + + public static final String SPRING_LOG_XML_FILE = "log4j2-spring.xml"; + + /** + * write spring boot log4j2-spring.xml file + */ + public static void writeSpringLogConf(String confDir) throws IOException { + Writer writer = null; + try { + // log4j2-spring.xml file path + File file = new File(confDir + SPRING_LOG_XML_FILE); + if (!file.exists()) { + file.createNewFile(); + //write file + writer = new FileWriter(file); + writer.write(Log4jConfig.getLogXmlConfTemplate()); + } else { + file.deleteOnExit(); + file.createNewFile(); + //write file + writer = new FileWriter(file); + writer.write(Log4jConfig.getLogXmlConfTemplate()); + } + System.out.println("=============================="); + } finally { + if (writer != null) { + writer.close(); + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/WebConfigurer.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/WebConfigurer.java new file mode 100644 index 0000000000..272e84b054 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/WebConfigurer.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.config; + +import org.apache.doris.httpv2.interceptor.AuthInterceptor; + +import org.springframework.boot.web.server.ErrorPage; +import org.springframework.boot.web.server.WebServerFactoryCustomizer; +import org.springframework.boot.web.servlet.server.ConfigurableServletWebServerFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.HttpStatus; +import org.springframework.web.servlet.config.annotation.CorsRegistry; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; +import org.springframework.web.servlet.config.annotation.ViewControllerRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +@Configuration +public class WebConfigurer implements WebMvcConfigurer { + + @Override + public void addInterceptors(InterceptorRegistry registry) { + registry.addInterceptor(new AuthInterceptor()) + .addPathPatterns("/rest/v1/**") + .excludePathPatterns("/", "/api/**", "/rest/v1/login", "/rest/v1/logout", "/static/**", "/metrics") + .excludePathPatterns("/image","/info","/version","/put","/journal_id","/role","/check","/dump"); + } + + @Override + public void addCorsMappings(CorsRegistry registry) { + registry.addMapping("/**") + .allowCredentials(false) + .allowedMethods("*") + .allowedOrigins("*") + .allowedHeaders("*") + .maxAge(3600); + } + + @Override + public void addViewControllers(ViewControllerRegistry registry) { + registry.addViewController("/notFound").setViewName("forward:/index.html"); + } + + + @Bean + public WebServerFactoryCustomizer containerCustomizer() { + return container -> { + container.addErrorPages(new ErrorPage(HttpStatus.NOT_FOUND, + "/notFound")); + }; + } +} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/BaseController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/BaseController.java new file mode 100644 index 0000000000..fbeb773ece --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/BaseController.java @@ -0,0 +1,296 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.controller; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.handler.codec.base64.Base64; +import io.netty.util.CharsetUtil; + +import org.apache.doris.analysis.CompoundPredicate; +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Catalog; +import org.apache.doris.cluster.ClusterNamespace; +import org.apache.doris.common.Config; +import org.apache.doris.httpv2.HttpAuthManager; +import org.apache.doris.httpv2.HttpAuthManager.SessionValue; +import org.apache.doris.httpv2.exception.UnauthorizedException; +import org.apache.doris.mysql.privilege.PaloPrivilege; +import org.apache.doris.mysql.privilege.PrivBitSet; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.service.FrontendOptions; +import org.apache.doris.system.SystemInfoService; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.UUID; + +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +public class BaseController { + + private static final Logger LOG = LogManager.getLogger(BaseController.class); + + public static final String PALO_SESSION_ID = "PALO_SESSION_ID"; + private static final int PALO_SESSION_EXPIRED_TIME = 3600 * 24; // one day + + // We first check cookie, if not admin, we check http's authority header + public void checkAuthWithCookie(HttpServletRequest request, HttpServletResponse response) { + checkWithCookie(request, response, true); + } + + public ActionAuthorizationInfo checkWithCookie(HttpServletRequest request, HttpServletResponse response, boolean checkAuth) { + ActionAuthorizationInfo authInfo = checkCookie(request, response, checkAuth); + if (authInfo != null) { + return authInfo; + } + + // cookie is invalid. check auth info in request + authInfo = getAuthorizationInfo(request); + UserIdentity currentUser = checkPassword(authInfo); + + if (checkAuth) { + checkGlobalAuth(currentUser, PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, + PaloPrivilege.NODE_PRIV), CompoundPredicate.Operator.OR)); + } + + SessionValue value = new SessionValue(); + value.currentUser = currentUser; + value.password = authInfo.password; + addSession(request, response, value); + + ConnectContext ctx = new ConnectContext(null); + ctx.setQualifiedUser(authInfo.fullUserName); + ctx.setRemoteIP(authInfo.remoteIp); + ctx.setCurrentUserIdentity(currentUser); + ctx.setCatalog(Catalog.getCurrentCatalog()); + ctx.setCluster(SystemInfoService.DEFAULT_CLUSTER); + ctx.setThreadLocalInfo(); + LOG.debug("check auth without cookie success for user: {}, thread: {}", + currentUser, Thread.currentThread().getId()); + return authInfo; + } + + protected void addSession(HttpServletRequest request, HttpServletResponse response, SessionValue value) { + String key = UUID.randomUUID().toString(); + Cookie cookie = new Cookie(PALO_SESSION_ID, key); + cookie.setMaxAge(PALO_SESSION_EXPIRED_TIME); + cookie.setPath("/"); + response.addCookie(cookie); + LOG.debug("add session cookie: {} {}", PALO_SESSION_ID, key); + HttpAuthManager.getInstance().addSessionValue(key, value); + } + + private ActionAuthorizationInfo checkCookie(HttpServletRequest request, HttpServletResponse response, + boolean checkAuth) { + List sessionIds = getCookieValues(request, PALO_SESSION_ID, response); + if (sessionIds.isEmpty()) { + return null; + } + + HttpAuthManager authMgr = HttpAuthManager.getInstance(); + SessionValue sessionValue = authMgr.getSessionValue(sessionIds); + if (sessionValue == null) { + return null; + } + + if (checkAuth && !Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(sessionValue.currentUser, + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, + PaloPrivilege.NODE_PRIV), CompoundPredicate.Operator.OR))) { + // need to check auth and check auth failed + return null; + } + + updateCookieAge(request, PALO_SESSION_ID, PALO_SESSION_EXPIRED_TIME, response); + + ConnectContext ctx = new ConnectContext(null); + ctx.setQualifiedUser(sessionValue.currentUser.getQualifiedUser()); + ctx.setRemoteIP(request.getRemoteHost()); + ctx.setCurrentUserIdentity(sessionValue.currentUser); + ctx.setCatalog(Catalog.getCurrentCatalog()); + ctx.setCluster(SystemInfoService.DEFAULT_CLUSTER); + ctx.setThreadLocalInfo(); + LOG.debug("check cookie success for user: {}, thread: {}", + sessionValue.currentUser, Thread.currentThread().getId()); + ActionAuthorizationInfo authInfo = new ActionAuthorizationInfo(); + authInfo.fullUserName = sessionValue.currentUser.getQualifiedUser(); + authInfo.remoteIp = request.getRemoteHost(); + authInfo.password = sessionValue.password; + authInfo.cluster = SystemInfoService.DEFAULT_CLUSTER; + return authInfo; + } + + public List getCookieValues(HttpServletRequest request, String cookieName, HttpServletResponse response) { + Cookie[] cookies = request.getCookies(); + List sessionIds = Lists.newArrayList(); + if (cookies != null) { + for (Cookie cookie : cookies) { + if (cookie.getName() != null && cookie.getName().equals(cookieName)) { + String sessionId = cookie.getValue(); + LOG.debug("get cookie value. {}: {}", cookie.getName(), sessionId); + sessionIds.add(sessionId); + } + } + } + return sessionIds; + } + + public void updateCookieAge(HttpServletRequest request, String cookieName, int age, HttpServletResponse response) { + Cookie[] cookies = request.getCookies(); + for (Cookie cookie : cookies) { + if (cookie.getName() != null && cookie.getName().equals(cookieName)) { + cookie.setMaxAge(age); + response.addCookie(cookie); + LOG.debug("get update cookie: {} {}", cookie.getName(), cookie.getValue()); + } + } + } + + public static class ActionAuthorizationInfo { + public String fullUserName; + public String remoteIp; + public String password; + public String cluster; + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("user: ").append(fullUserName).append(", remote ip: ").append(remoteIp); + sb.append(", password: ").append(password).append(", cluster: ").append(cluster); + return sb.toString(); + } + } + + protected void checkGlobalAuth(UserIdentity currentUser, PrivPredicate predicate) throws UnauthorizedException { + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(currentUser, predicate)) { + throw new UnauthorizedException("Access denied; you need (at least one of) the " + + predicate.getPrivs().toString() + " privilege(s) for this operation"); + } + } + + protected void checkDbAuth(UserIdentity currentUser, String db, PrivPredicate predicate) + throws UnauthorizedException { + if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(currentUser, db, predicate)) { + throw new UnauthorizedException("Access denied; you need (at least one of) the " + + predicate.getPrivs().toString() + " privilege(s) for this operation"); + } + } + + protected void checkTblAuth(UserIdentity currentUser, String db, String tbl, PrivPredicate predicate) + throws UnauthorizedException { + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(currentUser, db, tbl, predicate)) { + throw new UnauthorizedException("Access denied; you need (at least one of) the " + + predicate.getPrivs().toString() + " privilege(s) for this operation"); + } + } + + // return currentUserIdentity from Doris auth + protected UserIdentity checkPassword(ActionAuthorizationInfo authInfo) + throws UnauthorizedException { + List currentUser = Lists.newArrayList(); + if (!Catalog.getCurrentCatalog().getAuth().checkPlainPassword(authInfo.fullUserName, + authInfo.remoteIp, authInfo.password, currentUser)) { + throw new UnauthorizedException("Access denied for " + + authInfo.fullUserName + "@" + authInfo.remoteIp); + } + Preconditions.checkState(currentUser.size() == 1); + return currentUser.get(0); + } + + public ActionAuthorizationInfo getAuthorizationInfo(HttpServletRequest request) + throws UnauthorizedException { + ActionAuthorizationInfo authInfo = new ActionAuthorizationInfo(); + if (!parseAuthInfo(request, authInfo)) { + LOG.info("parse auth info failed, Authorization header {}, url {}", + request.getHeader("Authorization"), request.getRequestURI()); + throw new UnauthorizedException("Need auth information."); + } + LOG.debug("get auth info: {}", authInfo); + return authInfo; + } + + private boolean parseAuthInfo(HttpServletRequest request, ActionAuthorizationInfo authInfo) { + String encodedAuthString = request.getHeader("Authorization"); + if (Strings.isNullOrEmpty(encodedAuthString)) { + return false; + } + String[] parts = encodedAuthString.split(" "); + if (parts.length != 2) { + return false; + } + encodedAuthString = parts[1]; + ByteBuf buf = null; + ByteBuf decodeBuf = null; + try { + buf = Unpooled.copiedBuffer(ByteBuffer.wrap(encodedAuthString.getBytes())); + + // The authString is a string connecting user-name and password with + // a colon(':') + decodeBuf = Base64.decode(buf); + String authString = decodeBuf.toString(CharsetUtil.UTF_8); + // Note that password may contain colon, so can not simply use a + // colon to split. + int index = authString.indexOf(":"); + authInfo.fullUserName = authString.substring(0, index); + final String[] elements = authInfo.fullUserName.split("@"); + if (elements != null && elements.length < 2) { + authInfo.fullUserName = ClusterNamespace.getFullName(SystemInfoService.DEFAULT_CLUSTER, + authInfo.fullUserName); + authInfo.cluster = SystemInfoService.DEFAULT_CLUSTER; + } else if (elements != null && elements.length == 2) { + authInfo.fullUserName = ClusterNamespace.getFullName(elements[1], elements[0]); + authInfo.cluster = elements[1]; + } + authInfo.password = authString.substring(index + 1); + authInfo.remoteIp = request.getRemoteAddr(); + } finally { + // release the buf and decode buf after using Unpooled.copiedBuffer + // or it will get memory leak + if (buf != null) { + buf.release(); + } + + if (decodeBuf != null) { + decodeBuf.release(); + } + } + return true; + } + + protected int checkIntParam(String strParam) { + return Integer.parseInt(strParam); + } + + protected long checkLongParam(String strParam) { + return Long.parseLong(strParam); + } + + protected String getCurrentFrontendURL() { + return "http://" + FrontendOptions.getLocalHostAddress() + ":" + Config.http_port; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/ConfigController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/ConfigController.java new file mode 100644 index 0000000000..4353d8cf20 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/ConfigController.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.controller; + +import org.apache.doris.common.Config; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@RestController +@RequestMapping("/rest/v1") +public class ConfigController { + + private static final List CONFIG_TABLE_HEADER = Lists.newArrayList("Name", "Value"); + + @RequestMapping(path = "/config/fe", method = RequestMethod.GET) + public Object variable() { + Map result = Maps.newHashMap(); + appendConfigureInfo(result); + return ResponseEntityBuilder.ok(result); + } + + private void appendConfigureInfo(Map result) { + + result.put("column_names", CONFIG_TABLE_HEADER); + List> list = Lists.newArrayList(); + result.put("rows", list); + try { + Map confmap = Config.dump(); + for (String key : confmap.keySet()) { + Map info = new HashMap<>(); + info.put("Name", key); + info.put("Value", confmap.get(key)); + list.add(info); + } + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/HaController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/HaController.java new file mode 100644 index 0000000000..8ead5f5628 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/HaController.java @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.controller; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.Config; +import org.apache.doris.ha.HAProtocol; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.persist.Storage; +import org.apache.doris.system.Frontend; + +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@RestController +@RequestMapping("/rest/v1") +public class HaController { + + @RequestMapping(path = "/ha", method = RequestMethod.GET) + public Object ha() { + Map result = new HashMap<>(); + appendRoleInfo(result); + appendJournalInfo(result); + appendCanReadInfo(result); + appendNodesInfo(result); + appendImageInfo(result); + appendDbNames(result); + appendFe(result); + appendRemovedFe(result); + return ResponseEntityBuilder.ok(result); + } + + private void appendRoleInfo(Map result) { + Map info = new HashMap<>(); + List> list = new ArrayList<>(); + + info.put("Name", "FrontendRole"); + info.put("Value", Catalog.getCurrentCatalog().getFeType()); + list.add(info); + result.put("FrontendRole", list); + } + + private void appendJournalInfo(Map result) { + Map info = new HashMap<>(); + List> list = new ArrayList<>(); + + if (Catalog.getCurrentCatalog().isMaster()) { + info.put("Name", "FrontendRole"); + info.put("Value", Catalog.getCurrentCatalog().getEditLog().getMaxJournalId()); + } else { + info.put("Name", "FrontendRole"); + info.put("Value", Catalog.getCurrentCatalog().getReplayedJournalId()); + } + list.add(info); + result.put("CurrentJournalId", list); + } + + private void appendNodesInfo(Map result) { + HAProtocol haProtocol = Catalog.getCurrentCatalog().getHaProtocol(); + if (haProtocol == null) { + return; + } + List electableNodes = haProtocol.getElectableNodes(true); + if (electableNodes.isEmpty()) { + return; + } + + //buffer.append("

Electable nodes

"); + //buffer.append("
");
+        List> eleclist = new ArrayList<>();
+
+        for (InetSocketAddress node : electableNodes) {
+            Map info = new HashMap<>();
+            info.put("Name", node.getHostName());
+            info.put("Value", node.getAddress());
+            eleclist.add(info);
+        }
+        result.put("Electablenodes", eleclist);
+
+
+        List observerNodes = haProtocol.getObserverNodes();
+        if (observerNodes == null) {
+            return;
+        }
+        List> list = new ArrayList<>();
+
+        for (InetSocketAddress node : observerNodes) {
+            Map observer = new HashMap<>();
+            observer.put("Name", node.getHostName());
+            observer.put("Value", node.getHostString());
+            list.add(observer);
+        }
+        result.put("Observernodes", list);
+
+    }
+
+    private void appendCanReadInfo(Map result) {
+        Map canRead = new HashMap<>();
+        List> list = new ArrayList<>();
+
+        canRead.put("Name", "Status");
+        canRead.put("Value", Catalog.getCurrentCatalog().canRead());
+        list.add(canRead);
+        result.put("CanRead", list);
+    }
+
+    private void appendImageInfo(Map result) {
+        try {
+            List> list = new ArrayList<>();
+            Map checkPoint = new HashMap<>();
+            Storage storage = new Storage(Config.meta_dir + "/image");
+            checkPoint.put("Name", "Version");
+            checkPoint.put("Value", storage.getImageSeq());
+            list.add(checkPoint);
+            long lastCheckpointTime = storage.getCurrentImageFile().lastModified();
+            Date date = new Date(lastCheckpointTime);
+            Map checkPoint1 = new HashMap<>();
+            checkPoint1.put("Name", "lastCheckPointTime");
+            checkPoint1.put("Value", date);
+            list.add(checkPoint1);
+            result.put("CheckpointInfo", list);
+
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    private void appendDbNames(Map result) {
+        Map dbs = new HashMap<>();
+
+        List names = Catalog.getCurrentCatalog().getEditLog().getDatabaseNames();
+        if (names == null) {
+            return;
+        }
+
+        String msg = "";
+        for (long name : names) {
+            msg += name + " ";
+        }
+        List> list = new ArrayList<>();
+
+        dbs.put("Name", "DatabaseNames");
+        dbs.put("Value", msg);
+        list.add(dbs);
+        result.put("databaseNames", list);
+    }
+
+    private void appendFe(Map result) {
+        List fes = Catalog.getCurrentCatalog().getFrontends(null /* all */);
+        if (fes == null) {
+            return;
+        }
+        List> list = new ArrayList<>();
+        for (Frontend fe : fes) {
+            Map allowed = new HashMap<>();
+            allowed.put("Name", fe.getNodeName());
+            allowed.put("Value", fe.toString());
+            list.add(allowed);
+        }
+        result.put("allowedFrontends", list);
+    }
+
+    private void appendRemovedFe(Map result) {
+        List feNames = Catalog.getCurrentCatalog().getRemovedFrontendNames();
+        List> list = new ArrayList<>();
+        for (String feName : feNames) {
+            Map removed = new HashMap<>();
+            removed.put("Name", feName);
+            removed.put("Value", feName);
+            list.add(removed);
+        }
+        result.put("removedFronteds", list);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/HardwareInfoController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/HardwareInfoController.java
new file mode 100644
index 0000000000..abf76b4855
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/HardwareInfoController.java
@@ -0,0 +1,271 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.httpv2.controller;
+
+import oshi.SystemInfo;
+import oshi.hardware.CentralProcessor;
+import oshi.hardware.GlobalMemory;
+import oshi.hardware.HWDiskStore;
+import oshi.hardware.HWPartition;
+import oshi.hardware.HardwareAbstractionLayer;
+import oshi.hardware.NetworkIF;
+import oshi.hardware.VirtualMemory;
+import oshi.software.os.FileSystem;
+import oshi.software.os.NetworkParams;
+import oshi.software.os.OSFileStore;
+import oshi.software.os.OSProcess;
+import oshi.software.os.OperatingSystem;
+import oshi.util.FormatUtil;
+import oshi.util.Util;
+
+import org.apache.doris.common.Version;
+import org.apache.doris.httpv2.entity.ResponseEntityBuilder;
+
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@RestController
+@RequestMapping("/rest/v1")
+public class HardwareInfoController {
+
+    @RequestMapping(path = "/hardware_info/fe", method = RequestMethod.GET)
+    public Object index() {
+        Map> map = new HashMap<>();
+        appendVersionInfo(map);
+        appendHardwareInfo(map);
+        return ResponseEntityBuilder.ok(map);
+    }
+
+    private void appendVersionInfo(Map> content) {
+        Map map = new HashMap<>();
+        map.put("Version", Version.DORIS_BUILD_VERSION);
+        map.put("Git", Version.DORIS_BUILD_HASH);
+        map.put("BuildInfo", Version.DORIS_BUILD_INFO);
+        map.put("BuildTime", Version.DORIS_BUILD_TIME);
+        content.put("VersionInfo", map);
+    }
+
+    private void appendHardwareInfo(Map> content) {
+        SystemInfo si = new SystemInfo();
+        OperatingSystem os = si.getOperatingSystem();
+        HardwareAbstractionLayer hal = si.getHardware();
+        CentralProcessor processor = hal.getProcessor();
+        GlobalMemory memory = hal.getMemory();
+        Map map = new HashMap<>();
+        map.put("OS", String.join("
", getOperatingSystem(os))); + map.put("Processor", String.join("
", getProcessor(processor))); + map.put("Memory", String.join("
", getMemory(memory))); + map.put("Processes", String.join("
", getProcesses(os, memory))); + map.put("Disk", String.join("
", getDisks(hal.getDiskStores()))); + map.put("FileSystem", String.join("
", getFileSystem(os.getFileSystem()))); + map.put("NetworkInterface", String.join("
", getNetworkInterfaces(hal.getNetworkIFs()))); + map.put("NetworkParameter", String.join("
", getNetworkParameters(os.getNetworkParams()))); + content.put("HarewareInfo", map); + } + + private List getOperatingSystem(OperatingSystem os) { + List osInfo = new ArrayList<>(); + osInfo.add(String.valueOf(os)); + osInfo.add("Booted: " + Instant.ofEpochSecond(os.getSystemBootTime())); + osInfo.add("Uptime: " + FormatUtil.formatElapsedSecs(os.getSystemUptime())); + osInfo.add("Running with" + (os.isElevated() ? "" : "out") + " elevated permissions."); + return osInfo; + } + + private List getProcessor(CentralProcessor processor) { + List processorInfo = new ArrayList<>(); + processorInfo.add(String.valueOf(processor)); + processorInfo.add(" " + processor.getPhysicalPackageCount() + " physical CPU package(s)"); + processorInfo.add(" " + processor.getPhysicalProcessorCount() + " physical CPU core(s)"); + processorInfo.add(" " + processor.getLogicalProcessorCount() + " logical CPU(s)"); + + processorInfo.add("Identifier:   " + processor.getIdentifier()); + processorInfo.add("ProcessorID:   " + processor.getProcessorID()); + processorInfo.add("Context Switches/Interrupts:   " + processor.getContextSwitches() + + " / " + processor.getInterrupts() + "
"); + + long[] prevTicks = processor.getSystemCpuLoadTicks(); + long[][] prevProcTicks = processor.getProcessorCpuLoadTicks(); + processorInfo.add("CPU, IOWait, and IRQ ticks @ 0 sec:  " + Arrays.toString(prevTicks)); + // Wait a second... + Util.sleep(1000); + long[] ticks = processor.getSystemCpuLoadTicks(); + processorInfo.add("CPU, IOWait, and IRQ ticks @ 1 sec:  " + Arrays.toString(ticks)); + long user = ticks[CentralProcessor.TickType.USER.getIndex()] - prevTicks[CentralProcessor.TickType.USER.getIndex()]; + long nice = ticks[CentralProcessor.TickType.NICE.getIndex()] - prevTicks[CentralProcessor.TickType.NICE.getIndex()]; + long sys = ticks[CentralProcessor.TickType.SYSTEM.getIndex()] - prevTicks[CentralProcessor.TickType.SYSTEM.getIndex()]; + long idle = ticks[CentralProcessor.TickType.IDLE.getIndex()] - prevTicks[CentralProcessor.TickType.IDLE.getIndex()]; + long iowait = ticks[CentralProcessor.TickType.IOWAIT.getIndex()] - prevTicks[CentralProcessor.TickType.IOWAIT.getIndex()]; + long irq = ticks[CentralProcessor.TickType.IRQ.getIndex()] - prevTicks[CentralProcessor.TickType.IRQ.getIndex()]; + long softirq = ticks[CentralProcessor.TickType.SOFTIRQ.getIndex()] - prevTicks[CentralProcessor.TickType.SOFTIRQ.getIndex()]; + long steal = ticks[CentralProcessor.TickType.STEAL.getIndex()] - prevTicks[CentralProcessor.TickType.STEAL.getIndex()]; + long totalCpu = user + nice + sys + idle + iowait + irq + softirq + steal; + + processorInfo.add(String.format( + "User: %.1f%% Nice: %.1f%% System: %.1f%% Idle: %.1f%% IOwait: %.1f%% IRQ: %.1f%% SoftIRQ: %.1f%% Steal: %.1f%%", + 100d * user / totalCpu, 100d * nice / totalCpu, 100d * sys / totalCpu, 100d * idle / totalCpu, + 100d * iowait / totalCpu, 100d * irq / totalCpu, 100d * softirq / totalCpu, 100d * steal / totalCpu)); + processorInfo.add(String.format("CPU load:   %.1f%%", + processor.getSystemCpuLoadBetweenTicks(prevTicks) * 100)); + double[] loadAverage = processor.getSystemLoadAverage(3); + processorInfo.add("CPU load averages:  " + (loadAverage[0] < 0 ? " N/A" : String.format(" %.2f", loadAverage[0])) + + (loadAverage[1] < 0 ? " N/A" : String.format(" %.2f", loadAverage[1])) + + (loadAverage[2] < 0 ? " N/A" : String.format(" %.2f", loadAverage[2]))); + // per core CPU + StringBuilder procCpu = new StringBuilder("CPU load per processor:  "); + double[] load = processor.getProcessorCpuLoadBetweenTicks(prevProcTicks); + for (double avg : load) { + procCpu.append(String.format(" %.1f%%", avg * 100)); + } + processorInfo.add(procCpu.toString()); + long freq = processor.getVendorFreq(); + if (freq > 0) { + processorInfo.add("Vendor Frequency:   " + FormatUtil.formatHertz(freq)); + } + freq = processor.getMaxFreq(); + if (freq > 0) { + processorInfo.add("Max Frequency:   " + FormatUtil.formatHertz(freq)); + } + long[] freqs = processor.getCurrentFreq(); + if (freqs[0] > 0) { + StringBuilder sb = new StringBuilder("Current Frequencies:   "); + for (int i = 0; i < freqs.length; i++) { + if (i > 0) { + sb.append(", "); + } + sb.append(FormatUtil.formatHertz(freqs[i])); + } + processorInfo.add(sb.toString()); + } + return processorInfo; + } + + private List getMemory(GlobalMemory memory) { + List memoryInfo = new ArrayList<>(); + memoryInfo.add("Memory:   " + FormatUtil.formatBytes(memory.getAvailable()) + "/" + + FormatUtil.formatBytes(memory.getTotal())); + VirtualMemory vm = memory.getVirtualMemory(); + memoryInfo.add("Swap used:   " + FormatUtil.formatBytes(vm.getSwapUsed()) + "/" + + FormatUtil.formatBytes(vm.getSwapTotal())); + return memoryInfo; + } + + private List getProcesses(OperatingSystem os, GlobalMemory memory) { + List processInfo = new ArrayList<>(); + processInfo.add("Processes:   " + os.getProcessCount() + ", Threads:   " + os.getThreadCount()); + // Sort by highest CPU + List procs = Arrays.asList(os.getProcesses(5, OperatingSystem.ProcessSort.CPU)); + + processInfo.add("         PID %CPU %MEM VSZ RSS Name"); + for (int i = 0; i < procs.size() && i < 5; i++) { + OSProcess p = procs.get(i); + processInfo.add(String.format("         %5d %5.1f %4.1f %9s %9s %s", p.getProcessID(), + 100d * (p.getKernelTime() + p.getUserTime()) / p.getUpTime(), + 100d * p.getResidentSetSize() / memory.getTotal(), FormatUtil.formatBytes(p.getVirtualSize()), + FormatUtil.formatBytes(p.getResidentSetSize()), p.getName())); + } + return processInfo; + } + + private List getDisks(HWDiskStore[] diskStores) { + List diskInfo = new ArrayList<>(); + diskInfo.add("Disks:  "); + for (HWDiskStore disk : diskStores) { + boolean readwrite = disk.getReads() > 0 || disk.getWrites() > 0; + diskInfo.add(String.format("         %s: (model: %s - S/N: %s) size: %s, reads: %s (%s), writes: %s (%s), xfer: %s ms", + disk.getName(), disk.getModel(), disk.getSerial(), + disk.getSize() > 0 ? FormatUtil.formatBytesDecimal(disk.getSize()) : "?", + readwrite ? disk.getReads() : "?", readwrite ? FormatUtil.formatBytes(disk.getReadBytes()) : "?", + readwrite ? disk.getWrites() : "?", readwrite ? FormatUtil.formatBytes(disk.getWriteBytes()) : "?", + readwrite ? disk.getTransferTime() : "?")); + HWPartition[] partitions = disk.getPartitions(); + for (HWPartition part : partitions) { + diskInfo.add(String.format("         |-- %s: %s (%s) Maj:Min=%d:%d, size: %s%s", part.getIdentification(), + part.getName(), part.getType(), part.getMajor(), part.getMinor(), + FormatUtil.formatBytesDecimal(part.getSize()), + part.getMountPoint().isEmpty() ? "" : " @ " + part.getMountPoint())); + } + } + return diskInfo; + } + + private List getFileSystem(FileSystem fileSystem) { + List fsInfo = new ArrayList<>(); + fsInfo.add("File System:  "); + + fsInfo.add(String.format("    File Descriptors: %d/%d", fileSystem.getOpenFileDescriptors(), + fileSystem.getMaxFileDescriptors())); + + OSFileStore[] fsArray = fileSystem.getFileStores(); + for (OSFileStore fs : fsArray) { + long usable = fs.getUsableSpace(); + long total = fs.getTotalSpace(); + fsInfo.add(String.format("        %s (%s) [%s] %s of %s free (%.1f%%), %s of %s files free (%.1f%%) is %s " + + (fs.getLogicalVolume() != null && fs.getLogicalVolume().length() > 0 ? "[%s]" : "%s") + + " and is mounted at %s", + fs.getName(), fs.getDescription().isEmpty() ? "file system" : fs.getDescription(), fs.getType(), + FormatUtil.formatBytes(usable), FormatUtil.formatBytes(fs.getTotalSpace()), 100d * usable / total, + FormatUtil.formatValue(fs.getFreeInodes(), ""), FormatUtil.formatValue(fs.getTotalInodes(), ""), + 100d * fs.getFreeInodes() / fs.getTotalInodes(), fs.getVolume(), fs.getLogicalVolume(), + fs.getMount())); + } + return fsInfo; + } + + private List getNetworkInterfaces(NetworkIF[] networkIFs) { + List getNetwork = new ArrayList<>(); + getNetwork.add("Network interfaces:  "); + for (NetworkIF net : networkIFs) { + getNetwork.add(String.format("    Name: %s (%s)", net.getName(), net.getDisplayName())); + getNetwork.add(String.format("        MAC Address: %s", net.getMacaddr())); + getNetwork.add(String.format("        MTU: %s, Speed: %s", net.getMTU(), FormatUtil.formatValue(net.getSpeed(), "bps"))); + getNetwork.add(String.format("        IPv4: %s", Arrays.toString(net.getIPv4addr()))); + getNetwork.add(String.format("        IPv6: %s", Arrays.toString(net.getIPv6addr()))); + boolean hasData = net.getBytesRecv() > 0 || net.getBytesSent() > 0 || net.getPacketsRecv() > 0 + || net.getPacketsSent() > 0; + getNetwork.add(String.format("        Traffic: received %s/%s%s; transmitted %s/%s%s", + hasData ? net.getPacketsRecv() + " packets" : "?", + hasData ? FormatUtil.formatBytes(net.getBytesRecv()) : "?", + hasData ? " (" + net.getInErrors() + " err)" : "", + hasData ? net.getPacketsSent() + " packets" : "?", + hasData ? FormatUtil.formatBytes(net.getBytesSent()) : "?", + hasData ? " (" + net.getOutErrors() + " err)" : "")); + } + return getNetwork; + } + + private List getNetworkParameters(NetworkParams networkParams) { + List networkParameterInfo = new ArrayList<>(); + networkParameterInfo.add("Network parameters:    "); + networkParameterInfo.add(String.format("        Host name: %s", networkParams.getHostName())); + networkParameterInfo.add(String.format("         Domain name: %s", networkParams.getDomainName())); + networkParameterInfo.add(String.format("         DNS servers: %s", Arrays.toString(networkParams.getDnsServers()))); + networkParameterInfo.add(String.format("         IPv4 Gateway: %s", networkParams.getIpv4DefaultGateway())); + networkParameterInfo.add(String.format("         IPv6 Gateway: %s", networkParams.getIpv6DefaultGateway())); + return networkParameterInfo; + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/HelpController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/HelpController.java new file mode 100644 index 0000000000..d04c55b1b8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/HelpController.java @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.controller; + +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.qe.HelpModule; +import org.apache.doris.qe.HelpTopic; + +import com.google.common.base.Strings; + +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.servlet.http.HttpServletRequest; + +@RestController +@RequestMapping("/rest/v1") +public class HelpController { + + private String queryString = null; + + @RequestMapping(path = "/help", method = RequestMethod.GET) + public Object helpSearch(HttpServletRequest request) { + this.queryString = request.getParameter("query"); + if (Strings.isNullOrEmpty(queryString)) { + // ATTN: according to Mysql protocol, the default query should be "contents" + // when you want to get server side help. + queryString = "contents"; + } else { + queryString = queryString.trim(); + } + Map result = new HashMap<>(); + appendHelpInfo(result); + return ResponseEntityBuilder.ok(result); + } + + private void appendHelpInfo(Map result) { + appendExactMatchTopic(result); + appendFuzzyMatchTopic(result); + appendCategories(result); + } + + private void appendExactMatchTopic(Map result) { + HelpModule module = HelpModule.getInstance(); + HelpTopic topic = module.getTopic(queryString); + if (topic == null) { + result.put("matching", "No Exact Matching Topic."); + } else { + Map subMap = new HashMap<>(); + appendOneTopicInfo(subMap, topic, "matching"); + result.put("matchingTopic", subMap); + } + } + + private void appendFuzzyMatchTopic(Map result) { + HelpModule module = HelpModule.getInstance(); + List topics = module.listTopicByKeyword(queryString); + if (topics.isEmpty()) { + result.put("fuzzy", "No Fuzzy Matching Topic"); + } else if (topics.size() == 1) { + result.put("fuzzy", "Find only one topic, show you the detail info below"); + Map subMap = new HashMap<>(); + appendOneTopicInfo(subMap, module.getTopic(topics.get(0)), "fuzzy"); + result.put("fuzzyTopic", subMap); + } else { + result.put("size", topics.size()); + result.put("datas", topics); + } + } + + private void appendCategories(Map result) { + HelpModule module = HelpModule.getInstance(); + List categories = module.listCategoryByName(queryString); + if (categories.isEmpty()) { + result.put("matching", "No Matching Category"); + } else if (categories.size() == 1) { + result.put("matching", "Find only one category, so show you the detail info below"); + List topics = module.listTopicByCategory(categories.get(0)); + + if (topics.size() > 0) { + List> topic_list = new ArrayList<>(); + result.put("topicSize", topics.size()); + for (String topic : topics) { + Map top = new HashMap<>(); + top.put("name", topic); + topic_list.add(top); + } + result.put("topicdatas", topic_list); + } + + List subCategories = module.listCategoryByCategory(categories.get(0)); + if (subCategories.size() > 0) { + List> subCate = new ArrayList<>(); + result.put("subCateSize", subCategories.size()); + for (String sub : subCategories) { + Map subMap = new HashMap<>(); + subMap.put("name", sub); + subCate.add(subMap); + } + result.put("subdatas", subCate); + } + } else { + List> category_list = new ArrayList<>(); + if (categories.size() > 0) { + result.put("categoriesSize", categories.size()); + for (String cate : categories) { + Map subMap = new HashMap<>(); + subMap.put("name", cate); + category_list.add(subMap); + } + result.put("categoryDatas", category_list); + } + } + } + + // The browser will combine continuous whitespace to one, we use
 tag to solve this issue.
+    private void appendOneTopicInfo(Map result, HelpTopic topic, String prefix) {
+        result.put(prefix + "topic", escapeHtmlInPreTag(topic.getName()));
+        result.put(prefix + "description", escapeHtmlInPreTag(topic.getDescription()));
+        result.put(prefix + "example", escapeHtmlInPreTag(topic.getExample()));
+        result.put(prefix + "Keyword", escapeHtmlInPreTag(topic.getKeywords().toString()));
+        result.put(prefix + "Url", escapeHtmlInPreTag(topic.getUrl()));
+    }
+
+    protected String escapeHtmlInPreTag(String oriStr) {
+        if (oriStr == null) {
+            return "";
+        }
+        String content = oriStr.replaceAll("\n", "
"); + return content; + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LogController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LogController.java new file mode 100644 index 0000000000..10d7ba8d3f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LogController.java @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.controller; + +import org.apache.doris.common.Config; +import org.apache.doris.common.Log4jConfig; +import org.apache.doris.httpv2.config.ReadEnvironment; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.servlet.http.HttpServletRequest; + +@RestController +@RequestMapping("/rest/v1") +public class LogController { + + private static final Logger LOG = LogManager.getLogger(LogController.class); + private static long WEB_LOG_BYTES = 1024 * 1024; // 1MB + + private String addVerboseName; + private String delVerboseName; + + @Autowired + private ReadEnvironment readEnvironment; + + @RequestMapping(path = "/log", method = RequestMethod.GET) + public Object log(HttpServletRequest request) { + Map> map = new HashMap<>(); + appendLogConf(map); + appendLogInfo(map); + return ResponseEntityBuilder.ok(map); + } + + @RequestMapping(path = "/log", method = RequestMethod.POST) + public Object logLevel(HttpServletRequest request) { + Map> map = new HashMap<>(); + // get parameters + addVerboseName = request.getParameter("add_verbose"); + delVerboseName = request.getParameter("del_verbose"); + LOG.info("add verbose name: {}, del verbose name: {}", addVerboseName, delVerboseName); + appendLogConf(map); + return ResponseEntityBuilder.ok(map); + } + + private void appendLogConf(Map> content) { + Map map = new HashMap<>(); + + try { + Log4jConfig.Tuple configs = Log4jConfig.updateLogging(null, null, null); + if (!Strings.isNullOrEmpty(addVerboseName)) { + addVerboseName = addVerboseName.trim(); + List verboseNames = Lists.newArrayList(configs.y); + if (!verboseNames.contains(addVerboseName)) { + verboseNames.add(addVerboseName); + configs = Log4jConfig.updateLogging(null, verboseNames.toArray(new String[verboseNames.size()]), + null); + readEnvironment.reinitializeLoggingSystem(); + } + } + if (!Strings.isNullOrEmpty(delVerboseName)) { + delVerboseName = delVerboseName.trim(); + List verboseNames = Lists.newArrayList(configs.y); + if (verboseNames.contains(delVerboseName)) { + verboseNames.remove(delVerboseName); + configs = Log4jConfig.updateLogging(null, verboseNames.toArray(new String[verboseNames.size()]), + null); + readEnvironment.reinitializeLoggingSystem(); + } + } + + map.put("Level", configs.x); + map.put("VerboseNames", StringUtils.join(configs.y, ",")); + map.put("AuditNames", StringUtils.join(configs.z, ",")); + content.put("LogConfiguration", map); + } catch (IOException e) { + LOG.error(e); + e.printStackTrace(); + } + } + + private void appendLogInfo(Map> content) { + Map map = new HashMap<>(); + + final String logPath = Config.sys_log_dir + "/fe.warn.log"; + map.put("logPath", logPath); + + RandomAccessFile raf = null; + try { + raf = new RandomAccessFile(logPath, "r"); + long fileSize = raf.length(); + long startPos = fileSize < WEB_LOG_BYTES ? 0L : fileSize - WEB_LOG_BYTES; + long webContentLength = fileSize < WEB_LOG_BYTES ? fileSize : WEB_LOG_BYTES; + raf.seek(startPos); + map.put("showingLast", webContentLength + " bytes of log"); + StringBuilder sb = new StringBuilder(); + String line = ""; + sb.append("
");
+            while ((line = raf.readLine()) != null) {
+                sb.append(line).append("
"); + } + sb.append("
"); + map.put("log", sb.toString()); + + } catch (FileNotFoundException e) { + map.put("error", "Couldn't open log file: " + logPath); + } catch (IOException e) { + map.put("error", "Failed to read log file: " + logPath); + } finally { + try { + if (raf != null) { + raf.close(); + } + } catch (IOException e) { + LOG.warn("fail to close log file: " + logPath, e); + } + } + content.put("LogContents", map); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LoginController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LoginController.java new file mode 100644 index 0000000000..eb941ada24 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LoginController.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.controller; + +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.HashMap; +import java.util.Map; + +@RestController +@RequestMapping("/rest/v1") +public class LoginController extends BaseController { + + @RequestMapping(path = "/login", method = RequestMethod.POST) + public Object login(HttpServletRequest request, HttpServletResponse response) { + checkAuthWithCookie(request, response); + Map msg = new HashMap<>(); + msg.put("code", 200); + msg.put("msg", "Login success!"); + return msg; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LogoutController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LogoutController.java new file mode 100644 index 0000000000..68e7aca957 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LogoutController.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.controller; + +import org.apache.doris.httpv2.HttpAuthManager; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; + +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +@RestController +@RequestMapping("/rest/v1") +public class LogoutController extends BaseController { + + @RequestMapping(path = "/logout", method = RequestMethod.POST) + public Object login(HttpServletRequest request, HttpServletResponse response) { + Cookie[] cookies = request.getCookies(); + if (cookies != null) { + for (Cookie cookie : cookies) { + if (cookie.getName() != null && cookie.getName().equals(PALO_SESSION_ID)) { + String sessionId = cookie.getValue(); + HttpAuthManager.getInstance().removeSession(sessionId); + } + } + } + return ResponseEntityBuilder.ok(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/QueryProfileController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/QueryProfileController.java new file mode 100644 index 0000000000..985e87dd7b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/QueryProfileController.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.controller; + +import org.apache.doris.common.util.ProfileManager; +import org.apache.doris.httpv2.entity.ResponseBody; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@RestController +@RequestMapping("/rest/v1") +public class QueryProfileController extends BaseController { + private static final Logger LOG = LogManager.getLogger(QueryProfileController.class); + + private static final String QUERY_ID = "query_id"; + + @RequestMapping(path = "/query_profile/{" + QUERY_ID + "}", method = RequestMethod.GET) + public Object profile(@PathVariable(value = QUERY_ID) String queryId) { + String profile = ProfileManager.getInstance().getProfile(queryId); + if (profile == null) { + return ResponseEntityBuilder.okWithCommonError("Query " + queryId + " does not exist"); + } + profile = profile.replaceAll("\n", "
"); + profile = profile.replaceAll(" ", "  "); + return ResponseEntityBuilder.ok(profile); + } + + @RequestMapping(path = "/query_profile", method = RequestMethod.GET) + public Object query() { + Map result = Maps.newHashMap(); + addFinishedQueryInfo(result); + ResponseEntity entity = ResponseEntityBuilder.ok(result); + ((ResponseBody) entity.getBody()).setCount(result.size()); + return entity; + } + + private void addFinishedQueryInfo(Map result) { + List> finishedQueries = ProfileManager.getInstance().getAllQueries(); + List columnHeaders = ProfileManager.PROFILE_HEADERS; + int queryIdIndex = 0; // the first column is 'Query ID' by default + for (int i = 0; i < columnHeaders.size(); ++i) { + if (columnHeaders.get(i).equals(ProfileManager.QUERY_ID)) { + queryIdIndex = i; + break; + } + } + + result.put("column_names", columnHeaders); + result.put("href_column", Lists.newArrayList(ProfileManager.QUERY_ID)); + List> list = Lists.newArrayList(); + result.put("rows", list); + + for (List row : finishedQueries) { + String queryId = row.get(queryIdIndex); + Map rowMap = new HashMap<>(); + for (int i = 0; i < row.size(); ++i) { + rowMap.put(columnHeaders.get(i), row.get(i)); + } + + // add hyper link + if (Strings.isNullOrEmpty(queryId)) { + rowMap.put("__hrefPaths", Lists.newArrayList("/query_profile/-1")); + } else { + rowMap.put("__hrefPaths", Lists.newArrayList("/query_profile/" + queryId)); + } + list.add(rowMap); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/SessionController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/SessionController.java new file mode 100644 index 0000000000..ace361dbc4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/SessionController.java @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.controller; + +import org.apache.doris.httpv2.entity.ResponseBody; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.service.ExecuteEnv; + +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@RestController +@RequestMapping("/rest/v1") +public class SessionController extends BaseController { + + private static final List SESSION_TABLE_HEADER = Lists.newArrayList(); + + static { + SESSION_TABLE_HEADER.add("Id"); + SESSION_TABLE_HEADER.add("User"); + SESSION_TABLE_HEADER.add("Host"); + SESSION_TABLE_HEADER.add("Cluster"); + SESSION_TABLE_HEADER.add("Db"); + SESSION_TABLE_HEADER.add("Command"); + SESSION_TABLE_HEADER.add("Time"); + SESSION_TABLE_HEADER.add("State"); + SESSION_TABLE_HEADER.add("Info"); + } + + @RequestMapping(path = "/session", method = RequestMethod.GET) + public Object session() { + Map result = Maps.newHashMap(); + appendSessionInfo(result); + ResponseEntity entity = ResponseEntityBuilder.ok(result); + ((ResponseBody) entity.getBody()).setCount(result.size()); + return entity; + } + + private void appendSessionInfo(Map result) { + List threadInfos = ExecuteEnv.getInstance().getScheduler().listConnection("root"); + List> rows = Lists.newArrayList(); + + result.put("column_names", SESSION_TABLE_HEADER); + List> list = Lists.newArrayList(); + result.put("rows", list); + + long nowMs = System.currentTimeMillis(); + for (ConnectContext.ThreadInfo info : threadInfos) { + rows.add(info.toRow(nowMs)); + } + + for (List row : rows) { + Map record = new HashMap<>(); + for (int i = 0; i < row.size(); i++) { + record.put(SESSION_TABLE_HEADER.get(i), row.get(i)); + } + list.add(record); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/SystemController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/SystemController.java new file mode 100644 index 0000000000..78ccde4741 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/SystemController.java @@ -0,0 +1,199 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.controller; + +import org.apache.doris.analysis.RedirectStatus; +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.proc.ProcDirInterface; +import org.apache.doris.common.proc.ProcNodeInterface; +import org.apache.doris.common.proc.ProcResult; +import org.apache.doris.common.proc.ProcService; +import org.apache.doris.httpv2.entity.ResponseBody; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.MasterOpExecutor; +import org.apache.doris.qe.OriginStatement; +import org.apache.doris.qe.ShowResultSet; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import org.apache.commons.validator.routines.UrlValidator; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import javax.servlet.http.HttpServletRequest; + +@RestController +@RequestMapping("/rest/v1") +public class SystemController extends BaseController { + + private static final Logger LOG = LogManager.getLogger(SystemController.class); + + + @RequestMapping(path = "/system", method = RequestMethod.GET) + public Object system(HttpServletRequest request) { + String currentPath = request.getParameter("path"); + if (Strings.isNullOrEmpty(currentPath)) { + currentPath = "/"; + } + LOG.debug("get /system requset, thread id: {}", Thread.currentThread().getId()); + ResponseEntity entity = appendSystemInfo(currentPath, currentPath,request); + return entity; + } + + protected ProcNodeInterface getProcNode(String path) { + ProcService instance = ProcService.getInstance(); + ProcNodeInterface node; + try { + if (Strings.isNullOrEmpty(path)) { + node = instance.open("/"); + } else { + node = instance.open(path); + } + } catch (AnalysisException e) { + LOG.warn(e.getMessage()); + return null; + } + return node; + } + + private ResponseEntity appendSystemInfo(String procPath, String path, HttpServletRequest request) { + UrlValidator validator = new UrlValidator(); + Map map = new HashMap<>(); + + ProcNodeInterface procNode = getProcNode(procPath); + if (procNode == null) { + return ResponseEntityBuilder.notFound("No such proc path[" + path + "]"); + } + boolean isDir = (procNode instanceof ProcDirInterface); + + List columnNames = null; + List> rows = null; + if (!Catalog.getCurrentCatalog().isMaster()) { + // forward to master + String showProcStmt = "SHOW PROC \"" + procPath + "\""; + + MasterOpExecutor masterOpExecutor = new MasterOpExecutor(new OriginStatement(showProcStmt, 0), + ConnectContext.get(), RedirectStatus.FORWARD_NO_SYNC); + try { + masterOpExecutor.execute(); + } catch (Exception e) { + LOG.warn("Fail to forward. ", e); + return ResponseEntityBuilder.internalError("Failed to forward request to master: " + e.getMessage()); + } + + ShowResultSet resultSet = masterOpExecutor.getProxyResultSet(); + if (resultSet == null) { + return ResponseEntityBuilder.internalError("Failed to get result from master"); + } + + columnNames = resultSet.getMetaData().getColumns().stream().map(c -> c.getName()).collect( + Collectors.toList()); + rows = resultSet.getResultRows(); + } else { + ProcResult result; + try { + result = procNode.fetchResult(); + } catch (AnalysisException e) { + return ResponseEntityBuilder.internalError("The result is null." + + "Maybe haven't be implemented completely[" + e.getMessage() + "], please check." + + "INFO: ProcNode type is [" + procNode.getClass().getName() + "]: " + + e.getMessage()); + } + + columnNames = result.getColumnNames(); + rows = result.getRows(); + } + + Preconditions.checkNotNull(columnNames); + Preconditions.checkNotNull(rows); + + Map result = Maps.newHashMap(); + result.put("column_names", columnNames); + List hrefColumns = Lists.newArrayList(); + if (isDir) { + hrefColumns.add(columnNames.get(0)); + } + List> list = Lists.newArrayList(); + for (List strList : rows) { + Map rowColumns = new HashMap<>(); + List hrefPaths = Lists.newArrayList(); + + for (int i = 0; i < strList.size(); i++) { + String str = strList.get(i); + if (isDir && i == 0) { + // the first column of dir proc is always a href column + String escapeStr = str.replace("%", "%25"); + String uriPath = "path=" + path + "/" + escapeStr; + hrefPaths.add("/rest/v1/system?" + uriPath); + } else if (validator.isValid(str)) { + // if the value is a URL, add it to href columns, and change the content to "URL" + hrefPaths.add(str); + str = "URL"; + if (!hrefColumns.contains(columnNames.get(i))) { + hrefColumns.add(columnNames.get(i)); + } + } + + rowColumns.put(columnNames.get(i), str); + } + if (!hrefPaths.isEmpty()) { + rowColumns.put("__hrefPaths", hrefPaths); + } + list.add(rowColumns); + } + result.put("rows", list); + + // assemble href column names + if (!hrefColumns.isEmpty()) { + result.put("href_columns", hrefColumns); + } + + // add parent url + result.put("parent_url", getParentUrl(path)); + + ResponseEntity entity = ResponseEntityBuilder.ok(result); + ((ResponseBody) entity.getBody()).setCount(list.size()); + return entity; + } + + private String getParentUrl(String pathStr) { + Path path = Paths.get(pathStr); + path = path.getParent(); + if (path == null) { + return "/rest/v1/system"; + } else { + return "/rest/v1/system?path=" + path.toString(); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseBody.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseBody.java new file mode 100644 index 0000000000..74653b3e77 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseBody.java @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.entity; + +import org.apache.doris.httpv2.rest.RestApiStatusCode; + +/** + * The response body of restful api. + *

+ * The getter setter methods of all member variables need to be retained + * to ensure that Spring can perform json format conversion. + * + * @param type of data + */ +public class ResponseBody { + // Used to describe the error message. If there are no errors, it displays "OK" + private String msg; + // The user displays an error code. + // If there is no error, 0 is displayed. + // If there is an error, it is usually Doris's internal error code, not the HTTP standard error code. + // The HTTP standard error code should be reflected in the return value of the HTTP protocol. + private int code = RestApiStatusCode.OK.code; + // to save the response body + private T data; + // to save the number of records in response body. + // currently not used and always be 0. + private int count; + + public ResponseBody() { + } + + public ResponseBody msg(String msg) { + this.msg = msg; + return this; + } + + public ResponseBody code(RestApiStatusCode code) { + this.code = code.code; + return this; + } + + public ResponseBody data(T data) { + this.data = data; + return this; + } + + + public String getMsg() { + return msg; + } + + public void setMsg(String msg) { + this.msg = msg; + } + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + + public T getData() { + return data; + } + + public void setData(T data) { + this.data = data; + } + + public void setCount(int count) { + this.count = count; + } + + public int getCount() { + return count; + } + + public ResponseBody commonError(String msg) { + this.code = RestApiStatusCode.COMMON_ERROR.code; + this.msg = msg; + return this; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseEntityBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseEntityBuilder.java new file mode 100644 index 0000000000..1b46c090ec --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseEntityBuilder.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.entity; + +import org.apache.doris.httpv2.rest.RestApiStatusCode; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; + +/** + * A utility class for creating a ResponseEntity easier. + */ +public class ResponseEntityBuilder { + + public static ResponseEntity badRequest(Object data) { + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(data); + } + + public static ResponseEntity okWithCommonError(String msg) { + ResponseBody body = new ResponseBody().code(RestApiStatusCode.COMMON_ERROR).commonError(msg); + return ResponseEntity.status(HttpStatus.OK).body(body); + } + + public static ResponseEntity ok(Object data) { + ResponseBody body = new ResponseBody().code(RestApiStatusCode.OK).msg("success").data(data); + return ResponseEntity.status(HttpStatus.OK).body(body); + } + + public static ResponseEntity ok() { + ResponseBody body = new ResponseBody().code(RestApiStatusCode.OK).msg("success"); + return ResponseEntity.status(HttpStatus.OK).body(body); + } + + public static ResponseEntity unauthorized(Object data) { + return ResponseEntity.status(HttpStatus.UNAUTHORIZED).body(data); + } + + public static ResponseEntity internalError(Object data) { + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(data); + } + + public static ResponseEntity notFound(Object data) { + return ResponseEntity.status(HttpStatus.NOT_FOUND).body(data); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/exception/BadRequestException.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/exception/BadRequestException.java new file mode 100644 index 0000000000..13ad1a52dd --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/exception/BadRequestException.java @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.exception; + +public class BadRequestException extends RuntimeException { + public BadRequestException(String msg) { + super(msg); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/exception/RestApiExceptionHandler.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/exception/RestApiExceptionHandler.java new file mode 100644 index 0000000000..1acd7bae1d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/exception/RestApiExceptionHandler.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.exception; + +import org.apache.doris.common.UserException; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.ResponseBody; + +/** + * A handler to handle all exceptions of restful api + */ +@ControllerAdvice +public class RestApiExceptionHandler { + + private static final Logger LOG = LogManager.getLogger(RestApiExceptionHandler.class); + + @ExceptionHandler(UnauthorizedException.class) + @ResponseBody + public Object unauthorizedHandler(UnauthorizedException e) { + LOG.debug("unauthorized exception", e); + return ResponseEntityBuilder.unauthorized(e.getMessage()); + } + + @ExceptionHandler(UserException.class) + @ResponseBody + public Object userExceptionHandler(UserException e) { + LOG.debug("user exception", e); + return ResponseEntityBuilder.ok(e.getMessage()); + } + + @ExceptionHandler(BadRequestException.class) + @ResponseBody + public Object badRequestExceptionHandler(BadRequestException e) { + LOG.debug("bad request exception", e); + return ResponseEntityBuilder.badRequest(e.getMessage()); + } + + @ExceptionHandler(Exception.class) + @ResponseBody + public Object unexpectedExceptionHandler(Exception e) { + LOG.debug("unexpected exception", e); + return ResponseEntityBuilder.internalError(e.getMessage()); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/exception/UnauthorizedException.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/exception/UnauthorizedException.java new file mode 100644 index 0000000000..76ece0499d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/exception/UnauthorizedException.java @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.exception; + +public class UnauthorizedException extends RuntimeException { + public UnauthorizedException(String msg) { + super(msg); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/interceptor/AuthInterceptor.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/interceptor/AuthInterceptor.java new file mode 100644 index 0000000000..61f701fe29 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/interceptor/AuthInterceptor.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.interceptor; + +import org.apache.doris.httpv2.controller.BaseController; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONObject; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.servlet.HandlerInterceptor; +import org.springframework.web.servlet.ModelAndView; + +import java.util.Map; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +public class AuthInterceptor extends BaseController implements HandlerInterceptor { + private static final Logger LOG = LogManager.getLogger(AuthInterceptor.class); + + @Override + public boolean preHandle(HttpServletRequest request, + HttpServletResponse response, Object handler) throws Exception { + LOG.debug("get prehandle. thread: {}", Thread.currentThread().getId()); + // String sessionId = getCookieValue(request, BaseController.PALO_SESSION_ID, response); + // SessionValue sessionValue = HttpAuthManager.getInstance().getSessionValue(sessionId); + String method = request.getMethod(); + if (method.equalsIgnoreCase(RequestMethod.OPTIONS.toString())) { + response.setStatus(HttpStatus.NO_CONTENT.value()); + return true; + } + + checkAuthWithCookie(request, response); + return true; + } + + @Override + public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler, ModelAndView modelAndView) throws Exception { + } + + @Override + public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) throws Exception { + } + + private String toJson(Map map) { + JSONObject root = new JSONObject(map); + return root.toString(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/ColocateMetaService.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/ColocateMetaService.java new file mode 100644 index 0000000000..c467215dc3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/ColocateMetaService.java @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.meta; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.ColocateGroupSchema; +import org.apache.doris.catalog.ColocateTableIndex; +import org.apache.doris.catalog.ColocateTableIndex.GroupId; +import org.apache.doris.common.DdlException; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.httpv2.rest.RestBaseController; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.persist.ColocatePersistInfo; +import org.apache.doris.qe.ConnectContext; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.view.RedirectView; + +import com.google.common.base.Preconditions; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.lang.reflect.Type; +import java.util.List; + +/* + * the colocate meta define in {@link ColocateTableIndex} + * The actions in ColocateMetaService is for modifying or showing colocate group info manually. + * + * ColocateMetaAction: + * get all information in ColocateTableIndex, as a json string + * eg: + * GET /api/colocate + * return: + * {"colocate_meta":{"groupName2Id":{...},"group2Tables":{}, ...},"status":"OK"} + * + * eg: + * POST /api/colocate/group_stable?db_id=123&group_id=456 (mark group[123.456] as unstable) + * DELETE /api/colocate/group_stable?db_id=123&group_id=456 (mark group[123.456] as stable) + * + * BucketSeqAction: + * change the backends per bucket sequence of a group + * eg: + * POST /api/colocate/bucketseq?db_id=123&group_id=456 + */ + +@RestController +public class ColocateMetaService extends RestBaseController { + private static final Logger LOG = LogManager.getLogger(ColocateMetaService.class); + private static final String GROUP_ID = "group_id"; + private static final String DB_ID = "db_id"; + + private static ColocateTableIndex colocateIndex = Catalog.getCurrentColocateIndex(); + + private static GroupId checkAndGetGroupId(HttpServletRequest request) throws DdlException { + long grpId = Long.valueOf(request.getParameter(GROUP_ID).trim()); + long dbId = Long.valueOf(request.getParameter(DB_ID).trim()); + GroupId groupId = new GroupId(dbId, grpId); + + if (!colocateIndex.isGroupExist(groupId)) { + throw new DdlException("the group " + groupId + "isn't exist"); + } + return groupId; + } + + public Object executeWithoutPassword(HttpServletRequest request, HttpServletResponse response) + throws DdlException { + executeCheckPassword(request, response); + RedirectView redirectView = redirectToMaster(request, response); + if (redirectView != null) { + return redirectView; + } + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + return null; + } + + @RequestMapping(path = "/api/colocate", method = RequestMethod.GET) + public Object colocate(HttpServletRequest request, HttpServletResponse response) throws DdlException { + executeWithoutPassword(request, response); + return ResponseEntityBuilder.ok(Catalog.getCurrentColocateIndex()); + } + + @RequestMapping(path = "/api/colocate/group_stable", method = {RequestMethod.POST, RequestMethod.DELETE}) + public Object group_stable(HttpServletRequest request, HttpServletResponse response) + throws DdlException { + executeWithoutPassword(request, response); + GroupId groupId = checkAndGetGroupId(request); + + String method = request.getMethod(); + if ("POST".equalsIgnoreCase(method)) { + colocateIndex.markGroupUnstable(groupId, true); + } else if ("DELETE".equalsIgnoreCase(method)) { + colocateIndex.markGroupStable(groupId, true); + } + return ResponseEntityBuilder.ok(); + } + + @RequestMapping(path = "/api/colocate/bucketseq", method = RequestMethod.POST) + public Object bucketseq(HttpServletRequest request, HttpServletResponse response, @RequestBody String meta) + throws DdlException { + executeWithoutPassword(request, response); + final String clusterName = ConnectContext.get().getClusterName(); + GroupId groupId = checkAndGetGroupId(request); + + Type type = new TypeToken>>() { + }.getType(); + List> backendsPerBucketSeq = new Gson().fromJson(meta, type); + LOG.info("get buckets sequence: {}", backendsPerBucketSeq); + + ColocateGroupSchema groupSchema = Catalog.getCurrentColocateIndex().getGroupSchema(groupId); + if (backendsPerBucketSeq.size() != groupSchema.getBucketsNum()) { + return ResponseEntityBuilder.okWithCommonError("Invalid bucket num. expected: " + + groupSchema.getBucketsNum() + ", actual: " + backendsPerBucketSeq.size()); + } + + List clusterBackendIds = Catalog.getCurrentSystemInfo().getClusterBackendIds(clusterName, true); + //check the Backend id + for (List backendIds : backendsPerBucketSeq) { + if (backendIds.size() != groupSchema.getReplicationNum()) { + return ResponseEntityBuilder.okWithCommonError("Invalid backend num per bucket. expected: " + + groupSchema.getReplicationNum() + ", actual: " + backendIds.size()); + } + for (Long beId : backendIds) { + if (!clusterBackendIds.contains(beId)) { + return ResponseEntityBuilder.okWithCommonError("The backend " + beId + + " does not exist or not available"); + } + } + } + + int bucketsNum = colocateIndex.getBackendsPerBucketSeq(groupId).size(); + Preconditions.checkState(backendsPerBucketSeq.size() == bucketsNum, + backendsPerBucketSeq.size() + " vs. " + bucketsNum); + updateBackendPerBucketSeq(groupId, backendsPerBucketSeq); + LOG.info("the group {} backendsPerBucketSeq meta has been changed to {}", groupId, backendsPerBucketSeq); + return ResponseEntityBuilder.ok(); + } + + private void updateBackendPerBucketSeq(GroupId groupId, List> backendsPerBucketSeq) { + colocateIndex.addBackendsPerBucketSeq(groupId, backendsPerBucketSeq); + ColocatePersistInfo info2 = ColocatePersistInfo.createForBackendsPerBucketSeq(groupId, backendsPerBucketSeq); + Catalog.getCurrentCatalog().getEditLog().logColocateBackendsPerBucketSeq(info2); + } + + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/InvalidClientException.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/InvalidClientException.java new file mode 100644 index 0000000000..b3f7dfc76d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/InvalidClientException.java @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.meta; + +public class InvalidClientException extends RuntimeException { + public InvalidClientException(String msg) { + super(msg); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaBaseAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaBaseAction.java new file mode 100644 index 0000000000..ed6a31e853 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaBaseAction.java @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.meta; + +public class MetaBaseAction { + private static String CONTENT_DISPOSITION = "Content-disposition"; + + public static final String CLUSTER_ID = "cluster_id"; + public static final String TOKEN = "token"; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java new file mode 100644 index 0000000000..22796efb42 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java @@ -0,0 +1,245 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.meta; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; +import org.apache.doris.ha.FrontendNodeType; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.httpv2.rest.RestBaseController; +import org.apache.doris.master.MetaHelper; +import org.apache.doris.persist.MetaCleaner; +import org.apache.doris.persist.Storage; +import org.apache.doris.persist.StorageInfo; +import org.apache.doris.system.Frontend; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.google.common.base.Strings; +import com.google.common.collect.Maps; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Map; + +@RestController +public class MetaService extends RestBaseController { + private static final Logger LOG = LogManager.getLogger(MetaService.class); + + private static final int TIMEOUT_SECOND = 10; + + private static final String VERSION = "version"; + private static final String HOST = "host"; + private static final String PORT = "port"; + + private File imageDir = MetaHelper.getMasterImageDir(); + + private boolean isFromValidFe(HttpServletRequest request) { + String clientHost = request.getRemoteHost(); + Frontend fe = Catalog.getCurrentCatalog().getFeByHost(clientHost); + if (fe == null) { + LOG.warn("request is not from valid FE. client: {}", clientHost); + return false; + } + return true; + } + + + private void checkFromValidFe(HttpServletRequest request) + throws InvalidClientException { + if (!isFromValidFe(request)) { + throw new InvalidClientException("invalid client host: " + request.getRemoteHost()); + } + } + + @RequestMapping(path = "/image", method = RequestMethod.GET) + public Object image(HttpServletRequest request, HttpServletResponse response) { + checkFromValidFe(request); + + String versionStr = request.getParameter(VERSION); + + if (Strings.isNullOrEmpty(versionStr)) { + return ResponseEntityBuilder.badRequest("Miss version parameter"); + } + + long version = checkLongParam(versionStr); + if (version < 0) { + return ResponseEntityBuilder.badRequest("The version number cannot be less than 0"); + } + + File imageFile = Storage.getImageFile(imageDir, version); + if (!imageFile.exists()) { + return ResponseEntityBuilder.notFound("image file not found"); + } + + try { + writeFileResponse(request, response, imageFile); + return null; + } catch (IOException e) { + return ResponseEntityBuilder.internalError(e.getMessage()); + } + } + + @RequestMapping(path = "/info", method = RequestMethod.GET) + public Object info(HttpServletRequest request, HttpServletResponse response) throws DdlException { + checkFromValidFe(request); + + try { + Storage currentStorageInfo = new Storage(imageDir.getAbsolutePath()); + StorageInfo storageInfo = new StorageInfo(currentStorageInfo.getClusterID(), + currentStorageInfo.getImageSeq(), currentStorageInfo.getEditsSeq()); + return ResponseEntityBuilder.ok(storageInfo); + } catch (IOException e) { + return ResponseEntityBuilder.internalError(e.getMessage()); + } + } + + @RequestMapping(path = "/version", method = RequestMethod.GET) + public void version(HttpServletRequest request, HttpServletResponse response) throws IOException, DdlException { + checkFromValidFe(request); + File versionFile = new File(imageDir, Storage.VERSION_FILE); + writeFileResponse(request, response, versionFile); + } + + @RequestMapping(path = "/put", method = RequestMethod.GET) + public Object put(HttpServletRequest request, HttpServletResponse response) throws DdlException { + checkFromValidFe(request); + + String portStr = request.getParameter(PORT); + + // check port to avoid SSRF(Server-Side Request Forgery) + if (Strings.isNullOrEmpty(portStr)) { + return ResponseEntityBuilder.badRequest("Port number cannot be empty"); + } + + int port = Integer.parseInt(portStr); + if (port < 0 || port > 65535) { + return ResponseEntityBuilder.badRequest("port is invalid. The port number is between 0-65535"); + } + + String versionStr = request.getParameter(VERSION); + if (Strings.isNullOrEmpty(versionStr)) { + return ResponseEntityBuilder.badRequest("Miss version parameter"); + } + + checkLongParam(versionStr); + + String machine = request.getRemoteHost(); + String url = "http://" + machine + ":" + port + "/image?version=" + versionStr; + String filename = Storage.IMAGE + "." + versionStr; + File dir = new File(Catalog.getCurrentCatalog().getImageDir()); + try { + OutputStream out = MetaHelper.getOutputStream(filename, dir); + MetaHelper.getRemoteFile(url, TIMEOUT_SECOND * 1000, out); + MetaHelper.complete(filename, dir); + } catch (FileNotFoundException e) { + return ResponseEntityBuilder.notFound("file not found."); + } catch (IOException e) { + LOG.warn("failed to get remote file. url: {}", url, e); + return ResponseEntityBuilder.internalError("failed to get remote file: " + e.getMessage()); + } + + // Delete old image files + try { + MetaCleaner cleaner = new MetaCleaner(Config.meta_dir + "/image"); + cleaner.clean(); + } catch (Exception e) { + LOG.error("Follower/Observer delete old image file fail.", e); + } + return ResponseEntityBuilder.ok(); + } + + @RequestMapping(path = "/journal_id", method = RequestMethod.GET) + public Object journal_id(HttpServletRequest request, HttpServletResponse response) throws DdlException { + checkFromValidFe(request); + long id = Catalog.getCurrentCatalog().getReplayedJournalId(); + response.setHeader("id", Long.toString(id)); + return ResponseEntityBuilder.ok(); + } + + @RequestMapping(path = "/role", method = RequestMethod.GET) + public Object role(HttpServletRequest request, HttpServletResponse response) throws DdlException { + checkFromValidFe(request); + + String host = request.getParameter(HOST); + String portString = request.getParameter(PORT); + if (!Strings.isNullOrEmpty(host) && !Strings.isNullOrEmpty(portString)) { + int port = Integer.parseInt(portString); + Frontend fe = Catalog.getCurrentCatalog().checkFeExist(host, port); + if (fe == null) { + response.setHeader("role", FrontendNodeType.UNKNOWN.name()); + } else { + response.setHeader("role", fe.getRole().name()); + response.setHeader("name", fe.getNodeName()); + } + return ResponseEntityBuilder.ok(); + } else { + return ResponseEntityBuilder.badRequest("Miss parameter"); + } + } + + /* + * This action is used to get the electable_nodes config and the cluster id of + * the fe with the given ip and port. When one frontend start, it should check + * the local electable_nodes config and local cluster id with other frontends. + * If there is any difference, local fe will exit. This is designed to protect + * the consistency of the cluster. + */ + @RequestMapping(path = "/check", method = RequestMethod.GET) + public Object check(HttpServletRequest request, HttpServletResponse response) throws DdlException { + checkFromValidFe(request); + + try { + Storage storage = new Storage(imageDir.getAbsolutePath()); + response.setHeader(MetaBaseAction.CLUSTER_ID, Integer.toString(storage.getClusterID())); + response.setHeader(MetaBaseAction.TOKEN, storage.getToken()); + } catch (IOException e) { + return ResponseEntityBuilder.internalError(e.getMessage()); + } + return ResponseEntityBuilder.ok(); + } + + @RequestMapping(value = "/dump", method = RequestMethod.GET) + public Object dump(HttpServletRequest request, HttpServletResponse response) throws DdlException { + /* + * Before dump, we acquired the catalog read lock and all databases' read lock and all + * the jobs' read lock. This will guarantee the consistency of database and job queues. + * But Backend may still inconsistent. + * + * TODO: Still need to lock ClusterInfoService to prevent add or drop Backends + */ + String dumpFilePath = Catalog.getCurrentCatalog().dumpImage(); + + if (dumpFilePath == null) { + return ResponseEntityBuilder.okWithCommonError("dump failed."); + } + Map res = Maps.newHashMap(); + res.put("dumpFilePath", dumpFilePath); + return ResponseEntityBuilder.ok(res); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/BootstrapFinishAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/BootstrapFinishAction.java new file mode 100644 index 0000000000..254b32b8f9 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/BootstrapFinishAction.java @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.Config; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; + +import com.google.common.base.Strings; + +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Api for checking the whether the FE has been started successfully. + * Response + * { + * "msg": "OK", + * "code": 0, + * "data": { + * "queryPort": 9030, + * "rpcPort": 9020, + * "maxReplayedJournal": 17287 + * }, + * "count": 0 + * } + */ +@RestController +public class BootstrapFinishAction { + + private static final String CLUSTER_ID = "cluster_id"; + private static final String TOKEN = "token"; + + public static final String REPLAYED_JOURNAL_ID = "replayedJournalId"; + public static final String QUERY_PORT = "queryPort"; + public static final String RPC_PORT = "rpcPort"; + + @RequestMapping(path = "/api/bootstrap", method = RequestMethod.GET) + public ResponseEntity execute(HttpServletRequest request, HttpServletResponse response) { + boolean isReady = Catalog.getCurrentCatalog().isReady(); + + // to json response + BootstrapResult result = new BootstrapResult(); + if (isReady) { + String clusterIdStr = request.getParameter(CLUSTER_ID); + String token = request.getParameter(TOKEN); + if (!Strings.isNullOrEmpty(clusterIdStr) && !Strings.isNullOrEmpty(token)) { + // cluster id or token is provided, return more info + int clusterId = 0; + try { + clusterId = Integer.valueOf(clusterIdStr); + } catch (NumberFormatException e) { + return ResponseEntityBuilder.badRequest("invalid cluster id format: " + clusterIdStr); + } + + + if (clusterId != Catalog.getCurrentCatalog().getClusterId()) { + return ResponseEntityBuilder.okWithCommonError("invalid cluster id: " + clusterId); + } + + + if (!token.equals(Catalog.getCurrentCatalog().getToken())) { + return ResponseEntityBuilder.okWithCommonError("invalid token: " + token); + } + + // cluster id and token are valid, return replayed journal id + long replayedJournalId = Catalog.getCurrentCatalog().getReplayedJournalId(); + result.setReplayedJournalId(replayedJournalId); + result.setQueryPort(Config.query_port); + result.setRpcPort(Config.rpc_port); + } + + return ResponseEntityBuilder.ok(result); + } + + return ResponseEntityBuilder.okWithCommonError("not ready"); + } + + /** + * This class is also for json DeSer, so get/set method must be remained. + */ + private static class BootstrapResult { + private long replayedJournalId = 0; + private int queryPort = 0; + private int rpcPort = 0; + + public BootstrapResult() { + + } + + public void setReplayedJournalId(long replayedJournalId) { + this.replayedJournalId = replayedJournalId; + } + + public long getReplayedJournalId() { + return replayedJournalId; + } + + public void setQueryPort(int queryPort) { + this.queryPort = queryPort; + } + + public int getQueryPort() { + return queryPort; + } + + public void setRpcPort(int rpcPort) { + this.rpcPort = rpcPort; + } + + public int getRpcPort() { + return rpcPort; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/CancelLoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/CancelLoadAction.java new file mode 100644 index 0000000000..ff5f971b37 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/CancelLoadAction.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Database; +import org.apache.doris.common.UserException; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.httpv2.exception.UnauthorizedException; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.base.Strings; + +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.view.RedirectView; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * To cancel a load transaction with given load label + */ +@RestController +public class CancelLoadAction extends RestBaseController { + + @RequestMapping(path = "/api/{" + DB_KEY + "}/_cancel", method = RequestMethod.POST) + public Object execute(@PathVariable(value = DB_KEY) final String dbName, + HttpServletRequest request, HttpServletResponse response) { + executeCheckPassword(request, response); + + RedirectView redirectView = redirectToMaster(request, response); + if (redirectView != null) { + return redirectView; + } + + if (Strings.isNullOrEmpty(dbName)) { + return ResponseEntityBuilder.badRequest("No database selected"); + } + + String fullDbName = getFullDbName(dbName); + + String label = request.getParameter(LABEL_KEY); + if (Strings.isNullOrEmpty(label)) { + return ResponseEntityBuilder.badRequest("No label specified"); + } + + Database db = Catalog.getCurrentCatalog().getDb(fullDbName); + if (db == null) { + return ResponseEntityBuilder.okWithCommonError("unknown database, database=" + dbName); + } + + // TODO(cmy): Currently we only check priv in db level. + // Should check priv in table level. + if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(ConnectContext.get(), fullDbName, PrivPredicate.LOAD)) { + throw new UnauthorizedException("Access denied for user '" + ConnectContext.get().getQualifiedUser() + + "' to database '" + fullDbName + "'"); + } + + try { + Catalog.getCurrentGlobalTransactionMgr().abortTransaction(db.getId(), label, "user cancel"); + } catch (UserException e) { + return ResponseEntityBuilder.okWithCommonError(e.getMessage()); + } + + return ResponseEntityBuilder.ok(); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/CheckDecommissionAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/CheckDecommissionAction.java new file mode 100644 index 0000000000..3724113abc --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/CheckDecommissionAction.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.alter.SystemHandler; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.Pair; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; + +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.stream.Collectors; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * calc row count from replica to table + * fe_host:fe_http_port/api/check_decommission?host_ports=host:port,host2:port2... + * return: + * { + * "msg": "OK", + * "code": 0, + * "data": ["192.168.10.11:9050", "192.168.10.11:9050"], + * "count": 0 + * } + */ +@RestController +public class CheckDecommissionAction extends RestBaseController { + + public static final String HOST_PORTS = "host_ports"; + + @RequestMapping(path = "/api/check_decommission", method = RequestMethod.GET) + public Object execute(HttpServletRequest request, HttpServletResponse response) { + //check user auth + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.OPERATOR); + + String hostPorts = request.getParameter(HOST_PORTS); + if (Strings.isNullOrEmpty(hostPorts)) { + return ResponseEntityBuilder.badRequest("No host:port specified"); + } + + String[] hostPortArr = hostPorts.split(","); + if (hostPortArr.length == 0) { + return ResponseEntityBuilder.badRequest("No host:port specified"); + } + + List> hostPortPairs = Lists.newArrayList(); + for (String hostPort : hostPortArr) { + Pair pair; + try { + pair = SystemInfoService.validateHostAndPort(hostPort); + } catch (AnalysisException e) { + return ResponseEntityBuilder.badRequest(e.getMessage()); + } + hostPortPairs.add(pair); + } + + try { + List backends = SystemHandler.checkDecommission(hostPortPairs); + List backendsList = backends.stream().map(b -> b.getHost() + ":" + b.getHeartbeatPort()).collect(Collectors.toList()); + return ResponseEntityBuilder.ok(backendsList); + } catch (DdlException e) { + return ResponseEntityBuilder.okWithCommonError(e.getMessage()); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ConnectionAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ConnectionAction.java new file mode 100644 index 0000000000..f04ca74f59 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ConnectionAction.java @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.common.util.DebugUtil; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.service.ExecuteEnv; + +import com.google.common.base.Strings; +import com.google.common.collect.Maps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * This class is used to get current query_id of connection_id. + * Every connection holds at most one query at every point. + * So we can get query_id firstly, and get query by query_id. + * + * { + * "msg": "OK", + * "code": 0, + * "data": { + * "query_id": "b52513ce3f0841ca-9cb4a96a268f2dba" + * }, + * "count": 0 + * } + */ +@RestController +public class ConnectionAction extends RestBaseController { + private static final Logger LOG = LogManager.getLogger(ConnectionAction.class); + + @RequestMapping(path = "/api/connection", method = RequestMethod.GET) + protected Object connection(HttpServletRequest request, HttpServletResponse response) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + + String connStr = request.getParameter("connection_id"); + if (Strings.isNullOrEmpty(connStr)) { + return ResponseEntityBuilder.badRequest("Missing connection_id"); + } + + long connectionId = -1; + try { + connectionId = Long.valueOf(connStr.trim()); + } catch (NumberFormatException e) { + return ResponseEntityBuilder.badRequest("Invalid connection id: " + e.getMessage()); + } + + ConnectContext context = ExecuteEnv.getInstance().getScheduler().getContext(connectionId); + if (context == null || context.queryId() == null) { + return ResponseEntityBuilder.okWithCommonError("connection id " + connectionId + " not found."); + } + String queryId = DebugUtil.printId(context.queryId()); + + Map result = Maps.newHashMap(); + result.put("query_id", queryId); + return ResponseEntityBuilder.ok(result); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetDdlStmtAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetDdlStmtAction.java new file mode 100644 index 0000000000..3adda7a25e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetDdlStmtAction.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Table; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.Map; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/* + * used to get a table's ddl stmt + * eg: + * fe_host:http_port/api/_get_ddl?db=xxx&tbl=yyy + */ +@RestController +public class GetDdlStmtAction extends RestBaseController { + + private static final Logger LOG = LogManager.getLogger(GetDdlStmtAction.class); + + @RequestMapping(path = "/api/_get_ddl", method = RequestMethod.GET) + public Object execute(HttpServletRequest request, HttpServletResponse response) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + + String dbName = request.getParameter(DB_KEY); + String tableName = request.getParameter(TABLE_KEY); + + if (Strings.isNullOrEmpty(dbName) || Strings.isNullOrEmpty(tableName)) { + return ResponseEntityBuilder.badRequest("Missing params. Need database name and Table name"); + } + + String fullDbName = getFullDbName(dbName); + Database db = Catalog.getCurrentCatalog().getDb(fullDbName); + if (db == null) { + return ResponseEntityBuilder.okWithCommonError("Database[" + dbName + "] does not exist"); + } + + List createTableStmt = Lists.newArrayList(); + List addPartitionStmt = Lists.newArrayList(); + List createRollupStmt = Lists.newArrayList(); + + db.readLock(); + try { + Table table = db.getTable(tableName); + if (table == null) { + return ResponseEntityBuilder.okWithCommonError("Table[" + tableName + "] does not exist"); + } + + Catalog.getDdlStmt(table, createTableStmt, addPartitionStmt, createRollupStmt, true, false /* show password */); + } finally { + db.readUnlock(); + } + + Map> results = Maps.newHashMap(); + results.put("create_table", createTableStmt); + results.put("create_partition", addPartitionStmt); + results.put("create_rollup", createRollupStmt); + + return ResponseEntityBuilder.ok(results); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java new file mode 100644 index 0000000000..de83bc1faf --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.load.Load; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; + +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.view.RedirectView; + +import com.google.common.base.Strings; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +// Get load information of one load job +@RestController +public class GetLoadInfoAction extends RestBaseController { + + protected Catalog catalog; + + @RequestMapping(path = "/api/{" + DB_KEY + "}/_load_info", method = RequestMethod.GET) + public Object execute( + @PathVariable(value = DB_KEY) final String dbName, + HttpServletRequest request, HttpServletResponse response) { + executeCheckPassword(request, response); + + this.catalog = Catalog.getCurrentCatalog(); + String fullDbName = getFullDbName(dbName); + + Load.JobInfo info = new Load.JobInfo(fullDbName, + request.getParameter(LABEL_KEY), + ConnectContext.get().getClusterName()); + if (Strings.isNullOrEmpty(info.dbName)) { + return ResponseEntityBuilder.badRequest("No database selected"); + } + if (Strings.isNullOrEmpty(info.label)) { + return ResponseEntityBuilder.badRequest("No label selected"); + } + if (Strings.isNullOrEmpty(info.clusterName)) { + return ResponseEntityBuilder.badRequest("No cluster selected"); + } + + RedirectView redirectView = redirectToMaster(request, response); + if (redirectView != null) { + return redirectView; + } + + try { + catalog.getLoadInstance().getJobInfo(info); + if (info.tblNames.isEmpty()) { + checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), info.dbName, PrivPredicate.LOAD); + } else { + for (String tblName : info.tblNames) { + checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), info.dbName, tblName, + PrivPredicate.LOAD); + } + } + } catch (DdlException | MetaNotFoundException e) { + try { + catalog.getLoadManager().getLoadJobInfo(info); + } catch (DdlException e1) { + return ResponseEntityBuilder.okWithCommonError(e1.getMessage()); + } + } + return ResponseEntityBuilder.ok(info); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLogFileAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLogFileAction.java new file mode 100644 index 0000000000..609b896d98 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLogFileAction.java @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.common.Config; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.base.Strings; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +import org.codehaus.jackson.map.ObjectMapper; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.io.File; +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/* + * get log file infos: + * curl -I http://fe_host:http_port/api/get_log_file?type=fe.audit.log + * return: + * HTTP/1.1 200 OK + * file_infos: {"fe.audit.log":24759,"fe.audit.log.20190528.1":132934} + * content-type: text/html + * connection: keep-alive + * + * get log file: + * curl -X GET http://fe_host:http_port/api/get_log_file?type=fe.audit.log&file=fe.audit.log.20190528.1 + */ +@RestController +public class GetLogFileAction extends RestBaseController { + private final Set logFileTypes = Sets.newHashSet("fe.audit.log"); + + @RequestMapping(path = "/api/get_log_file", method = {RequestMethod.GET, RequestMethod.HEAD}) + public Object execute(HttpServletRequest request, HttpServletResponse response) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + + String logType = request.getParameter("type"); + String logFile = request.getParameter("file"); + + // check param empty + if (Strings.isNullOrEmpty(logType)) { + return ResponseEntityBuilder.badRequest("Miss type parameter"); + } + + // check type valid or not + if (!logFileTypes.contains(logType)) { + return ResponseEntityBuilder.badRequest("log type: " + logType + " is invalid!"); + } + + String method = request.getMethod(); + if (method.equals(RequestMethod.HEAD.name())) { + String fileInfos = getFileInfos(logType); + response.setHeader("file_infos", fileInfos); + return ResponseEntityBuilder.ok(); + } else if (method.equals(RequestMethod.GET.name())) { + File log = getLogFile(logType, logFile); + if (!log.exists() || !log.isFile()) { + return ResponseEntityBuilder.okWithCommonError("Log file not exist: " + log.getName()); + } + if (log != null) { + try { + getFile(request, response, log, log.getName()); + } catch (IOException e) { + return ResponseEntityBuilder.internalError(e.getMessage()); + } + } else { + return ResponseEntityBuilder.okWithCommonError("Log file not exist: " + log.getName()); + } + } + return ResponseEntityBuilder.ok(); + } + + private String getFileInfos(String logType) { + Map fileInfos = Maps.newTreeMap(); + if (logType.equals("fe.audit.log")) { + File logDir = new File(Config.audit_log_dir); + File[] files = logDir.listFiles(); + for (int i = 0; i < files.length; i++) { + if (files[i].isFile() && files[i].getName().startsWith("fe.audit.log")) { + fileInfos.put(files[i].getName(), files[i].length()); + } + } + } + + String result = ""; + ObjectMapper mapper = new ObjectMapper(); + try { + result = mapper.writeValueAsString(fileInfos); + } catch (Exception e) { + // do nothing + } + return result; + } + + private File getLogFile(String logType, String logFile) { + String logPath = ""; + if ("fe.audit.log".equals(logType)) { + logPath = Config.audit_log_dir + "/" + logFile; + } + return new File(logPath); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetSmallFileAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetSmallFileAction.java new file mode 100644 index 0000000000..8e70312eb3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetSmallFileAction.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.util.SmallFileMgr; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; + +import com.google.common.base.Strings; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.io.IOException; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +@RestController +public class GetSmallFileAction extends RestBaseController { + private static final Logger LOG = LogManager.getLogger(GetSmallFileAction.class); + + @RequestMapping(path = "/api/get_small_file", method = RequestMethod.GET) + public Object execute(HttpServletRequest request, HttpServletResponse response) { + String token = request.getParameter("token"); + String fileIdStr = request.getParameter("file_id"); + // check param empty + if (Strings.isNullOrEmpty(token) || Strings.isNullOrEmpty(fileIdStr)) { + return ResponseEntityBuilder.badRequest("Missing parameter. Need token and file id"); + } + + // check token + if (!token.equals(Catalog.getCurrentCatalog().getToken())) { + return ResponseEntityBuilder.okWithCommonError("Invalid token"); + } + + long fileId = -1; + try { + fileId = Long.valueOf(fileIdStr); + } catch (NumberFormatException e) { + return ResponseEntityBuilder.badRequest("Invalid file id format: " + fileIdStr); + } + + SmallFileMgr fileMgr = Catalog.getCurrentCatalog().getSmallFileMgr(); + SmallFileMgr.SmallFile smallFile = fileMgr.getSmallFile(fileId); + if (smallFile == null || !smallFile.isContent) { + return ResponseEntityBuilder.okWithCommonError("File not found or is not content"); + } + + String method = request.getMethod(); + if (method.equalsIgnoreCase("GET")) { + try { + getFile(request, response, smallFile.getContentBytes(), smallFile.name); + } catch (IOException e) { + return ResponseEntityBuilder.internalError(e.getMessage()); + } + } + return ResponseEntityBuilder.ok(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetStreamLoadState.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetStreamLoadState.java new file mode 100644 index 0000000000..194b63fafa --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetStreamLoadState.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Database; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; + +import com.google.common.base.Strings; + +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.view.RedirectView; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +@RestController +public class GetStreamLoadState extends RestBaseController { + + @RequestMapping(path = "/api/{" + DB_KEY + "}/get_load_state", method = RequestMethod.GET) + public Object execute(@PathVariable(value = DB_KEY) final String dbName, + HttpServletRequest request, HttpServletResponse response) { + executeCheckPassword(request, response); + + RedirectView redirectView = redirectToMaster(request, response); + if (redirectView != null) { + return redirectView; + } + + String label = request.getParameter(LABEL_KEY); + if (Strings.isNullOrEmpty(label)) { + return ResponseEntityBuilder.badRequest("No label selected"); + } + + final String fullDbName = getFullDbName(dbName); + + Database db = Catalog.getCurrentCatalog().getDb(fullDbName); + if (db == null) { + return ResponseEntityBuilder.okWithCommonError("unknown database, database=" + dbName); + } + + String state = Catalog.getCurrentGlobalTransactionMgr().getLabelState(db.getId(), label).toString(); + return ResponseEntityBuilder.ok(state); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/HealthAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/HealthAction.java new file mode 100644 index 0000000000..57b42ce567 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/HealthAction.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; + +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.HashMap; +import java.util.Map; + +@RestController +public class HealthAction extends RestBaseController { + + @RequestMapping(path = "/api/health", method = RequestMethod.GET) + public Object execute() { + Map result = new HashMap<>(); + result.put("total_backend_num", Catalog.getCurrentSystemInfo().getBackendIds(false).size()); + result.put("online_backend_num", Catalog.getCurrentSystemInfo().getBackendIds(true).size()); + return ResponseEntityBuilder.ok(result); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java new file mode 100644 index 0000000000..696467cac5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import io.netty.handler.codec.http.HttpHeaderNames; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.cluster.ClusterNamespace; +import org.apache.doris.common.DdlException; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.service.ExecuteEnv; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TNetworkAddress; + +import com.google.common.base.Strings; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.view.RedirectView; + +import java.util.List; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +@RestController +public class LoadAction extends RestBaseController { + + private static final Logger LOG = LogManager.getLogger(LoadAction.class); + + public static final String SUB_LABEL_NAME_PARAM = "sub_label"; + + private ExecuteEnv execEnv = ExecuteEnv.getInstance(); + + private boolean isStreamLoad = false; + + @RequestMapping(path = "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_load", method = RequestMethod.PUT) + public Object load(HttpServletRequest request, HttpServletResponse response, + @PathVariable(value = DB_KEY) String db, @PathVariable(value = TABLE_KEY) String table) + throws DdlException { + this.isStreamLoad = false; + executeCheckPassword(request, response); + return executeWithoutPassword(request, response, db, table); + } + + @RequestMapping(path = "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_stream_load", method = RequestMethod.PUT) + public Object streamLoad(HttpServletRequest request, + HttpServletResponse response, + @PathVariable(value = DB_KEY) String db, @PathVariable(value = TABLE_KEY) String table) { + this.isStreamLoad = true; + executeCheckPassword(request, response); + return executeWithoutPassword(request, response, db, table); + } + + private Object executeWithoutPassword(HttpServletRequest request, + HttpServletResponse response, String db, String table) { + String dbName = db; + String tableName = table; + String urlStr = request.getRequestURI(); + // A 'Load' request must have 100-continue header + if (request.getHeader(HttpHeaderNames.EXPECT.toString()) == null) { + return ResponseEntityBuilder.notFound("There is no 100-continue header"); + } + + final String clusterName = ConnectContext.get().getClusterName(); + if (Strings.isNullOrEmpty(clusterName)) { + return ResponseEntityBuilder.badRequest("No cluster selected."); + } + + if (Strings.isNullOrEmpty(dbName)) { + return ResponseEntityBuilder.badRequest("No database selected."); + } + + if (Strings.isNullOrEmpty(tableName)) { + return ResponseEntityBuilder.badRequest("No table selected."); + } + + String fullDbName = ClusterNamespace.getFullName(clusterName, dbName); + + String label = request.getParameter(LABEL_KEY); + if (isStreamLoad) { + label = request.getHeader(LABEL_KEY); + } + + if (!isStreamLoad && Strings.isNullOrEmpty(label)) { + // for stream load, the label can be generated by system automatically + return ResponseEntityBuilder.badRequest("No label selected."); + } + + // check auth + checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tableName, PrivPredicate.LOAD); + + if (!isStreamLoad && !Strings.isNullOrEmpty(request.getParameter(SUB_LABEL_NAME_PARAM))) { + // only multi mini load need to redirect to Master, because only Master has the info of table to + // the Backend which the file exists. + RedirectView redirectView = redirectToMaster(request, response); + if (redirectView != null) { + return redirectView; + } + } + + // Choose a backend sequentially. + List backendIds = Catalog.getCurrentSystemInfo().seqChooseBackendIds(1, true, false, clusterName); + if (backendIds == null) { + return ResponseEntityBuilder.okWithCommonError("No backend alive."); + } + + Backend backend = Catalog.getCurrentSystemInfo().getBackend(backendIds.get(0)); + if (backend == null) { + return ResponseEntityBuilder.okWithCommonError("No backend alive."); + } + + TNetworkAddress redirectAddr = new TNetworkAddress(backend.getHost(), backend.getHttpPort()); + + if (!isStreamLoad) { + String subLabel = request.getParameter(SUB_LABEL_NAME_PARAM); + if (!Strings.isNullOrEmpty(subLabel)) { + try { + redirectAddr = execEnv.getMultiLoadMgr().redirectAddr(fullDbName, label, tableName, redirectAddr); + } catch (DdlException e) { + return ResponseEntityBuilder.okWithCommonError(e.getMessage()); + } + } + } + + LOG.info("redirect load action to destination={}, stream: {}, db: {}, tbl: {}, label: {}", + redirectAddr.toString(), isStreamLoad, dbName, tableName, label); + + RedirectView redirectView = redirectTo(request, redirectAddr); + return redirectView; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetaInfoAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetaInfoAction.java new file mode 100644 index 0000000000..6b4831041c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetaInfoAction.java @@ -0,0 +1,333 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.cluster.ClusterNamespace; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; +import org.apache.doris.common.proc.ProcNodeInterface; +import org.apache.doris.common.proc.ProcResult; +import org.apache.doris.common.proc.ProcService; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.httpv2.exception.BadRequestException; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.SystemInfoService; + +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * And meta info like databases, tables and schema + */ +@RestController +public class MetaInfoAction extends RestBaseController { + + private static final String NAMESPACES = "namespaces"; + private static final String DATABASES = "databases"; + private static final String TABLES = "tables"; + private static final String PARAM_LIMIT = "limit"; + private static final String PARAM_OFFSET = "offset"; + private static final String PARAM_WITH_MV = "with_mv"; + + + /** + * Get all databases + * { + * "msg": "success", + * "code": 0, + * "data": [ + * "default_cluster:db1", + * "default_cluster:doris_audit_db__", + * "default_cluster:information_schema" + * ], + * "count": 0 + * } + */ + @RequestMapping(path = "/api/meta/" + NAMESPACES + "/{" + NS_KEY + "}/" + DATABASES, + method = {RequestMethod.GET}) + public Object getAllDatabases( + @PathVariable(value = NS_KEY) String ns, + HttpServletRequest request, HttpServletResponse response) { + checkWithCookie(request, response, false); + + if (!ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + return ResponseEntityBuilder.badRequest("Only support 'default_cluster' now"); + } + + // 1. get all database with priviledge + List dbNames = null; + try { + dbNames = Catalog.getCurrentCatalog().getClusterDbNames(ns); + } catch (AnalysisException e) { + return ResponseEntityBuilder.okWithCommonError("namespace does not exist: " + ns); + } + List dbNameSet = Lists.newArrayList(); + for (String fullName : dbNames) { + final String db = ClusterNamespace.getNameFromFullName(fullName); + if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(ConnectContext.get(), fullName, + PrivPredicate.SHOW)) { + continue; + } + dbNameSet.add(db); + } + + Collections.sort(dbNames); + + // handle limit offset + Pair fromToIndex = getFromToIndex(request, dbNames.size()); + return ResponseEntityBuilder.ok(dbNames.subList(fromToIndex.first, fromToIndex.second)); + } + + /** Get all tables of a database + * { + * "msg": "success", + * "code": 0, + * "data": [ + * "tbl1", + * "tbl2" + * ], + * "count": 0 + * } + */ + + @RequestMapping(path = "/api/meta/" + NAMESPACES + "/{" + NS_KEY + "}/" + DATABASES + "/{" + DB_KEY + "}/" + TABLES, + method = {RequestMethod.GET}) + public Object getTables( + @PathVariable(value = NS_KEY) String ns, @PathVariable(value = DB_KEY) String dbName, + HttpServletRequest request, HttpServletResponse response) { + checkWithCookie(request, response, false); + + if (!ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + return ResponseEntityBuilder.badRequest("Only support 'default_cluster' now"); + } + + + String fullDbName = getFullDbName(dbName); + Database db = Catalog.getCurrentCatalog().getDb(fullDbName); + if (db == null) { + return ResponseEntityBuilder.okWithCommonError("Database does not exist: " + fullDbName); + } + + List tblNames = Lists.newArrayList(); + db.readLock(); + try { + for (Table tbl : db.getTables()) { + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), fullDbName, tbl.getName(), + PrivPredicate.SHOW)) { + continue; + } + tblNames.add(tbl.getName()); + } + } finally { + db.readUnlock(); + } + + Collections.sort(tblNames); + + // handle limit offset + Pair fromToIndex = getFromToIndex(request, tblNames.size()); + return ResponseEntityBuilder.ok(tblNames.subList(fromToIndex.first, fromToIndex.second)); + } + + /** Get schema of a table + * { + * "msg": "success", + * "code": 0, + * "data": { + * "tbl1": { + * "schema": [{ + * "Field": "k1", + * "Type": "INT", + * "Null": "Yes", + * "Extra": "", + * "Default": null, + * "Key": "true" + * }, { + * "Field": "k2", + * "Type": "INT", + * "Null": "Yes", + * "Extra": "", + * "Default": null, + * "Key": "true" + * }], + * "is_base": true + * }, + * "r1": { + * "schema": [{ + * "Field": "k1", + * "Type": "INT", + * "Null": "Yes", + * "Extra": "", + * "Default": null, + * "Key": "true" + * }], + * "is_base": false + * } + * }, + * "count": 0 + * } + */ + @RequestMapping(path = "/api/meta/" + NAMESPACES + "/{" + NS_KEY + "}/" + DATABASES + "/{" + DB_KEY + "}/" + TABLES + + "/{" + TABLE_KEY + "}/schema", + method = {RequestMethod.GET}) + public Object getTableSchema( + @PathVariable(value = NS_KEY) String ns, @PathVariable(value = DB_KEY) String dbName, + @PathVariable(value = TABLE_KEY) String tblName, + HttpServletRequest request, HttpServletResponse response) throws UserException { + checkWithCookie(request, response, false); + + if (!ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + return ResponseEntityBuilder.badRequest("Only support 'default_cluster' now"); + } + + String fullDbName = getFullDbName(dbName); + checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tblName, PrivPredicate.SHOW); + + Database db = Catalog.getCurrentCatalog().getDb(fullDbName); + if (db == null) { + return ResponseEntityBuilder.okWithCommonError("Database does not exist: " + fullDbName); + } + + String withMvPara = request.getParameter(PARAM_WITH_MV); + boolean withMv = Strings.isNullOrEmpty(withMvPara) ? false : withMvPara.equals("1"); + + // get all proc paths + Map> result = Maps.newHashMap(); + db.readLock(); + try { + Table tbl = db.getTable(tblName); + if (tbl == null) { + return ResponseEntityBuilder.okWithCommonError("Table does not exist: " + tblName); + } + long baseId = -1; + if (tbl.getType() == Table.TableType.OLAP) { + baseId = ((OlapTable) tbl).getBaseIndexId(); + } else { + baseId += tbl.getId(); + } + String procPath = Joiner.on("/").join("", "dbs", db.getId(), tbl.getId(), "index_schema/", baseId); + generateResult(tblName, true, procPath, result); + + if (withMv && tbl.getType() == Table.TableType.OLAP) { + OlapTable olapTable = (OlapTable) tbl; + for (long indexId : olapTable.getIndexIdListExceptBaseIndex()) { + procPath = Joiner.on("/").join("", "dbs", db.getId(), tbl.getId(), "index_schema/", indexId); + generateResult(olapTable.getIndexNameById(indexId), false, procPath, result); + } + } + } finally { + db.readUnlock(); + } + + return ResponseEntityBuilder.ok(result); + } + + private void generateResult(String indexName, boolean isBaseIndex, String procPath, + Map> result) throws UserException { + Map propMap = result.get(indexName); + if (propMap == null) { + propMap = Maps.newHashMap(); + result.put(indexName, propMap); + } + + propMap.put("is_base", isBaseIndex); + propMap.put("schema", generateSchema(procPath)); + } + + List> generateSchema(String procPath) throws UserException { + ProcNodeInterface node = ProcService.getInstance().open(procPath); + if (node == null) { + throw new DdlException("get schema with proc path failed: " + procPath); + } + + List> schema = Lists.newArrayList(); + ProcResult procResult = node.fetchResult(); + List colNames = procResult.getColumnNames(); + List> rows = procResult.getRows(); + for (List row : rows) { + Preconditions.checkState(row.size() == colNames.size()); + Map fieldMap = Maps.newHashMap(); + for (int i = 0; i < row.size(); i++) { + fieldMap.put(colNames.get(i), convertIfNull(row.get(i))); + } + schema.add(fieldMap); + } + return schema; + } + + private String convertIfNull(String val) { + return val.equals(FeConstants.null_string) ? null : val; + } + + // get limit and offset from query parameter + // and return fromIndex and toIndex of a list + private Pair getFromToIndex(HttpServletRequest request, int maxNum) { + String limitStr = request.getParameter(PARAM_LIMIT); + String offsetStr = request.getParameter(PARAM_OFFSET); + + int offset = 0; + int limit = Integer.MAX_VALUE; + if (Strings.isNullOrEmpty(limitStr)) { + // limit not set + if (!Strings.isNullOrEmpty(offsetStr)) { + throw new BadRequestException("Param offset should be set with param limit"); + } + } else { + // limit is set + limit = Integer.valueOf(limitStr); + if (limit < 0) { + throw new BadRequestException("Param limit should >= 0"); + } + + offset = 0; + if (!Strings.isNullOrEmpty(offsetStr)) { + offset = Integer.valueOf(offsetStr); + if (offset < 0) { + throw new BadRequestException("Param offset should >= 0"); + } + } + } + + if (maxNum <= 0) { + return Pair.create(0, 0); + } + return Pair.create(Math.min(offset, maxNum - 1), Math.min(limit + offset, maxNum)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetaReplayerCheckAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetaReplayerCheckAction.java new file mode 100644 index 0000000000..5b413a58ba --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetaReplayerCheckAction.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; + +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/* + * used to get meta replay info + * eg: + * fe_host:http_port/api/_meta_replay_state + */ +@RestController +public class MetaReplayerCheckAction extends RestBaseController { + + @RequestMapping(path = "/api/_meta_replay_state", method = RequestMethod.GET) + public Object execute(HttpServletRequest request, HttpServletResponse response) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + + Map resultMap = Catalog.getCurrentCatalog().getMetaReplayState().getInfo(); + + return ResponseEntityBuilder.ok(resultMap); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetricsAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetricsAction.java new file mode 100644 index 0000000000..a82840a477 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetricsAction.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.metric.JsonMetricVisitor; +import org.apache.doris.metric.MetricRepo; +import org.apache.doris.metric.MetricVisitor; +import org.apache.doris.metric.PrometheusMetricVisitor; +import org.apache.doris.metric.SimpleCoreMetricVisitor; + +import com.google.common.base.Strings; + +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.io.IOException; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +//fehost:port/metrics +//fehost:port/metrics?type=core +@RestController +public class MetricsAction { + + private static final String TYPE_PARAM = "type"; + + @RequestMapping(path = "/metrics") + public void execute(HttpServletRequest request, HttpServletResponse response) { + String type = request.getParameter(TYPE_PARAM); + MetricVisitor visitor = null; + if (!Strings.isNullOrEmpty(type) && type.equalsIgnoreCase("core")) { + visitor = new SimpleCoreMetricVisitor("doris_fe"); + } else if (!Strings.isNullOrEmpty(type) && type.equalsIgnoreCase("agent")) { + visitor = new JsonMetricVisitor("doris_fe"); + } else { + visitor = new PrometheusMetricVisitor("doris_fe"); + } + response.setContentType("text/plain"); + try { + response.getWriter().write(MetricRepo.getMetric(visitor)); + } catch (IOException e) { + e.printStackTrace(); + } + + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java new file mode 100644 index 0000000000..dd0a99387c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java @@ -0,0 +1,223 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.analysis.LoadStmt; +import org.apache.doris.common.DdlException; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.service.ExecuteEnv; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.view.RedirectView; + +import java.util.List; +import java.util.Map; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +// List all labels of one multi-load +@RestController +public class MultiAction extends RestBaseController { + private ExecuteEnv execEnv; + private static final String SUB_LABEL_KEY = "sub_label"; + + + @RequestMapping(path = "/api/{" + DB_KEY + "}/_multi_desc", method = RequestMethod.POST) + public Object multi_desc( + @PathVariable(value = DB_KEY) final String dbName, + HttpServletRequest request, HttpServletResponse response) + throws DdlException { + executeCheckPassword(request, response); + + execEnv = ExecuteEnv.getInstance(); + String label = request.getParameter(LABEL_KEY); + if (Strings.isNullOrEmpty(label)) { + return ResponseEntityBuilder.badRequest("No label selected"); + } + + String fullDbName = getFullDbName(dbName); + checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); + + // only Master has these load info + RedirectView redirectView = redirectToMaster(request, response); + if (redirectView != null) { + return redirectView; + } + + execEnv = ExecuteEnv.getInstance(); + final List labels = Lists.newArrayList(); + execEnv.getMultiLoadMgr().desc(fullDbName, label, labels); + return ResponseEntityBuilder.ok(labels); + } + + + @RequestMapping(path = "/api/{" + DB_KEY + "}/_multi_list", method = RequestMethod.POST) + public Object multi_list( + @PathVariable(value = DB_KEY) final String dbName, + HttpServletRequest request, HttpServletResponse response) + throws DdlException { + executeCheckPassword(request, response); + execEnv = ExecuteEnv.getInstance(); + + String fullDbName = getFullDbName(dbName); + checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); + + // only Master has these load info + + RedirectView redirectView = redirectToMaster(request, response); + if (redirectView != null) { + return redirectView; + } + + final List labels = Lists.newArrayList(); + execEnv.getMultiLoadMgr().list(fullDbName, labels); + return ResponseEntityBuilder.ok(labels); + } + + @RequestMapping(path = "/api/{" + DB_KEY + "}/_multi_start", method = RequestMethod.POST) + public Object multi_start( + @PathVariable(value = DB_KEY) final String dbName, + HttpServletRequest request, HttpServletResponse response) + throws DdlException { + executeCheckPassword(request, response); + execEnv = ExecuteEnv.getInstance(); + + String label = request.getParameter(LABEL_KEY); + if (Strings.isNullOrEmpty(label)) { + return ResponseEntityBuilder.badRequest("No label selected"); + } + String fullDbName = getFullDbName(dbName); + checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); + + // Mutli start request must redirect to master, because all following sub requests will be handled + // on Master + + RedirectView redirectView = redirectToMaster(request, response); + if (redirectView != null) { + return redirectView; + } + + Map properties = Maps.newHashMap(); + String[] keys = {LoadStmt.TIMEOUT_PROPERTY, LoadStmt.MAX_FILTER_RATIO_PROPERTY}; + for (String key : keys) { + String value = request.getParameter(key); + if (!Strings.isNullOrEmpty(value)) { + properties.put(key, value); + } + } + execEnv.getMultiLoadMgr().startMulti(fullDbName, label, properties); + return ResponseEntityBuilder.ok(); + } + + @RequestMapping(path = "/api/{" + DB_KEY + "}/_multi_unload", method = RequestMethod.POST) + public Object multi_unload( + @PathVariable(value = DB_KEY) final String dbName, + HttpServletRequest request, HttpServletResponse response) + throws DdlException { + executeCheckPassword(request, response); + execEnv = ExecuteEnv.getInstance(); + + String label = request.getParameter(LABEL_KEY); + if (Strings.isNullOrEmpty(label)) { + return ResponseEntityBuilder.badRequest("No label selected"); + } + + String subLabel = request.getParameter(SUB_LABEL_KEY); + if (Strings.isNullOrEmpty(subLabel)) { + return ResponseEntityBuilder.badRequest("No sub label selected"); + } + + String fullDbName = getFullDbName(dbName); + checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); + + + RedirectView redirectView = redirectToMaster(request, response); + if (redirectView != null) { + return redirectView; + } + + execEnv.getMultiLoadMgr().unload(fullDbName, label, subLabel); + return ResponseEntityBuilder.ok(); + } + + + @RequestMapping(path = "/api/{" + DB_KEY + "}/_multi_commit", method = RequestMethod.POST) + public Object multi_commit( + @PathVariable(value = DB_KEY) final String dbName, + HttpServletRequest request, HttpServletResponse response) + throws DdlException { + executeCheckPassword(request, response); + execEnv = ExecuteEnv.getInstance(); + + String label = request.getParameter(LABEL_KEY); + if (Strings.isNullOrEmpty(label)) { + return ResponseEntityBuilder.badRequest("No label selected"); + } + + String fullDbName = getFullDbName(dbName); + checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); + + // only Master has these load info + + RedirectView redirectView = redirectToMaster(request, response); + if (redirectView != null) { + return redirectView; + } + + execEnv.getMultiLoadMgr().commit(fullDbName, label); + return ResponseEntityBuilder.ok(); + } + + @RequestMapping(path = "/api/{" + DB_KEY + "}/_multi_abort", method = RequestMethod.POST) + public Object multi_abort( + @PathVariable(value = DB_KEY) final String dbName, + HttpServletRequest request, HttpServletResponse response) + throws DdlException { + executeCheckPassword(request, response); + execEnv = ExecuteEnv.getInstance(); + + String label = request.getParameter(LABEL_KEY); + if (Strings.isNullOrEmpty(label)) { + return ResponseEntityBuilder.badRequest("No label selected"); + } + + String fullDbName = getFullDbName(dbName); + checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); + + // only Master has these load info + + RedirectView redirectView = redirectToMaster(request, response); + if (redirectView != null) { + return redirectView; + } + + execEnv.getMultiLoadMgr().abort(fullDbName, label); + return ResponseEntityBuilder.ok(); + } +} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ProfileAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ProfileAction.java new file mode 100644 index 0000000000..66993297c7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ProfileAction.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.common.util.ProfileManager; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.base.Strings; +import com.google.common.collect.Maps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +// This class is a RESTFUL interface to get query profile. +// It will be used in query monitor to collect profiles. +// Usage: +// wget http://fe_host:fe_http_port/api/profile?query_id=123456 +@RestController +public class ProfileAction extends RestBaseController { + private static final Logger LOG = LogManager.getLogger(ProfileAction.class); + + @RequestMapping(path = "/api/profile", method = RequestMethod.GET) + protected Object profile(HttpServletRequest request, HttpServletResponse response) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + + String queryId = request.getParameter("query_id"); + if (Strings.isNullOrEmpty(queryId)) { + return ResponseEntityBuilder.badRequest("Missing query_id"); + } + + String queryProfileStr = ProfileManager.getInstance().getProfile(queryId); + if (queryProfileStr == null) { + return ResponseEntityBuilder.okWithCommonError("query id " + queryId + " not found."); + } + + Map result = Maps.newHashMap(); + result.put("profile", queryProfileStr); + return ResponseEntityBuilder.ok(result); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/QueryDetailAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/QueryDetailAction.java new file mode 100644 index 0000000000..5d674f2e89 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/QueryDetailAction.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.QueryDetail; +import org.apache.doris.qe.QueryDetailQueue; + +import com.google.common.base.Strings; +import com.google.common.collect.Maps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.Map; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +// This class is used to get current query_id of connection_id. +// Every connection holds at most one query at every point. +// Some we can get query_id firstly, and get query by query_id. +@RestController +public class QueryDetailAction extends RestBaseController { + private static final Logger LOG = LogManager.getLogger(QueryDetailAction.class); + + @RequestMapping(path = "/api/query_detail", method = RequestMethod.GET) + protected Object query_detail(HttpServletRequest request, HttpServletResponse response) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + + String eventTimeStr = request.getParameter("event_time"); + if (Strings.isNullOrEmpty(eventTimeStr)) { + return ResponseEntityBuilder.badRequest("Missing event_time"); + } + + long eventTime = Long.valueOf(eventTimeStr.trim()); + List queryDetails = QueryDetailQueue.getQueryDetails(eventTime); + + Map> result = Maps.newHashMap(); + result.put("query_details", queryDetails); + return ResponseEntityBuilder.ok(result); + } +} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestApiStatusCode.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestApiStatusCode.java new file mode 100644 index 0000000000..d260cc36b5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestApiStatusCode.java @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +public enum RestApiStatusCode { + OK(0), + COMMON_ERROR(1); + + public int code; + + RestApiStatusCode(int code) { + this.code = code; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java new file mode 100644 index 0000000000..89fcc230bf --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java @@ -0,0 +1,157 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Catalog; +import org.apache.doris.cluster.ClusterNamespace; +import org.apache.doris.httpv2.controller.BaseController; +import org.apache.doris.httpv2.exception.UnauthorizedException; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.thrift.TNetworkAddress; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.web.servlet.view.RedirectView; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; + +public class RestBaseController extends BaseController { + + protected static final String NS_KEY = "ns"; + protected static final String DB_KEY = "db"; + protected static final String TABLE_KEY = "table"; + protected static final String LABEL_KEY = "label"; + private static final Logger LOG = LogManager.getLogger(RestBaseController.class); + + public ActionAuthorizationInfo executeCheckPassword(HttpServletRequest request, + HttpServletResponse response) throws UnauthorizedException { + ActionAuthorizationInfo authInfo = getAuthorizationInfo(request); + // check password + UserIdentity currentUser = checkPassword(authInfo); + ConnectContext ctx = new ConnectContext(null); + ctx.setCatalog(Catalog.getCurrentCatalog()); + ctx.setQualifiedUser(authInfo.fullUserName); + ctx.setRemoteIP(authInfo.remoteIp); + ctx.setCurrentUserIdentity(currentUser); + ctx.setCluster(authInfo.cluster); + ctx.setThreadLocalInfo(); + return authInfo; + } + + + public RedirectView redirectTo(HttpServletRequest request, TNetworkAddress addr) { + URI urlObj = null; + URI resultUriObj = null; + String urlStr = request.getRequestURI(); + try { + urlObj = new URI(urlStr); + resultUriObj = new URI("http", null, addr.getHostname(), + addr.getPort(), urlObj.getPath(), "", null); + } catch (Exception e) { + throw new RuntimeException(e); + } + String redirectUrl = resultUriObj.toASCIIString(); + if (!Strings.isNullOrEmpty(request.getQueryString())) { + redirectUrl += request.getQueryString(); + } + LOG.info("redirect url: {}", redirectUrl); + RedirectView redirectView = new RedirectView(redirectUrl); + redirectView.setContentType("text/html;charset=utf-8"); + redirectView.setStatusCode(org.springframework.http.HttpStatus.TEMPORARY_REDIRECT); + return redirectView; + } + + public RedirectView redirectToMaster(HttpServletRequest request, HttpServletResponse response) { + Catalog catalog = Catalog.getCurrentCatalog(); + if (catalog.isMaster()) { + return null; + } + RedirectView redirectView = redirectTo(request, new TNetworkAddress(catalog.getMasterIp(), catalog.getMasterHttpPort())); + return redirectView; + } + + public void getFile(HttpServletRequest request, HttpServletResponse response, Object obj, String fileName) + throws IOException { + response.setHeader("Content-type", "application/octet-stream"); + response.addHeader("Content-Disposition", "attachment;fileName=" + fileName); // set file name + if (obj instanceof File) { + File file = (File) obj; + byte[] buffer = new byte[1024]; + FileInputStream fis = null; + BufferedInputStream bis = null; + try { + fis = new FileInputStream(file); + bis = new BufferedInputStream(fis); + OutputStream os = response.getOutputStream(); + int i = bis.read(buffer); + while (i != -1) { + os.write(buffer, 0, i); + i = bis.read(buffer); + } + return; + } finally { + if (bis != null) { + try { + bis.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + if (fis != null) { + try { + fis.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } else if (obj instanceof byte[]) { + OutputStream os = response.getOutputStream(); + os.write((byte[]) obj); + } + } + + public void writeFileResponse(HttpServletRequest request, HttpServletResponse response, File imageFile) throws IOException { + Preconditions.checkArgument(imageFile != null && imageFile.exists()); + response.setHeader("Content-type", "application/octet-stream"); + response.addHeader("Content-Disposition", "attachment;fileName=" + imageFile.getName()); + response.setHeader("X-Image-Size", imageFile.length() + ""); + getFile(request, response, imageFile, imageFile.getName()); + } + + public String getFullDbName(String dbName) { + String fullDbName = dbName; + String clusterName = ClusterNamespace.getClusterNameFromFullName(fullDbName); + if (clusterName == null) { + fullDbName = ClusterNamespace.getFullName(SystemInfoService.DEFAULT_CLUSTER, dbName); + } + return fullDbName; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RowCountAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RowCountAction.java new file mode 100644 index 0000000000..4c4a4adbad --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RowCountAction.java @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.Replica; +import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.Table.TableType; +import org.apache.doris.catalog.Tablet; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; + +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.google.common.base.Strings; +import com.google.common.collect.Maps; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.Map; + +/* + * calc row count from replica to table + * fe_host:fe_http_port/api/rowcount?db=dbname&table=tablename + */ +@RestController +public class RowCountAction extends RestBaseController { + + @RequestMapping(path = "/api/rowcount", method = RequestMethod.GET) + protected Object rowcount(HttpServletRequest request, HttpServletResponse response) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + + String dbName = request.getParameter(DB_KEY); + if (Strings.isNullOrEmpty(dbName)) { + return ResponseEntityBuilder.badRequest("No database selected"); + } + + String tableName = request.getParameter(TABLE_KEY); + if (Strings.isNullOrEmpty(tableName)) { + return ResponseEntityBuilder.badRequest("No table selected"); + } + + String fullDbName = getFullDbName(dbName); + Map indexRowCountMap = Maps.newHashMap(); + Catalog catalog = Catalog.getCurrentCatalog(); + Database db = catalog.getDb(fullDbName); + if (db == null) { + return ResponseEntityBuilder.okWithCommonError("Database[" + fullDbName + "] does not exist"); + } + db.writeLock(); + try { + Table table = db.getTable(tableName); + if (table == null) { + return ResponseEntityBuilder.okWithCommonError("Table[" + tableName + "] does not exist"); + } + + if (table.getType() != TableType.OLAP) { + return ResponseEntityBuilder.okWithCommonError("Table[" + tableName + "] is not OLAP table"); + } + + OlapTable olapTable = (OlapTable) table; + for (Partition partition : olapTable.getAllPartitions()) { + long version = partition.getVisibleVersion(); + long versionHash = partition.getVisibleVersionHash(); + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + long indexRowCount = 0L; + for (Tablet tablet : index.getTablets()) { + long tabletRowCount = 0L; + for (Replica replica : tablet.getReplicas()) { + if (replica.checkVersionCatchUp(version, versionHash, false) + && replica.getRowCount() > tabletRowCount) { + tabletRowCount = replica.getRowCount(); + } + } + indexRowCount += tabletRowCount; + } // end for tablets + index.setRowCount(indexRowCount); + indexRowCountMap.put(olapTable.getIndexNameById(index.getId()), indexRowCount); + } // end for indices + } // end for partitions + } finally { + db.writeUnlock(); + } + return ResponseEntityBuilder.ok(indexRowCountMap); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/SetConfigAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/SetConfigAction.java new file mode 100644 index 0000000000..ae547fe224 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/SetConfigAction.java @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.ConfigBase; +import org.apache.doris.common.ConfigBase.ConfField; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.google.common.collect.Maps; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.lang.reflect.Field; +import java.util.Map; + +/* + * used to set fe config + * eg: + * fe_host:http_port/api/_set_config?config_key1=config_value1&config_key2=config_value2&... + */ +@RestController +public class SetConfigAction extends RestBaseController { + private static final Logger LOG = LogManager.getLogger(SetConfigAction.class); + + @RequestMapping(path = "/api/_set_config", method = RequestMethod.GET) + protected Object set_config(HttpServletRequest request, HttpServletResponse response) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + + Map configs = request.getParameterMap(); + Map setConfigs = Maps.newHashMap(); + Map errConfigs = Maps.newHashMap(); + + LOG.debug("get config from url: {}", configs); + + Field[] fields = ConfigBase.confClass.getFields(); + for (Field f : fields) { + // ensure that field has "@ConfField" annotation + ConfField anno = f.getAnnotation(ConfField.class); + if (anno == null || !anno.mutable()) { + continue; + } + + if (anno.masterOnly() && !Catalog.getCurrentCatalog().isMaster()) { + continue; + } + + // ensure that field has property string + String confKey = anno.value().equals("") ? f.getName() : anno.value(); + String[] confVals = configs.get(confKey); + if (confVals == null || confVals.length == 0) { + continue; + } + + if (confVals.length > 1) { + continue; + } + + try { + ConfigBase.setConfigField(f, confVals[0]); + } catch (Exception e) { + LOG.warn("failed to set config {}:{}, {}", confKey, confVals[0], e.getMessage()); + continue; + } + + setConfigs.put(confKey, confVals[0]); + } + + for (String key : configs.keySet()) { + if (!setConfigs.containsKey(key)) { + String[] confVals = configs.get(key); + String confVal = confVals.length == 1 ? confVals[0] : "invalid value"; + errConfigs.put(key, confVal); + } + } + + Map> resultMap = Maps.newHashMap(); + resultMap.put("set", setConfigs); + resultMap.put("err", errConfigs); + + return ResponseEntityBuilder.ok(resultMap); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java new file mode 100644 index 0000000000..f08a7a2a9f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java @@ -0,0 +1,284 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.doris.httpv2.rest; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.Table.TableType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; +import org.apache.doris.common.proc.ProcNodeInterface; +import org.apache.doris.common.proc.ProcResult; +import org.apache.doris.common.proc.ProcService; +import org.apache.doris.ha.HAProtocol; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.persist.Storage; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.Maps; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.view.RedirectView; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +@RestController +public class ShowAction extends RestBaseController { + + private static final Logger LOG = LogManager.getLogger(ShowAction.class); + + private enum Action { + SHOW_DB_SIZE, + SHOW_HA, + INVALID; + + public static Action getAction(String str) { + try { + return valueOf(str); + } catch (Exception ex) { + return INVALID; + } + } + } + + @RequestMapping(path = "/api/show_meta_info", method = RequestMethod.GET) + public Object show_meta_info(HttpServletRequest request, HttpServletResponse response) { + String action = request.getParameter("action"); + if (Strings.isNullOrEmpty(action)) { + return ResponseEntityBuilder.badRequest("Missing action parameter"); + } + switch (Action.getAction(action.toUpperCase())) { + case SHOW_DB_SIZE: + return ResponseEntityBuilder.ok(getDataSize()); + case SHOW_HA: + try { + return ResponseEntityBuilder.ok(getHaInfo()); + } catch (IOException e) { + return ResponseEntityBuilder.internalError(e.getMessage()); + } + default: + return ResponseEntityBuilder.badRequest("Unknown action: " + action); + } + } + + // Format: + //http://username:password@192.168.1.1:8030/api/show_proc?path=/ + @RequestMapping(path = "/api/show_proc", method = RequestMethod.GET) + public Object show_proc(HttpServletRequest request, HttpServletResponse response) { + executeCheckPassword(request, response); + // check authority + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + + String path = request.getParameter("path"); + String forward = request.getParameter("forward"); + boolean isForward = false; + if (!Strings.isNullOrEmpty(forward) && forward.equals("true")) { + isForward = true; + } + + // forward to master if necessary + if (!Catalog.getCurrentCatalog().isMaster() && isForward) { + RedirectView redirectView = redirectToMaster(request, response); + Preconditions.checkNotNull(redirectView); + return redirectView; + } else { + ProcNodeInterface procNode = null; + ProcService instance = ProcService.getInstance(); + try { + if (Strings.isNullOrEmpty(path)) { + procNode = instance.open("/"); + } else { + procNode = instance.open(path); + } + } catch (AnalysisException e) { + return ResponseEntityBuilder.okWithCommonError(e.getMessage()); + } + + if (procNode != null) { + ProcResult result; + try { + result = procNode.fetchResult(); + List> rows = result.getRows(); + return ResponseEntityBuilder.ok(rows); + } catch (AnalysisException e) { + return ResponseEntityBuilder.okWithCommonError(e.getMessage()); + } + } else { + return ResponseEntityBuilder.badRequest("Invalid proc path: " + path); + } + } + } + + @RequestMapping(path = "/api/show_runtime_info", method = RequestMethod.GET) + public Object show_runtime_info(HttpServletRequest request, HttpServletResponse response) { + HashMap feInfo = new HashMap(); + + // Get memory info + Runtime r = Runtime.getRuntime(); + feInfo.put("free_mem", String.valueOf(r.freeMemory())); + feInfo.put("total_mem", String.valueOf(r.totalMemory())); + feInfo.put("max_mem", String.valueOf(r.maxMemory())); + + // Get thread count + ThreadGroup parentThread; + for (parentThread = Thread.currentThread().getThreadGroup(); + parentThread.getParent() != null; + parentThread = parentThread.getParent()) { + } + ; + feInfo.put("thread_cnt", String.valueOf(parentThread.activeCount())); + + return ResponseEntityBuilder.ok(feInfo); + } + + @RequestMapping(path = "/api/show_data", method = RequestMethod.GET) + public Object show_data(HttpServletRequest request, HttpServletResponse response) { + + Map oneEntry = Maps.newHashMap(); + + String dbName = request.getParameter(DB_KEY); + ConcurrentHashMap fullNameToDb = Catalog.getCurrentCatalog().getFullNameToDb(); + long totalSize = 0; + if (dbName != null) { + String fullDbName = getFullDbName(dbName); + Database db = fullNameToDb.get(fullDbName); + if (db == null) { + return ResponseEntityBuilder.okWithCommonError("database " + fullDbName + " not found."); + } + totalSize = getDataSizeOfDatabase(db); + oneEntry.put(fullDbName, totalSize); + } else { + for (Database db : fullNameToDb.values()) { + if (db.isInfoSchemaDb()) { + continue; + } + totalSize += getDataSizeOfDatabase(db); + } + oneEntry.put("__total_size", totalSize); + } + return ResponseEntityBuilder.ok(oneEntry); + } + + private Map getHaInfo() throws IOException { + HashMap feInfo = new HashMap(); + feInfo.put("role", Catalog.getCurrentCatalog().getFeType().toString()); + if (Catalog.getCurrentCatalog().isMaster()) { + feInfo.put("current_journal_id", + String.valueOf(Catalog.getCurrentCatalog().getEditLog().getMaxJournalId())); + } else { + feInfo.put("current_journal_id", + String.valueOf(Catalog.getCurrentCatalog().getReplayedJournalId())); + } + + HAProtocol haProtocol = Catalog.getCurrentCatalog().getHaProtocol(); + if (haProtocol != null) { + + InetSocketAddress master = null; + try { + master = haProtocol.getLeader(); + } catch (Exception e) { + // this may happen when majority of FOLLOWERS are down and no MASTER right now. + LOG.warn("failed to get leader: {}", e.getMessage()); + } + if (master != null) { + feInfo.put("master", master.getHostString()); + } else { + feInfo.put("master", "unknown"); + } + + List electableNodes = haProtocol.getElectableNodes(false); + ArrayList electableNodeNames = new ArrayList(); + if (electableNodes != null) { + for (InetSocketAddress node : electableNodes) { + electableNodeNames.add(node.getHostString()); + } + feInfo.put("electable_nodes", StringUtils.join(electableNodeNames.toArray(), ",")); + } + + List observerNodes = haProtocol.getObserverNodes(); + ArrayList observerNodeNames = new ArrayList(); + if (observerNodes != null) { + for (InetSocketAddress node : observerNodes) { + observerNodeNames.add(node.getHostString()); + } + feInfo.put("observer_nodes", StringUtils.join(observerNodeNames.toArray(), ",")); + } + } + + feInfo.put("can_read", String.valueOf(Catalog.getCurrentCatalog().canRead())); + feInfo.put("is_ready", String.valueOf(Catalog.getCurrentCatalog().isReady())); + + Storage storage = new Storage(Config.meta_dir + "/image"); + feInfo.put("last_checkpoint_version", String.valueOf(storage.getImageSeq())); + long lastCheckpointTime = storage.getCurrentImageFile().lastModified(); + feInfo.put("last_checkpoint_time", String.valueOf(lastCheckpointTime)); + + return feInfo; + } + + public long getDataSizeOfDatabase(Database db) { + long totalSize = 0; + db.readLock(); + try { + // sort by table name + List tables = db.getTables(); + for (Table table : tables) { + if (table.getType() != TableType.OLAP) { + continue; + } + + long tableSize = ((OlapTable) table).getDataSize(); + totalSize += tableSize; + } // end for tables + } finally { + db.readUnlock(); + } + return totalSize; + } + + private Map getDataSize() { + Map result = new HashMap(); + List dbNames = Catalog.getCurrentCatalog().getDbNames(); + + for (int i = 0; i < dbNames.size(); i++) { + String dbName = dbNames.get(i); + Database db = Catalog.getCurrentCatalog().getDb(dbName); + long totalSize = getDataSizeOfDatabase(db); + result.put(dbName, Long.valueOf(totalSize)); + } // end for dbs + return result; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java new file mode 100644 index 0000000000..b0a4f376cb --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.common.DdlException; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.httpv2.util.ExecutionResultSet; +import org.apache.doris.httpv2.util.StatementSubmitter; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.SystemInfoService; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.google.common.base.Strings; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.lang.reflect.Type; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +/** + * For execute stmt via http + */ +@RestController +public class StmtExecutionAction extends RestBaseController { + private static final Logger LOG = LogManager.getLogger(StmtExecutionAction.class); + private static StatementSubmitter stmtSubmitter = new StatementSubmitter(); + + private static final String PARAM_SYNC = "sync"; + private static final String PARAM_LIMIT = "limit"; + + private static final long DEFAULT_ROW_LIMIT = 1000; + private static final long MAX_ROW_LIMIT = 10000; + + /** + * Execute a SQL. + * Request body: + * { + * "stmt" : "select * from tbl1" + * } + */ + @RequestMapping(path = "/api/query/{" + NS_KEY + "}/{" + DB_KEY + "}", method = {RequestMethod.POST}) + public Object exeuteSQL( + @PathVariable(value = NS_KEY) String ns, + @PathVariable(value = DB_KEY) String dbName, + HttpServletRequest request, HttpServletResponse response, + @RequestBody String stmtBody) throws DdlException { + ActionAuthorizationInfo authInfo = checkWithCookie(request, response, false); + + if (!ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + return ResponseEntityBuilder.badRequest("Only support 'default_cluster' now"); + } + + boolean isSync = true; + String syncParam = request.getParameter(PARAM_SYNC); + if (!Strings.isNullOrEmpty(syncParam)) { + isSync = syncParam.equals("1"); + } + + String limitParam = request.getParameter(PARAM_LIMIT); + long limit = DEFAULT_ROW_LIMIT; + if (!Strings.isNullOrEmpty(limitParam)) { + limit = Math.min(Long.valueOf(limitParam), MAX_ROW_LIMIT); + } + + Type type = new TypeToken() { + }.getType(); + StmtRequestBody stmtRequestBody = new Gson().fromJson(stmtBody, type); + + if (Strings.isNullOrEmpty(stmtRequestBody.stmt)) { + return ResponseEntityBuilder.badRequest("Missing statement request body"); + } + LOG.info("stmt: {}", stmtRequestBody.stmt); + + ConnectContext.get().setDatabase(getFullDbName(dbName)); + + // 2. Submit stmt + StatementSubmitter.StmtContext stmtCtx = new StatementSubmitter.StmtContext( + stmtRequestBody.stmt, authInfo.fullUserName, authInfo.password, limit + ); + Future future = stmtSubmitter.submit(stmtCtx); + + if (isSync) { + try { + ExecutionResultSet resultSet = future.get(); + return ResponseEntityBuilder.ok(resultSet.getResult()); + } catch (InterruptedException e) { + LOG.warn("failed to execute stmt", e); + return ResponseEntityBuilder.okWithCommonError("Failed to execute sql: " + e.getMessage()); + } catch (ExecutionException e) { + LOG.warn("failed to execute stmt", e); + return ResponseEntityBuilder.okWithCommonError("Failed to execute sql: " + e.getMessage()); + } + } else { + return ResponseEntityBuilder.okWithCommonError("Not support async query execution"); + } + } + + private static class StmtRequestBody { + public String stmt; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StorageTypeCheckAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StorageTypeCheckAction.java new file mode 100644 index 0000000000..90c730f936 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StorageTypeCheckAction.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.MaterializedIndexMeta; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.Table.TableType; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TStorageType; + +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.google.common.base.Strings; +import com.google.common.collect.Maps; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.List; +import java.util.Map; + +@RestController +public class StorageTypeCheckAction extends RestBaseController { + + @RequestMapping(path = "/api/_check_storagetype", method = RequestMethod.GET) + protected Object check_storagetype(HttpServletRequest request, HttpServletResponse response) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + + String dbName = request.getParameter(DB_KEY); + if (Strings.isNullOrEmpty(dbName)) { + return ResponseEntityBuilder.badRequest("No database selected"); + } + + String fullDbName = getFullDbName(dbName); + Database db = Catalog.getCurrentCatalog().getDb(fullDbName); + if (db == null) { + return ResponseEntityBuilder.badRequest("Database " + dbName + " does not exist"); + } + + Map> result = Maps.newHashMap(); + db.readLock(); + try { + List
tbls = db.getTables(); + for (Table tbl : tbls) { + if (tbl.getType() != TableType.OLAP) { + continue; + } + + OlapTable olapTbl = (OlapTable) tbl; + Map indexMap = Maps.newHashMap(); + for (Map.Entry entry : olapTbl.getIndexIdToMeta().entrySet()) { + MaterializedIndexMeta indexMeta = entry.getValue(); + if (indexMeta.getStorageType() == TStorageType.ROW) { + indexMap.put(olapTbl.getIndexNameById(entry.getKey()), indexMeta.getStorageType().name()); + } + } + result.put(tbl.getName(), indexMap); + } + } finally { + db.readUnlock(); + } + return ResponseEntityBuilder.ok(result); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java new file mode 100644 index 0000000000..1c261a7ca6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java @@ -0,0 +1,292 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import io.netty.handler.codec.http.HttpResponseStatus; + +import org.apache.doris.analysis.InlineViewRef; +import org.apache.doris.analysis.SelectStmt; +import org.apache.doris.analysis.StatementBase; +import org.apache.doris.analysis.TableName; +import org.apache.doris.analysis.TableRef; +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Table; +import org.apache.doris.common.DorisHttpException; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.httpv2.util.HttpUtil; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.planner.PlanFragment; +import org.apache.doris.planner.Planner; +import org.apache.doris.planner.ScanNode; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.thrift.TDataSink; +import org.apache.doris.thrift.TDataSinkType; +import org.apache.doris.thrift.TMemoryScratchSink; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TPaloScanRange; +import org.apache.doris.thrift.TPlanFragment; +import org.apache.doris.thrift.TQueryOptions; +import org.apache.doris.thrift.TQueryPlanInfo; +import org.apache.doris.thrift.TScanRangeLocations; +import org.apache.doris.thrift.TTabletVersionInfo; +import org.apache.doris.thrift.TUniqueId; + +import com.google.common.base.Strings; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.thrift.TException; +import org.apache.thrift.TSerializer; +import org.json.JSONException; +import org.json.JSONObject; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * This class responsible for parse the sql and generate the query plan fragment for a (only one) table{@see OlapTable} + * the related tablet maybe pruned by query planer according the `where` predicate. + */ +@RestController +public class TableQueryPlanAction extends RestBaseController { + public static final Logger LOG = LogManager.getLogger(TableQueryPlanAction.class); + + @RequestMapping(path = "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_query_plan", method = {RequestMethod.GET, RequestMethod.POST}) + public Object query_plan( + @PathVariable(value = DB_KEY) final String dbName, + @PathVariable(value = TABLE_KEY) final String tblName, + HttpServletRequest request, HttpServletResponse response) { + executeCheckPassword(request, response); + // just allocate 2 slot for top holder map + Map resultMap = new HashMap<>(4); + + String postContent = HttpUtil.getBody(request); + try { + // may be these common validate logic should be moved to one base class + String sql; + if (Strings.isNullOrEmpty(postContent)) { + return ResponseEntityBuilder.badRequest("POST body must contains [sql] root object"); + } + JSONObject jsonObject; + try { + jsonObject = new JSONObject(postContent); + } catch (JSONException e) { + return ResponseEntityBuilder.badRequest("malformed json: " + e.getMessage()); + } + + sql = String.valueOf(jsonObject.opt("sql")); + if (Strings.isNullOrEmpty(sql)) { + return ResponseEntityBuilder.badRequest("POST body must contains [sql] root object"); + } + LOG.info("receive SQL statement [{}] from external service [ user [{}]] for database [{}] table [{}]", + sql, ConnectContext.get().getCurrentUserIdentity(), dbName, tblName); + + String fullDbName = getFullDbName(dbName); + // check privilege for select, otherwise return HTTP 401 + checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tblName, PrivPredicate.SELECT); + Database db = Catalog.getCurrentCatalog().getDb(fullDbName); + if (db == null) { + return ResponseEntityBuilder.okWithCommonError("Database [" + dbName + "] " + "does not exists"); + } + // may be should acquire writeLock + db.readLock(); + try { + Table table = db.getTable(tblName); + if (table == null) { + return ResponseEntityBuilder.okWithCommonError("Table [" + tblName + "] " + "does not exists"); + } + // just only support OlapTable, ignore others such as ESTable + if (table.getType() != Table.TableType.OLAP) { + return ResponseEntityBuilder.okWithCommonError("only support OlapTable currently, " + + "but Table [" + tblName + "] " + "is not a OlapTable"); + } + // parse/analysis/plan the sql and acquire tablet distributions + handleQuery(ConnectContext.get(), fullDbName, tblName, sql, resultMap); + } finally { + db.readUnlock(); + } + } catch (DorisHttpException e) { + // status code should conforms to HTTP semantic + resultMap.put("status", e.getCode().code()); + resultMap.put("exception", e.getMessage()); + } + return ResponseEntityBuilder.ok(resultMap); + } + + + /** + * process the sql syntax and return the resolved pruned tablet + * + * @param context context for analyzer + * @param sql the single table select statement + * @param result the acquired results + * @return + * @throws DorisHttpException + */ + private void handleQuery(ConnectContext context, String requestDb, String requestTable, String sql, + Map result) throws DorisHttpException { + // use SE to resolve sql + StmtExecutor stmtExecutor = new StmtExecutor(context, new OriginStatement(sql, 0), false); + try { + TQueryOptions tQueryOptions = context.getSessionVariable().toThrift(); + // Conduct Planner create SingleNodePlan#createPlanFragments + tQueryOptions.num_nodes = 1; + // analyze sql + stmtExecutor.analyze(tQueryOptions); + } catch (Exception e) { + throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, e.getMessage()); + } + // the parsed logical statement + StatementBase query = stmtExecutor.getParsedStmt(); + // only process select semantic + if (!(query instanceof SelectStmt)) { + throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "Select statement needed, but found [" + sql + " ]"); + } + SelectStmt stmt = (SelectStmt) query; + // just only process sql like `select * from table where `, only support executing scan semantic + if (stmt.hasAggInfo() || stmt.hasAnalyticInfo() + || stmt.hasOrderByClause() || stmt.hasOffset() || stmt.hasLimit() || stmt.isExplain()) { + throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "only support single table filter-prune-scan, but found [ " + sql + "]"); + } + // process only one table by one http query + List fromTables = stmt.getTableRefs(); + if (fromTables.size() != 1) { + throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "Select statement must hava only one table"); + } + + TableRef fromTable = fromTables.get(0); + if (fromTable instanceof InlineViewRef) { + throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "Select statement must not embed another statement"); + } + // check consistent http requested resource with sql referenced + // if consistent in this way, can avoid check privilege + TableName tableAndDb = fromTables.get(0).getName(); + if (!(tableAndDb.getDb().equals(requestDb) && tableAndDb.getTbl().equals(requestTable))) { + throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "requested database and table must consistent with sql: request [ " + + requestDb + "." + requestTable + "]" + "and sql [" + tableAndDb.toString() + "]"); + } + + // acquired Planner to get PlanNode and fragment templates + Planner planner = stmtExecutor.planner(); + // acquire ScanNode to obtain pruned tablet + // in this way, just retrieve only one scannode + List scanNodes = planner.getScanNodes(); + if (scanNodes.size() != 1) { + throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Planner should plan just only one ScanNode but found [ " + scanNodes.size() + "]"); + } + List scanRangeLocations = scanNodes.get(0).getScanRangeLocations(0); + // acquire the PlanFragment which the executable template + List fragments = planner.getFragments(); + if (fragments.size() != 1) { + throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Planner should plan just only one PlanFragment but found [ " + fragments.size() + "]"); + } + + TQueryPlanInfo tQueryPlanInfo = new TQueryPlanInfo(); + + + // acquire TPlanFragment + TPlanFragment tPlanFragment = fragments.get(0).toThrift(); + // set up TMemoryScratchSink + TDataSink tDataSink = new TDataSink(); + tDataSink.type = TDataSinkType.MEMORY_SCRATCH_SINK; + tDataSink.memory_scratch_sink = new TMemoryScratchSink(); + tPlanFragment.output_sink = tDataSink; + + tQueryPlanInfo.plan_fragment = tPlanFragment; + tQueryPlanInfo.desc_tbl = query.getAnalyzer().getDescTbl().toThrift(); + // set query_id + UUID uuid = UUID.randomUUID(); + tQueryPlanInfo.query_id = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()); + + Map tablet_info = new HashMap<>(); + // acquire resolved tablet distribution + Map tabletRoutings = assemblePrunedPartitions(scanRangeLocations); + tabletRoutings.forEach((tabletId, node) -> { + long tablet = Long.parseLong(tabletId); + tablet_info.put(tablet, new TTabletVersionInfo(tablet, node.version, node.versionHash, node.schemaHash)); + }); + tQueryPlanInfo.tablet_info = tablet_info; + + // serialize TQueryPlanInfo and encode plan with Base64 to string in order to translate by json format + TSerializer serializer = new TSerializer(); + String opaqued_query_plan; + try { + byte[] query_plan_stream = serializer.serialize(tQueryPlanInfo); + opaqued_query_plan = Base64.getEncoder().encodeToString(query_plan_stream); + } catch (TException e) { + throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "TSerializer failed to serialize PlanFragment, reason [ " + e.getMessage() + " ]"); + } + result.put("partitions", tabletRoutings); + result.put("opaqued_query_plan", opaqued_query_plan); + result.put("status", 200); + } + + /** + * acquire all involved (already pruned) tablet routing + * + * @param scanRangeLocationsList + * @return + */ + private Map assemblePrunedPartitions(List scanRangeLocationsList) { + Map result = new HashMap<>(); + for (TScanRangeLocations scanRangeLocations : scanRangeLocationsList) { + // only process palo(doris) scan range + TPaloScanRange scanRange = scanRangeLocations.scan_range.palo_scan_range; + Node tabletRouting = new Node(Long.parseLong(scanRange.version), + Long.parseLong(scanRange.version_hash), Integer.parseInt(scanRange.schema_hash)); + for (TNetworkAddress address : scanRange.hosts) { + tabletRouting.addRouting(address.hostname + ":" + address.port); + } + result.put(String.valueOf(scanRange.tablet_id), tabletRouting); + } + return result; + } + + // helper class for json transformation + final class Node { + // ["host1:port1", "host2:port2", "host3:port3"] + public List routings = new ArrayList<>(); + public long version; + public long versionHash; + public int schemaHash; + + public Node(long version, long versionHash, int schemaHash) { + this.version = version; + this.versionHash = versionHash; + this.schemaHash = schemaHash; + } + + private void addRouting(String routing) { + routings.add(routing); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableRowCountAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableRowCountAction.java new file mode 100644 index 0000000000..429d1d0a8c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableRowCountAction.java @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.common.DorisHttpException; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; + +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.HashMap; +import java.util.Map; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * This class is responsible for fetch the approximate row count of the specified table from cluster-meta data, + * the approximate row maybe used for some computing system to decide use which compute-algorithm can be used + * such as shuffle join or broadcast join. + *

+ * This API is not intended to compute the exact row count of the specified table, if you need the exact row count, + * please consider using the sql syntax `select count(*) from {table}` + */ +@RestController +public class TableRowCountAction extends RestBaseController { + + @RequestMapping(path = "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_count", method = RequestMethod.GET) + public Object count( + @PathVariable(value = DB_KEY) final String dbName, + @PathVariable(value = TABLE_KEY) final String tblName, + HttpServletRequest request, HttpServletResponse response) { + executeCheckPassword(request, response); + // just allocate 2 slot for top holder map + Map resultMap = new HashMap<>(4); + try { + String fullDbName = getFullDbName(dbName); + // check privilege for select, otherwise return HTTP 401 + checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tblName, PrivPredicate.SELECT); + Database db = Catalog.getCurrentCatalog().getDb(fullDbName); + if (db == null) { + return ResponseEntityBuilder.okWithCommonError("Database [" + dbName + "] " + "does not exists"); + } + db.writeLock(); + try { + Table table = db.getTable(tblName); + if (table == null) { + return ResponseEntityBuilder.okWithCommonError("Table [" + tblName + "] " + "does not exists"); + } + // just only support OlapTable, ignore others such as ESTable + if (!(table instanceof OlapTable)) { + return ResponseEntityBuilder.okWithCommonError("Table [" + tblName + "] " + + "is not a OlapTable, only support OlapTable currently"); + } + OlapTable olapTable = (OlapTable) table; + resultMap.put("status", 200); + resultMap.put("size", olapTable.proximateRowCount()); + } finally { + db.writeUnlock(); + } + } catch (DorisHttpException e) { + // status code should conforms to HTTP semantic + resultMap.put("status", e.getCode().code()); + resultMap.put("exception", e.getMessage()); + } + + return ResponseEntityBuilder.ok(resultMap); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java new file mode 100644 index 0000000000..840770b942 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.DorisHttpException; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; + +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Get table schema for specified cluster.database.table with privilege checking + */ +@RestController +public class TableSchemaAction extends RestBaseController { + + @RequestMapping(path = "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_schema", method = RequestMethod.GET) + protected Object schema( + @PathVariable(value = DB_KEY) final String dbName, + @PathVariable(value = TABLE_KEY) final String tblName, + HttpServletRequest request, HttpServletResponse response) { + executeCheckPassword(request, response); + // just allocate 2 slot for top holder map + Map resultMap = new HashMap<>(2); + + try { + String fullDbName = getFullDbName(dbName); + // check privilege for select, otherwise return 401 HTTP status + checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tblName, PrivPredicate.SELECT); + Database db = Catalog.getCurrentCatalog().getDb(fullDbName); + if (db == null) { + return ResponseEntityBuilder.okWithCommonError("Database [" + dbName + "] " + "does not exists"); + } + db.readLock(); + try { + Table table = db.getTable(tblName); + if (table == null) { + return ResponseEntityBuilder.okWithCommonError("Table [" + tblName + "] " + "does not exists"); + } + // just only support OlapTable, ignore others such as ESTable + if (!(table instanceof OlapTable)) { + return ResponseEntityBuilder.okWithCommonError("Table [" + tblName + "] " + + "is not a OlapTable, only support OlapTable currently"); + } + try { + List columns = table.getBaseSchema(); + List> propList = new ArrayList(columns.size()); + for (Column column : columns) { + Map baseInfo = new HashMap<>(2); + Type colType = column.getOriginType(); + PrimitiveType primitiveType = colType.getPrimitiveType(); + if (primitiveType == PrimitiveType.DECIMALV2 || primitiveType == PrimitiveType.DECIMAL) { + ScalarType scalarType = (ScalarType) colType; + baseInfo.put("precision", scalarType.getPrecision() + ""); + baseInfo.put("scale", scalarType.getScalarScale() + ""); + } + baseInfo.put("type", primitiveType.toString()); + baseInfo.put("comment", column.getComment()); + baseInfo.put("name", column.getDisplayName()); + propList.add(baseInfo); + } + resultMap.put("status", 200); + resultMap.put("properties", propList); + } catch (Exception e) { + // Transform the general Exception to custom DorisHttpException + return ResponseEntityBuilder.okWithCommonError(e.getMessage()); + } + } finally { + db.readUnlock(); + } + } catch (DorisHttpException e) { + // status code should conforms to HTTP semantic + resultMap.put("status", e.getCode().code()); + resultMap.put("exception", e.getMessage()); + } + + return ResponseEntityBuilder.ok(resultMap); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/UploadAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/UploadAction.java new file mode 100644 index 0000000000..d8373989bf --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/UploadAction.java @@ -0,0 +1,306 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.common.Config; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.httpv2.util.LoadSubmitter; +import org.apache.doris.httpv2.util.TmpFileMgr; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.SystemInfoService; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +/** + * Upload file + */ +@RestController +public class UploadAction extends RestBaseController { + private static final Logger LOG = LogManager.getLogger(UploadAction.class); + private static TmpFileMgr fileMgr = new TmpFileMgr(Config.tmp_dir); + private static LoadSubmitter loadSubmitter = new LoadSubmitter(); + + private static final String PARAM_COLUMN_SEPARATOR = "column_separator"; + private static final String PARAM_PREVIEW = "preview"; + private static final String PARAM_FILE_ID = "file_id"; + private static final String PARAM_FILE_UUID = "file_uuid"; + + /** + * Upload the file + * @param ns + * @param dbName + * @param tblName + * @param file + * @param request + * @param response + * @return + */ + @RequestMapping(path = "/api/{" + NS_KEY + "}/{" + DB_KEY + "}/{" + TABLE_KEY + "}/upload", method = {RequestMethod.POST}) + public Object upload( + @PathVariable(value = NS_KEY) String ns, + @PathVariable(value = DB_KEY) String dbName, + @PathVariable(value = TABLE_KEY) String tblName, + @RequestParam("file") MultipartFile file, + HttpServletRequest request, HttpServletResponse response) { + + checkWithCookie(request, response, false); + + if (!ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + return ResponseEntityBuilder.badRequest("Only support 'default_cluster' now"); + } + + String fullDbName = getFullDbName(dbName); + checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tblName, PrivPredicate.LOAD); + + String columnSeparator = request.getParameter(PARAM_COLUMN_SEPARATOR); + if (Strings.isNullOrEmpty(columnSeparator)) { + columnSeparator = "\t"; + } + + String preview = request.getParameter(PARAM_PREVIEW); + if (Strings.isNullOrEmpty(preview)) { + preview = "false"; // default is false + } + + if (file.isEmpty()) { + return ResponseEntityBuilder.badRequest("Empty file"); + } + + try { + TmpFileMgr.TmpFile tmpFile = fileMgr.upload(new TmpFileMgr.UploadFile(file, columnSeparator)); + TmpFileMgr.TmpFile copiedFile = tmpFile.copy(); + if (preview.equalsIgnoreCase("true")) { + copiedFile.setPreview(); + } + return ResponseEntityBuilder.ok(copiedFile); + } catch (TmpFileMgr.TmpFileException | IOException e) { + return ResponseEntityBuilder.okWithCommonError(e.getMessage()); + } + } + + /** + * Load the uploaded file + * @param ns + * @param dbName + * @param tblName + * @param request + * @param response + * @return + */ + @RequestMapping(path = "/api/{" + NS_KEY + "}/{" + DB_KEY + "}/{" + TABLE_KEY + "}/upload", method = {RequestMethod.PUT}) + public Object submit( + @PathVariable(value = NS_KEY) String ns, + @PathVariable(value = DB_KEY) String dbName, + @PathVariable(value = TABLE_KEY) String tblName, + HttpServletRequest request, HttpServletResponse response) { + + ActionAuthorizationInfo authInfo = checkWithCookie(request, response, false); + + if (!ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + return ResponseEntityBuilder.badRequest("Only support 'default_cluster' now"); + } + + String fullDbName = getFullDbName(dbName); + checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tblName, PrivPredicate.LOAD); + + String fileIdStr = request.getParameter(PARAM_FILE_ID); + if (Strings.isNullOrEmpty(fileIdStr)) { + return ResponseEntityBuilder.badRequest("Missing file id parameter"); + } + String fileUUIDStr = request.getParameter(PARAM_FILE_UUID); + if (Strings.isNullOrEmpty(fileUUIDStr)) { + return ResponseEntityBuilder.badRequest("Missing file id parameter"); + } + + TmpFileMgr.TmpFile tmpFile = null; + try { + tmpFile = fileMgr.getFile(Long.valueOf(fileIdStr), fileUUIDStr); + } catch (TmpFileMgr.TmpFileException e) { + return ResponseEntityBuilder.okWithCommonError("file not found"); + } + Preconditions.checkNotNull(tmpFile, fileIdStr); + + LoadContext loadContext = new LoadContext(request, dbName, tblName, authInfo.fullUserName, authInfo.password, tmpFile); + Future future = loadSubmitter.submit(loadContext); + + try { + LoadSubmitter.SubmitResult res = future.get(); + return ResponseEntityBuilder.ok(res); + } catch (InterruptedException | ExecutionException e) { + return ResponseEntityBuilder.okWithCommonError(e.getMessage()); + } + } + + /** + * Get all uploaded file or specified file + * If preview is true, also return the the preview of the file + * @param ns + * @param dbName + * @param tblName + * @param request + * @param response + * @return + */ + @RequestMapping(path = "/api/{" + NS_KEY + "}/{" + DB_KEY + "}/{" + TABLE_KEY + "}/upload", method = {RequestMethod.GET}) + public Object list( + @PathVariable(value = NS_KEY) String ns, + @PathVariable(value = DB_KEY) String dbName, + @PathVariable(value = TABLE_KEY) String tblName, + HttpServletRequest request, HttpServletResponse response) { + + checkWithCookie(request, response, false); + + if (!ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + return ResponseEntityBuilder.badRequest("Only support 'default_cluster' now"); + } + + String fullDbName = getFullDbName(dbName); + checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tblName, PrivPredicate.LOAD); + + String fileIdStr = request.getParameter(PARAM_FILE_ID); + String fileUUIDStr = request.getParameter(PARAM_FILE_UUID); + + if (Strings.isNullOrEmpty(fileIdStr) || Strings.isNullOrEmpty(fileUUIDStr)) { + // not specified file id, return all files list + List files = fileMgr.listFiles(); + return ResponseEntityBuilder.ok(files); + } + + // return specified file + String preview = request.getParameter(PARAM_PREVIEW); + if (Strings.isNullOrEmpty(preview)) { + preview = "true"; // default is true + } + + try { + TmpFileMgr.TmpFile tmpFile = fileMgr.getFile(Long.valueOf(fileIdStr), fileUUIDStr); + TmpFileMgr.TmpFile copiedFile = tmpFile.copy(); + if (preview.equalsIgnoreCase("true")) { + copiedFile.setPreview(); + } + return ResponseEntityBuilder.ok(copiedFile); + } catch (TmpFileMgr.TmpFileException | IOException e) { + return ResponseEntityBuilder.okWithCommonError(e.getMessage()); + } + } + + @RequestMapping(path = "/api/{" + NS_KEY + "}/{" + DB_KEY + "}/{" + TABLE_KEY + "}/upload", method = {RequestMethod.DELETE}) + public Object delete( + @PathVariable(value = NS_KEY) String ns, + @PathVariable(value = DB_KEY) String dbName, + @PathVariable(value = TABLE_KEY) String tblName, + HttpServletRequest request, HttpServletResponse response) { + + checkWithCookie(request, response, false); + + if (!ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + return ResponseEntityBuilder.badRequest("Only support 'default_cluster' now"); + } + + String fullDbName = getFullDbName(dbName); + checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tblName, PrivPredicate.LOAD); + + String fileIdStr = request.getParameter(PARAM_FILE_ID); + if (Strings.isNullOrEmpty(fileIdStr)) { + return ResponseEntityBuilder.badRequest("Missing file id parameter"); + } + String fileUUIDStr = request.getParameter(PARAM_FILE_UUID); + if (Strings.isNullOrEmpty(fileUUIDStr)) { + return ResponseEntityBuilder.badRequest("Missing file id parameter"); + } + + fileMgr.deleteFile(Long.valueOf(fileIdStr), fileUUIDStr); + return ResponseEntityBuilder.ok(); + } + + /** + * A context to save infos of stream load + */ + public static class LoadContext { + public String user; + public String passwd; + public String db; + public String tbl; + public TmpFileMgr.TmpFile file; + + public String label; + public String columnSeparator; + public String columns; + public String where; + public String maxFilterRatio; + public String partitions; + public String timeout; + public String strictMode; + public String timezone; + public String execMemLimit; + public String format; + public String jsonPaths; + public String stripOuterArray; + public String jsonRoot; + + + public LoadContext(HttpServletRequest request, String db, String tbl, String user, String passwd, TmpFileMgr.TmpFile file) { + this.db = db; + this.tbl = tbl; + this.user = user; + this.passwd = passwd; + this.file = file; + + parseHeader(request); + } + + private void parseHeader(HttpServletRequest request) { + this.label = request.getHeader("label"); + this.columnSeparator = file.columnSeparator; + if (!Strings.isNullOrEmpty(request.getHeader("column_separator"))) { + this.columnSeparator = request.getHeader("column_separator"); + } + this.columns = request.getHeader("columns"); + this.where = request.getHeader("where"); + this.maxFilterRatio = request.getHeader("max_filter_ratio"); + this.partitions = request.getHeader("partitions"); + this.timeout = request.getHeader("timeout"); + this.strictMode = request.getHeader("strict_mode"); + this.timezone = request.getHeader("timezone"); + this.execMemLimit = request.getHeader("exec_mem_limit"); + this.format = request.getHeader("format"); + this.jsonPaths = request.getHeader("jsonpaths"); + this.stripOuterArray = request.getHeader("strip_outer_array"); + this.jsonRoot = request.getHeader("json_root"); + } + } +} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/ExecutionResultSet.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/ExecutionResultSet.java new file mode 100644 index 0000000000..271b11ba05 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/ExecutionResultSet.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.util; + + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.util.Map; + +public class ExecutionResultSet { + + private Map result; + + public ExecutionResultSet(Map result) { + this.result = result; + } + + public void setResult(Map result) { + this.result = result; + } + + public Map getResult() { + return result; + } + + public static ExecutionResultSet emptyResult() { + Map result = Maps.newHashMap(); + result.put("meta", Lists.newArrayList()); + result.put("data", Lists.newArrayList()); + return new ExecutionResultSet(result); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/HttpUtil.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/HttpUtil.java new file mode 100644 index 0000000000..a2554b51be --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/HttpUtil.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.util; + +import com.google.common.base.Strings; + +import java.io.BufferedReader; +import java.io.IOException; + +import javax.servlet.http.HttpServletRequest; + +import static org.springframework.http.HttpHeaders.CONNECTION; + +public class HttpUtil { + public static boolean isKeepAlive(HttpServletRequest request) { + if (!request.getHeader(CONNECTION).equals("close") && + (request.getProtocol().equals("") || + request.getHeader(CONNECTION).equals("keep-alive"))) { + return true; + } + return false; + } + + public static boolean isSslEnable(HttpServletRequest request) { + String url = request.getRequestURL().toString(); + if (!Strings.isNullOrEmpty(url) && url.startsWith("https")) { + return true; + } + return false; + + } + + public static String getBody(HttpServletRequest request) { + StringBuffer data = new StringBuffer(); + String line = null; + BufferedReader reader = null; + try { + reader = request.getReader(); + while (null != (line = reader.readLine())) + data.append(new String(line.getBytes("utf-8"))); + } catch (IOException e) { + } finally { + } + return data.toString(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/LoadSubmitter.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/LoadSubmitter.java new file mode 100644 index 0000000000..9bdaaf56c7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/LoadSubmitter.java @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.util; + +import org.apache.doris.cluster.ClusterNamespace; +import org.apache.doris.common.Config; +import org.apache.doris.common.ThreadPoolManager; +import org.apache.doris.httpv2.rest.UploadAction; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.common.base.Strings; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Type; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.concurrent.Callable; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; + +public class LoadSubmitter { + private static final Logger LOG = LogManager.getLogger(LoadSubmitter.class); + + private ThreadPoolExecutor executor = ThreadPoolManager.newDaemonCacheThreadPool(2, "Load submitter", true); + + private static final String STREAM_LOAD_URL_PATTERN = "http://%s:%d/api/%s/%s/_stream_load"; + + public Future submit(UploadAction.LoadContext loadContext) { + LoadSubmitter.Worker worker = new LoadSubmitter.Worker(loadContext); + return executor.submit(worker); + } + + private static class Worker implements Callable { + + private UploadAction.LoadContext loadContext; + + public Worker(UploadAction.LoadContext loadContext) { + this.loadContext = loadContext; + } + + @Override + public SubmitResult call() throws Exception { + String auth = String.format("%s:%s", ClusterNamespace.getNameFromFullName(loadContext.user), loadContext.passwd); + String authEncoding = Base64.getEncoder().encodeToString(auth.getBytes(StandardCharsets.UTF_8)); + + String loadUrlStr = String.format(STREAM_LOAD_URL_PATTERN, "127.0.0.1", Config.http_port, loadContext.db, loadContext.tbl); + URL loadUrl = new URL(loadUrlStr); + HttpURLConnection conn = (HttpURLConnection) loadUrl.openConnection(); + conn.setRequestMethod("PUT"); + conn.setRequestProperty("Authorization", "Basic " + authEncoding); + conn.addRequestProperty("Expect", "100-continue"); + conn.addRequestProperty("Content-Type", "text/plain; charset=UTF-8"); + if (!Strings.isNullOrEmpty(loadContext.columns)) { + conn.addRequestProperty("columns", loadContext.columns); + } + if (!Strings.isNullOrEmpty(loadContext.columnSeparator)) { + conn.addRequestProperty("column_separator", loadContext.columnSeparator); + } + if (!Strings.isNullOrEmpty(loadContext.label)) { + conn.addRequestProperty("label", loadContext.label); + } + conn.setDoOutput(true); + conn.setDoInput(true); + + File loadFile = checkAndGetFile(loadContext.file); + try(BufferedOutputStream bos = new BufferedOutputStream(conn.getOutputStream()); + BufferedInputStream bis = new BufferedInputStream(new FileInputStream(loadFile));) { + int i; + while ((i = bis.read()) > 0) { + bos.write(i); + } + } + + int status = conn.getResponseCode(); + String respMsg = conn.getResponseMessage(); + + LOG.info("get status: {}, response msg: {}", status, respMsg); + + InputStream stream = (InputStream) conn.getContent(); + BufferedReader br = new BufferedReader(new InputStreamReader(stream)); + StringBuilder sb = new StringBuilder(); + String line; + while ((line = br.readLine()) != null) { + sb.append(line); + } + Type type = new TypeToken() { + }.getType(); + SubmitResult result = new Gson().fromJson(sb.toString(), type); + return result; + } + + private File checkAndGetFile(TmpFileMgr.TmpFile tmpFile) { + File file = new File(tmpFile.absPath); + return file; + } + } + + public static class SubmitResult { + public String TxnId; + public String Label; + public String Status; + public String Message; + public String NumberTotalRows; + public String NumberLoadedRows; + public String NumberFilteredRows; + public String NumberUnselectedRows; + public String LoadBytes; + public String LoadTimeMs; + public String BeginTxnTimeMs; + public String StreamLoadPutTimeMs; + public String ReadDataTimeMs; + public String WriteDataTimeMs; + public String CommitAndPublishTimeMs; + public String ErrorURL; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java new file mode 100644 index 0000000000..65f888656e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java @@ -0,0 +1,217 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.util; + + +import org.apache.doris.analysis.DdlStmt; +import org.apache.doris.analysis.ExportStmt; +import org.apache.doris.analysis.InsertStmt; +import org.apache.doris.analysis.QueryStmt; +import org.apache.doris.analysis.ShowStmt; +import org.apache.doris.analysis.SqlParser; +import org.apache.doris.analysis.SqlScanner; +import org.apache.doris.analysis.StatementBase; +import org.apache.doris.common.Config; +import org.apache.doris.common.ThreadPoolManager; +import org.apache.doris.common.util.SqlParserUtils; +import org.apache.doris.qe.ConnectContext; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.io.StringReader; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; + +/** + * This is a simple stmt submitter for submitting a statement to the local FE. + * It uses a fixed-size thread pool to receive query requests, + * so it is only suitable for a small number of low-frequency request scenarios. + * Now it support submitting the following type of stmt: + * QueryStmt + * ShowStmt + * InsertStmt + * DdlStmt + * ExportStmt + */ +public class StatementSubmitter { + private static final Logger LOG = LogManager.getLogger(StatementSubmitter.class); + + private static final String TYPE_RESULT_SET = "result_set"; + private static final String TYPE_EXEC_STATUS = "exec_status"; + + private static final String JDBC_DRIVER = "com.mysql.jdbc.Driver"; + private static final String DB_URL_PATTERN = "jdbc:mysql://127.0.0.1:%d/%s"; + + private ThreadPoolExecutor executor = ThreadPoolManager.newDaemonCacheThreadPool(2, "SQL submitter", true); + + public Future submit(StmtContext queryCtx) { + Worker worker = new Worker(ConnectContext.get(), queryCtx); + return executor.submit(worker); + } + + private static class Worker implements Callable { + + private ConnectContext ctx; + private StmtContext queryCtx; + + public Worker(ConnectContext ctx, StmtContext queryCtx) { + this.ctx = ctx; + this.queryCtx = queryCtx; + } + + @Override + public ExecutionResultSet call() throws Exception { + StatementBase stmtBase = analyzeStmt(queryCtx.stmt); + + Connection conn = null; + Statement stmt = null; + String dbUrl = String.format(DB_URL_PATTERN, Config.query_port, ctx.getDatabase()); + try { + Class.forName(JDBC_DRIVER); + conn = DriverManager.getConnection(dbUrl, queryCtx.user, queryCtx.passwd); + + if (stmtBase instanceof QueryStmt || stmtBase instanceof ShowStmt) { + stmt = conn.prepareStatement(queryCtx.stmt, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + ResultSet rs = stmt.executeQuery(queryCtx.stmt); + ExecutionResultSet resultSet = generateResultSet(rs); + rs.close(); + return resultSet; + } else if (stmtBase instanceof InsertStmt || stmtBase instanceof DdlStmt || stmtBase instanceof ExportStmt) { + stmt = conn.createStatement(); + stmt.execute(queryCtx.stmt); + ExecutionResultSet resultSet = generateExecStatus(); + return resultSet; + } else { + throw new Exception("Unsupported statement type"); + } + } finally { + try { + if (stmt != null) { + stmt.close(); + } + } catch (SQLException se2) { + LOG.warn("failed to close stmt", se2); + } + try { + if (conn != null) conn.close(); + } catch (SQLException se) { + LOG.warn("failed to close connection", se); + } + } + } + + /** + * Result json sample: + * { + * "type": "result_set", + * "data": [ + * [1], + * [2] + * ], + * "meta": [{ + * "name": "k1", + * "type": "INT" + * }], + * "status": {} + * } + */ + private ExecutionResultSet generateResultSet(ResultSet rs) throws SQLException { + Map result = Maps.newHashMap(); + result.put("type", TYPE_RESULT_SET); + if (rs == null) { + return new ExecutionResultSet(result); + } + ResultSetMetaData metaData = rs.getMetaData(); + int colNum = metaData.getColumnCount(); + // 1. metadata + List> metaFields = Lists.newArrayList(); + // index start from 1 + for (int i = 1; i <= colNum; ++i) { + Map field = Maps.newHashMap(); + field.put("name", metaData.getColumnName(i)); + field.put("type", metaData.getColumnTypeName(i)); + metaFields.add(field); + } + // 2. data + List> rows = Lists.newArrayList(); + long rowCount = 0; + while (rs.next() && rowCount < queryCtx.limit) { + List row = Lists.newArrayListWithCapacity(colNum); + // index start from 1 + for (int i = 1; i <= colNum; ++i) { + row.add(rs.getObject(i)); + } + rows.add(row); + rowCount++; + } + result.put("meta", metaFields); + result.put("data", rows); + return new ExecutionResultSet(result); + } + + /** + * Result json sample: + * { + * "type": "exec_status", + * "status": {} + * } + */ + private ExecutionResultSet generateExecStatus() throws SQLException { + Map result = Maps.newHashMap(); + result.put("type", TYPE_EXEC_STATUS); + result.put("status", Maps.newHashMap()); + return new ExecutionResultSet(result); + } + + private StatementBase analyzeStmt(String stmtStr) throws Exception { + SqlParser parser = new SqlParser(new SqlScanner(new StringReader(stmtStr))); + try { + return SqlParserUtils.getFirstStmt(parser); + } catch (Exception e) { + throw new Exception("error happens when parsing stmt: " + e.getMessage()); + } + } + } + + public static class StmtContext { + public String stmt; + public String user; + public String passwd; + public long limit; // limit the number of rows returned by the stmt + + public StmtContext(String stmt, String user, String passwd, long limit) { + this.stmt = stmt; + this.user = user; + this.passwd = passwd; + this.limit = limit; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/TmpFileMgr.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/TmpFileMgr.java new file mode 100644 index 0000000000..0d1cc79915 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/TmpFileMgr.java @@ -0,0 +1,306 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.util; + +import org.apache.doris.common.util.Util; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.web.multipart.MultipartFile; + +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; + +/** + * Manager the file uploaded. + * This file manager is currently only used to manage files + * uploaded through the Upload RESTFul API. + * And limit the number and size of the maximum upload file. + * It can also browse or delete files through the RESTFul API. + */ +public class TmpFileMgr { + public static final Logger LOG = LogManager.getLogger(TmpFileMgr.class); + + private static final long MAX_TOTAL_FILE_SIZE_BYTES = 1 * 1024 * 1024 * 1024L; // 1GB + private static final long MAX_TOTAL_FILE_NUM = 100; + public static final long MAX_SINGLE_FILE_SIZE = 100 * 1024 * 1024L; // 100MB + private static final String UPLOAD_DIR = "_doris_upload"; + + private AtomicLong fileIdGenerator = new AtomicLong(0); + private String rootDir; + private Map fileMap = Maps.newConcurrentMap(); + + private long totalFileSize = 0; + + public TmpFileMgr(String dir) { + this.rootDir = dir + "/" + UPLOAD_DIR; + init(); + } + + private void init() { + File root = new File(rootDir); + if (!root.exists()) { + root.mkdirs(); + } else if (!root.isDirectory()) { + throw new IllegalStateException("Path " + rootDir + " is not directory"); + } + + // delete all files under this dir at startup. + // This means that all uploaded files will be lost after FE restarts. + // This is just for simplicity. + Util.deleteDirectory(root); + root.mkdirs(); + } + + /** + * Simply used `synchronized` to allow only one user upload file at one time. + * So that we can easily control the number of files and total size of files. + * + * @param uploadFile + * @return + * @throws TmpFileException + */ + public synchronized TmpFile upload(UploadFile uploadFile) throws TmpFileException { + if (uploadFile.file.getSize() > MAX_SINGLE_FILE_SIZE) { + throw new TmpFileException("File size " + uploadFile.file.getSize() + " exceed limit " + MAX_SINGLE_FILE_SIZE); + } + + if (totalFileSize + uploadFile.file.getSize() > MAX_TOTAL_FILE_SIZE_BYTES) { + throw new TmpFileException("Total file size will exceed limit " + MAX_TOTAL_FILE_SIZE_BYTES); + } + + if(fileMap.size() > MAX_TOTAL_FILE_NUM) { + throw new TmpFileException("Number of temp file " + fileMap.size() + " exceed limit " + MAX_TOTAL_FILE_NUM); + } + + long fileId = fileIdGenerator.incrementAndGet(); + String fileUUID = UUID.randomUUID().toString(); + + TmpFile tmpFile = new TmpFile(fileId, fileUUID, uploadFile.file.getOriginalFilename(), + uploadFile.file.getSize(), uploadFile.columnSeparator); + try { + tmpFile.save(uploadFile.file); + } catch (IOException e) { + throw new TmpFileException("Failed to upload file. Reason: " + e.getMessage()); + } + fileMap.put(tmpFile.id, tmpFile); + totalFileSize += uploadFile.file.getSize(); + return tmpFile; + } + + public TmpFile getFile(long id, String uuid) throws TmpFileException { + TmpFile tmpFile = fileMap.get(id); + if (tmpFile == null || !tmpFile.uuid.equals(uuid)) { + throw new TmpFileException("File with [" + id + "-" + uuid + "] does not exist"); + } + return tmpFile; + } + + public List listFiles() { + return fileMap.values().stream().map(t -> new TmpFileBrief(t)).collect(Collectors.toList()); + } + + /** + * Delete the specified file and remove it from fileMap + * @param fileId + * @param fileUUID + */ + public void deleteFile(Long fileId, String fileUUID) { + Iterator> iterator = fileMap.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + if (entry.getValue().id == fileId && entry.getValue().uuid.equals(fileUUID)) { + entry.getValue().delete(); + iterator.remove(); + } + } + return; + } + + public class TmpFile { + public final long id; + public final String uuid; + public final String originFileName; + public final long fileSize; + public String columnSeparator; + public String absPath; + + public List> lines = null; + public int maxColNum = 0; + + private static final int MAX_PREVIEW_LINES = 10; + + public TmpFile(long id, String uuid, String originFileName, long fileSize, String columnSeparator) { + this.id = id; + this.uuid = uuid; + this.originFileName = originFileName; + this.fileSize = fileSize; + this.columnSeparator = columnSeparator; + } + + public void save(MultipartFile file) throws IOException { + File dest = new File(Joiner.on("/").join(rootDir, uuid)); + boolean uploadSucceed = false; + try { + file.transferTo(dest); + this.absPath = dest.getAbsolutePath(); + uploadSucceed = true; + LOG.info("upload file {} succeed at {}", this, dest.getAbsolutePath()); + } catch (IOException e) { + LOG.warn("failed to upload file {}, dest: {}", this, dest.getAbsolutePath(), e); + throw e; + } finally { + if (!uploadSucceed) { + dest.delete(); + } + } + } + + public void setPreview() throws IOException { + lines = Lists.newArrayList(); + try (FileReader fr = new FileReader(absPath); + BufferedReader bf = new BufferedReader(fr)) { + String str; + while ((str = bf.readLine()) != null) { + String[] cols = str.split(columnSeparator); + lines.add(Lists.newArrayList(cols)); + if (cols.length > maxColNum) { + maxColNum = cols.length; + } + if (lines.size() >= MAX_PREVIEW_LINES) { + break; + } + } + } + } + + // make a copy without lines and maxColNum. + // so that can call `setPreview` and will not affect other instance + public TmpFile copy() { + TmpFile copiedFile = new TmpFile(this.id, this.uuid, this.originFileName, this.fileSize, this.columnSeparator); + copiedFile.absPath = this.absPath; + return copiedFile; + } + + public void delete() { + File file = new File(absPath); + file.delete(); + LOG.info("delete tmp file: {}", this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("[id=").append(id).append(", uuid=").append(uuid).append(", origin name=").append(originFileName) + .append(", size=").append(fileSize).append("]"); + return sb.toString(); + } + } + + // a brief of TmpFile. + // TODO(cmy): it can be removed by using Lombok's annotation in TmpFile class + public static class TmpFileBrief { + public long id; + public String uuid; + public String originFileName; + public long fileSize; + public String columnSeparator; + + public TmpFileBrief(TmpFile tmpFile) { + this.id = tmpFile.id; + this.uuid = tmpFile.uuid; + this.originFileName = tmpFile.originFileName; + this.fileSize = tmpFile.fileSize; + this.columnSeparator = tmpFile.columnSeparator; + } + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getOriginFileName() { + return originFileName; + } + + public void setOriginFileName(String originFileName) { + this.originFileName = originFileName; + } + + public long getFileSize() { + return fileSize; + } + + public void setFileSize(long fileSize) { + this.fileSize = fileSize; + } + + public String getColumnSeparator() { + return columnSeparator; + } + + public void setColumnSeparator(String columnSeparator) { + this.columnSeparator = columnSeparator; + } + } + + + public static class UploadFile { + public MultipartFile file; + public String columnSeparator; + + public UploadFile(MultipartFile file, String columnSeparator) { + this.file = file; + this.columnSeparator = columnSeparator; + } + } + + public static class TmpFileException extends Exception { + public TmpFileException(String msg) { + super(msg); + } + + public TmpFileException(String msg, Throwable t) { + super(msg, t); + } + } +} +