[Refactor][httpv2]remove http v1 code (#8848)
http v2 has been actually tested in production, and it is completely replaceable to have http code. In order to simplify code maintenance, remove the previous http part of the code
This commit is contained in:
jiafeng.zhang
@ -26,7 +26,7 @@ import org.apache.doris.common.ThreadPoolManager;
|
||||
import org.apache.doris.common.Version;
|
||||
import org.apache.doris.common.util.JdkUtils;
|
||||
import org.apache.doris.common.util.NetUtils;
|
||||
import org.apache.doris.http.HttpServer;
|
||||
import org.apache.doris.httpv2.HttpServer;
|
||||
import org.apache.doris.journal.bdbje.BDBDebugger;
|
||||
import org.apache.doris.journal.bdbje.BDBTool;
|
||||
import org.apache.doris.journal.bdbje.BDBToolOptions;
|
||||
@ -134,25 +134,14 @@ public class PaloFe {
|
||||
|
||||
feServer.start();
|
||||
|
||||
if (!Config.enable_http_server_v2) {
|
||||
HttpServer httpServer = new HttpServer(
|
||||
Config.http_port,
|
||||
Config.http_max_line_length,
|
||||
Config.http_max_header_size,
|
||||
Config.http_max_chunk_size
|
||||
);
|
||||
httpServer.setup();
|
||||
httpServer.start();
|
||||
} else {
|
||||
org.apache.doris.httpv2.HttpServer httpServer2 = new org.apache.doris.httpv2.HttpServer();
|
||||
httpServer2.setPort(Config.http_port);
|
||||
httpServer2.setMaxHttpPostSize(Config.jetty_server_max_http_post_size);
|
||||
httpServer2.setAcceptors(Config.jetty_server_acceptors);
|
||||
httpServer2.setSelectors(Config.jetty_server_selectors);
|
||||
httpServer2.setWorkers(Config.jetty_server_workers);
|
||||
httpServer2.start(dorisHomeDir);
|
||||
}
|
||||
|
||||
HttpServer httpServer = new HttpServer();
|
||||
httpServer.setPort(Config.http_port);
|
||||
httpServer.setMaxHttpPostSize(Config.jetty_server_max_http_post_size);
|
||||
httpServer.setAcceptors(Config.jetty_server_acceptors);
|
||||
httpServer.setSelectors(Config.jetty_server_selectors);
|
||||
httpServer.setWorkers(Config.jetty_server_workers);
|
||||
httpServer.start(dorisHomeDir);
|
||||
|
||||
qeService.start();
|
||||
|
||||
ThreadPoolManager.registerAllThreadPoolMetric();
|
||||
|
||||
@ -156,7 +156,7 @@ import org.apache.doris.ha.BDBHA;
|
||||
import org.apache.doris.ha.FrontendNodeType;
|
||||
import org.apache.doris.ha.HAProtocol;
|
||||
import org.apache.doris.ha.MasterInfo;
|
||||
import org.apache.doris.http.meta.MetaBaseAction;
|
||||
import org.apache.doris.httpv2.meta.MetaBaseAction;
|
||||
import org.apache.doris.journal.JournalCursor;
|
||||
import org.apache.doris.journal.JournalEntity;
|
||||
import org.apache.doris.journal.bdbje.Timestamp;
|
||||
|
||||
@ -327,22 +327,6 @@ public class Config extends ConfigBase {
|
||||
*/
|
||||
@ConfField public static int http_port = 8030;
|
||||
|
||||
/*
|
||||
* Netty http param
|
||||
*/
|
||||
@ConfField public static int http_max_line_length = HttpServer.DEFAULT_MAX_LINE_LENGTH;
|
||||
|
||||
@ConfField public static int http_max_header_size = HttpServer.DEFAULT_MAX_HEADER_SIZE;
|
||||
|
||||
@ConfField public static int http_max_chunk_size = HttpServer.DEFAULT_MAX_CHUNK_SIZE;
|
||||
|
||||
/**
|
||||
* The backlog_num for netty http server
|
||||
* When you enlarge this backlog_num, you should enlarge the value in
|
||||
* the linux /proc/sys/net/core/somaxconn file at the same time
|
||||
*/
|
||||
@ConfField public static int http_backlog_num = 1024;
|
||||
|
||||
/**
|
||||
* Jetty container default configuration
|
||||
* Jetty's thread architecture model is very simple, divided into three thread pools:
|
||||
|
||||
@ -1,88 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http;
|
||||
|
||||
import org.apache.doris.common.path.PathTrie;
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
public class ActionController {
|
||||
private final PathTrie<IAction> getHandlers = new PathTrie<>(WebUtils.REST_DECODER);
|
||||
private final PathTrie<IAction> postHandlers = new PathTrie<>(WebUtils.REST_DECODER);
|
||||
private final PathTrie<IAction> putHandlers = new PathTrie<>(WebUtils.REST_DECODER);
|
||||
private final PathTrie<IAction> deleteHandlers = new PathTrie<>(WebUtils.REST_DECODER);
|
||||
private final PathTrie<IAction> headHandlers = new PathTrie<>(WebUtils.REST_DECODER);
|
||||
private final PathTrie<IAction> optionsHandlers = new PathTrie<>(WebUtils.REST_DECODER);
|
||||
|
||||
// Registers a rest handler to be execute when the provided method and path match the request.
|
||||
public void registerHandler(HttpMethod method, String path, IAction handler)
|
||||
throws IllegalArgException {
|
||||
if (method.equals(HttpMethod.GET)) {
|
||||
getHandlers.insert(path, handler);
|
||||
} else if (method.equals(HttpMethod.POST)) {
|
||||
postHandlers.insert(path, handler);
|
||||
} else if (method.equals(HttpMethod.HEAD)) {
|
||||
headHandlers.insert(path, handler);
|
||||
} else if (method.equals(HttpMethod.DELETE)) {
|
||||
deleteHandlers.insert(path, handler);
|
||||
} else if (method.equals(HttpMethod.OPTIONS)) {
|
||||
optionsHandlers.insert(path, handler);
|
||||
} else if (method.equals(HttpMethod.PUT)) {
|
||||
putHandlers.insert(path, handler);
|
||||
} else {
|
||||
throw new IllegalArgException(
|
||||
"Can't handle [" + method + "] for path [" + path + "]");
|
||||
}
|
||||
}
|
||||
|
||||
public IAction getHandler(BaseRequest request) {
|
||||
String path = getPath(request.getRequest().uri());
|
||||
HttpMethod method = request.getRequest().method();
|
||||
if (method.equals(HttpMethod.GET)) {
|
||||
return getHandlers.retrieve(path, request.getParams());
|
||||
} else if (method.equals(HttpMethod.POST)) {
|
||||
return postHandlers.retrieve(path, request.getParams());
|
||||
} else if (method.equals(HttpMethod.PUT)) {
|
||||
return putHandlers.retrieve(path, request.getParams());
|
||||
} else if (method.equals(HttpMethod.DELETE)) {
|
||||
return deleteHandlers.retrieve(path, request.getParams());
|
||||
} else if (method.equals(HttpMethod.HEAD)) {
|
||||
return headHandlers.retrieve(path, request.getParams());
|
||||
} else if (method.equals(HttpMethod.OPTIONS)) {
|
||||
return optionsHandlers.retrieve(path, request.getParams());
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// e.g.
|
||||
// in: /www/system?path=//jobs
|
||||
// out: /www/system
|
||||
private String getPath(String uri) {
|
||||
if (Strings.isNullOrEmpty(uri)) {
|
||||
return "";
|
||||
}
|
||||
int pathEndIndex = uri.indexOf('?');
|
||||
if (pathEndIndex < 0) {
|
||||
return uri;
|
||||
} else {
|
||||
return uri.substring(0, pathEndIndex);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,375 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http;
|
||||
|
||||
import org.apache.doris.analysis.UserIdentity;
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.cluster.ClusterNamespace;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.system.SystemInfoService;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import io.netty.buffer.ByteBuf;
|
||||
import io.netty.buffer.Unpooled;
|
||||
import io.netty.channel.ChannelFuture;
|
||||
import io.netty.channel.ChannelFutureListener;
|
||||
import io.netty.channel.ChannelProgressiveFuture;
|
||||
import io.netty.channel.ChannelProgressiveFutureListener;
|
||||
import io.netty.channel.DefaultFileRegion;
|
||||
import io.netty.handler.codec.base64.Base64;
|
||||
import io.netty.handler.codec.http.DefaultFullHttpResponse;
|
||||
import io.netty.handler.codec.http.DefaultHttpResponse;
|
||||
import io.netty.handler.codec.http.FullHttpResponse;
|
||||
import io.netty.handler.codec.http.HttpChunkedInput;
|
||||
import io.netty.handler.codec.http.HttpHeaderNames;
|
||||
import io.netty.handler.codec.http.HttpHeaderValues;
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponse;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
import io.netty.handler.codec.http.HttpUtil;
|
||||
import io.netty.handler.codec.http.HttpVersion;
|
||||
import io.netty.handler.codec.http.LastHttpContent;
|
||||
import io.netty.handler.codec.http.cookie.Cookie;
|
||||
import io.netty.handler.codec.http.cookie.ServerCookieEncoder;
|
||||
import io.netty.handler.ssl.SslHandler;
|
||||
import io.netty.handler.stream.ChunkedFile;
|
||||
import io.netty.handler.stream.ChunkedInput;
|
||||
import io.netty.handler.stream.ChunkedStream;
|
||||
import io.netty.util.CharsetUtil;
|
||||
|
||||
public abstract class BaseAction implements IAction {
|
||||
private static final Logger LOG = LogManager.getLogger(BaseAction.class);
|
||||
|
||||
protected ActionController controller;
|
||||
protected Catalog catalog;
|
||||
|
||||
public BaseAction(ActionController controller) {
|
||||
this.controller = controller;
|
||||
// TODO(zc): remove this instance
|
||||
this.catalog = Catalog.getCurrentCatalog();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handleRequest(BaseRequest request) throws Exception {
|
||||
BaseResponse response = new BaseResponse();
|
||||
LOG.debug("receive http request. url={}", request.getRequest().uri());
|
||||
try {
|
||||
execute(request, response);
|
||||
} catch (Exception e) {
|
||||
LOG.warn("fail to process url: {}", request.getRequest().uri(), e);
|
||||
if (e instanceof UnauthorizedException) {
|
||||
response.updateHeader(HttpHeaderNames.WWW_AUTHENTICATE.toString(), "Basic realm=\"\"");
|
||||
writeResponse(request, response, HttpResponseStatus.UNAUTHORIZED);
|
||||
} else {
|
||||
writeResponse(request, response, HttpResponseStatus.NOT_FOUND);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public abstract void execute(BaseRequest request, BaseResponse response) throws DdlException;
|
||||
|
||||
protected void writeResponse(BaseRequest request, BaseResponse response, HttpResponseStatus status) {
|
||||
// if (HttpHeaders.is100ContinueExpected(request.getRequest())) {
|
||||
// ctx.write(new DefaultFullHttpResponse(HttpVersion.HTTP_1_1,
|
||||
// HttpResponseStatus.CONTINUE));
|
||||
// }
|
||||
|
||||
FullHttpResponse responseObj = null;
|
||||
try {
|
||||
responseObj = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, status,
|
||||
Unpooled.wrappedBuffer(response.getContent().toString().getBytes("UTF-8")));
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
LOG.warn("get exception.", e);
|
||||
responseObj = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, status,
|
||||
Unpooled.wrappedBuffer(response.getContent().toString().getBytes()));
|
||||
}
|
||||
Preconditions.checkNotNull(responseObj);
|
||||
HttpMethod method = request.getRequest().method();
|
||||
|
||||
checkDefaultContentTypeHeader(response, responseObj);
|
||||
if (!method.equals(HttpMethod.HEAD)) {
|
||||
response.updateHeader(HttpHeaderNames.CONTENT_LENGTH.toString(),
|
||||
String.valueOf(responseObj.content().readableBytes()));
|
||||
}
|
||||
writeCustomHeaders(response, responseObj);
|
||||
writeCookies(response, responseObj);
|
||||
|
||||
boolean keepAlive = HttpUtil.isKeepAlive(request.getRequest());
|
||||
if (!keepAlive) {
|
||||
request.getContext().write(responseObj).addListener(ChannelFutureListener.CLOSE);
|
||||
} else {
|
||||
responseObj.headers().set(HttpHeaderNames.CONNECTION.toString(), HttpHeaderValues.KEEP_ALIVE.toString());
|
||||
request.getContext().write(responseObj);
|
||||
}
|
||||
}
|
||||
|
||||
// Object only support File or byte[]
|
||||
protected void writeObjectResponse(BaseRequest request, BaseResponse response, HttpResponseStatus status,
|
||||
Object obj, String fileName, boolean isOctStream) {
|
||||
Preconditions.checkState((obj instanceof File) || (obj instanceof byte[]));
|
||||
|
||||
HttpResponse responseObj = new DefaultHttpResponse(HttpVersion.HTTP_1_1, status);
|
||||
|
||||
if (HttpUtil.isKeepAlive(request.getRequest())) {
|
||||
response.updateHeader(HttpHeaderNames.CONNECTION.toString(), HttpHeaderValues.KEEP_ALIVE.toString());
|
||||
}
|
||||
|
||||
if (isOctStream) {
|
||||
response.updateHeader(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.APPLICATION_OCTET_STREAM.toString());
|
||||
response.updateHeader(HttpHeaderNames.CONTENT_DISPOSITION.toString(),
|
||||
HttpHeaderValues.ATTACHMENT.toString() + "; " + HttpHeaderValues.FILENAME.toString() + "=" + fileName);
|
||||
}
|
||||
|
||||
ChannelFuture sendFileFuture;
|
||||
ChannelFuture lastContentFuture;
|
||||
|
||||
try {
|
||||
Object writable = null;
|
||||
long contentLen = 0;
|
||||
boolean sslEnable = request.getContext().pipeline().get(SslHandler.class) != null;
|
||||
if (obj instanceof File) {
|
||||
RandomAccessFile rafFile = new RandomAccessFile((File) obj, "r");
|
||||
contentLen = rafFile.length();
|
||||
if (!sslEnable) {
|
||||
// use zero-copy file transfer.
|
||||
writable = new DefaultFileRegion(rafFile.getChannel(), 0, contentLen);
|
||||
} else {
|
||||
// cannot use zero-copy file transfer.
|
||||
writable = new ChunkedFile(rafFile, 0, contentLen, 8192);
|
||||
}
|
||||
} else if (obj instanceof byte[]) {
|
||||
contentLen = ((byte[]) obj).length;
|
||||
if (!sslEnable) {
|
||||
writable = Unpooled.wrappedBuffer((byte[]) obj);
|
||||
} else {
|
||||
writable = new ChunkedStream(new ByteArrayInputStream((byte[]) obj));
|
||||
}
|
||||
}
|
||||
|
||||
response.updateHeader(HttpHeaderNames.CONTENT_LENGTH.toString(), String.valueOf(contentLen));
|
||||
writeCookies(response, responseObj);
|
||||
writeCustomHeaders(response, responseObj);
|
||||
|
||||
// Write headers
|
||||
request.getContext().write(responseObj);
|
||||
|
||||
// Write object
|
||||
if (!sslEnable) {
|
||||
sendFileFuture = request.getContext().write(writable, request.getContext().newProgressivePromise());
|
||||
// Write the end marker.
|
||||
lastContentFuture = request.getContext().writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
|
||||
} else {
|
||||
sendFileFuture = request.getContext().writeAndFlush(
|
||||
new HttpChunkedInput((ChunkedInput<ByteBuf>) writable),
|
||||
request.getContext().newProgressivePromise());
|
||||
// HttpChunkedInput will write the end marker (LastHttpContent) for us.
|
||||
lastContentFuture = sendFileFuture;
|
||||
}
|
||||
} catch (FileNotFoundException ignore) {
|
||||
writeResponse(request, response, HttpResponseStatus.NOT_FOUND);
|
||||
return;
|
||||
} catch (IOException e1) {
|
||||
writeResponse(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR);
|
||||
return;
|
||||
}
|
||||
|
||||
sendFileFuture.addListener(new ChannelProgressiveFutureListener() {
|
||||
@Override
|
||||
public void operationProgressed(ChannelProgressiveFuture future, long progress, long total) {
|
||||
if (total < 0) { // total unknown
|
||||
LOG.debug("{} Transfer progress: {}", future.channel(), progress);
|
||||
} else {
|
||||
LOG.debug("{} Transfer progress: {} / {}", future.channel(), progress, total);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void operationComplete(ChannelProgressiveFuture future) {
|
||||
LOG.debug("{} Transfer complete.", future.channel());
|
||||
if (!future.isSuccess()) {
|
||||
Throwable cause = future.cause();
|
||||
LOG.error("something wrong. ", cause);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Decide whether to close the connection or not.
|
||||
boolean keepAlive = HttpUtil.isKeepAlive(request.getRequest());
|
||||
if (!keepAlive) {
|
||||
// Close the connection when the whole content is written out.
|
||||
lastContentFuture.addListener(ChannelFutureListener.CLOSE);
|
||||
}
|
||||
}
|
||||
|
||||
// Set 'CONTENT_TYPE' header if it hasn't been set.
|
||||
protected void checkDefaultContentTypeHeader(BaseResponse response, Object responseOj) {
|
||||
if (!Strings.isNullOrEmpty(response.getContentType())) {
|
||||
response.updateHeader(HttpHeaderNames.CONTENT_TYPE.toString(), response.getContentType());
|
||||
} else {
|
||||
response.updateHeader(HttpHeaderNames.CONTENT_TYPE.toString(), "text/html");
|
||||
}
|
||||
}
|
||||
|
||||
protected void writeCustomHeaders(BaseResponse response, HttpResponse responseObj) {
|
||||
for (Map.Entry<String, List<String>> entry : response.getCustomHeaders().entrySet()) {
|
||||
responseObj.headers().add(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
protected void writeCookies(BaseResponse response, HttpResponse responseObj) {
|
||||
for (Cookie cookie : response.getCookies()) {
|
||||
responseObj.headers().add(HttpHeaderNames.SET_COOKIE.toString(), ServerCookieEncoder.LAX.encode(cookie));
|
||||
}
|
||||
}
|
||||
|
||||
public static class ActionAuthorizationInfo {
|
||||
public String fullUserName;
|
||||
public String remoteIp;
|
||||
public String password;
|
||||
public String cluster;
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("user: ").append(fullUserName).append(", remote ip: ").append(remoteIp);
|
||||
sb.append(", password: ").append(password).append(", cluster: ").append(cluster);
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
|
||||
protected void checkGlobalAuth(UserIdentity currentUser, PrivPredicate predicate) throws UnauthorizedException {
|
||||
if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(currentUser, predicate)) {
|
||||
throw new UnauthorizedException("Access denied; you need (at least one of) the "
|
||||
+ predicate.getPrivs().toString() + " privilege(s) for this operation");
|
||||
}
|
||||
}
|
||||
|
||||
protected void checkDbAuth(UserIdentity currentUser, String db, PrivPredicate predicate)
|
||||
throws UnauthorizedException {
|
||||
if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(currentUser, db, predicate)) {
|
||||
throw new UnauthorizedException("Access denied; you need (at least one of) the "
|
||||
+ predicate.getPrivs().toString() + " privilege(s) for this operation");
|
||||
}
|
||||
}
|
||||
|
||||
protected void checkTblAuth(UserIdentity currentUser, String db, String tbl, PrivPredicate predicate)
|
||||
throws UnauthorizedException {
|
||||
if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(currentUser, db, tbl, predicate)) {
|
||||
throw new UnauthorizedException("Access denied; you need (at least one of) the "
|
||||
+ predicate.getPrivs().toString() + " privilege(s) for this operation");
|
||||
}
|
||||
}
|
||||
|
||||
// return currentUserIdentity from Doris auth
|
||||
protected UserIdentity checkPassword(ActionAuthorizationInfo authInfo)
|
||||
throws UnauthorizedException {
|
||||
List<UserIdentity> currentUser = Lists.newArrayList();
|
||||
if (!Catalog.getCurrentCatalog().getAuth().checkPlainPassword(authInfo.fullUserName,
|
||||
authInfo.remoteIp, authInfo.password, currentUser)) {
|
||||
throw new UnauthorizedException("Access denied for "
|
||||
+ authInfo.fullUserName + "@" + authInfo.remoteIp);
|
||||
}
|
||||
Preconditions.checkState(currentUser.size() == 1);
|
||||
return currentUser.get(0);
|
||||
}
|
||||
|
||||
public ActionAuthorizationInfo getAuthorizationInfo(BaseRequest request)
|
||||
throws UnauthorizedException {
|
||||
ActionAuthorizationInfo authInfo = new ActionAuthorizationInfo();
|
||||
if (!parseAuthInfo(request, authInfo)) {
|
||||
LOG.info("parse auth info failed, Authorization header {}, url {}",
|
||||
request.getAuthorizationHeader(), request.getRequest().uri());
|
||||
throw new UnauthorizedException("Need auth information.");
|
||||
}
|
||||
LOG.debug("get auth info: {}", authInfo);
|
||||
return authInfo;
|
||||
}
|
||||
|
||||
private boolean parseAuthInfo(BaseRequest request, ActionAuthorizationInfo authInfo) {
|
||||
String encodedAuthString = request.getAuthorizationHeader();
|
||||
if (Strings.isNullOrEmpty(encodedAuthString)) {
|
||||
return false;
|
||||
}
|
||||
String[] parts = encodedAuthString.split(" ");
|
||||
if (parts.length != 2) {
|
||||
return false;
|
||||
}
|
||||
encodedAuthString = parts[1];
|
||||
ByteBuf buf = null;
|
||||
ByteBuf decodeBuf = null;
|
||||
try {
|
||||
buf = Unpooled.copiedBuffer(ByteBuffer.wrap(encodedAuthString.getBytes()));
|
||||
|
||||
// The authString is a string connecting user-name and password with
|
||||
// a colon(':')
|
||||
decodeBuf = Base64.decode(buf);
|
||||
String authString = decodeBuf.toString(CharsetUtil.UTF_8);
|
||||
// Note that password may contain colon, so can not simply use a
|
||||
// colon to split.
|
||||
int index = authString.indexOf(":");
|
||||
authInfo.fullUserName = authString.substring(0, index);
|
||||
final String[] elements = authInfo.fullUserName.split("@");
|
||||
if (elements != null && elements.length < 2) {
|
||||
authInfo.fullUserName = ClusterNamespace.getFullName(SystemInfoService.DEFAULT_CLUSTER,
|
||||
authInfo.fullUserName);
|
||||
authInfo.cluster = SystemInfoService.DEFAULT_CLUSTER;
|
||||
} else if (elements != null && elements.length == 2) {
|
||||
authInfo.fullUserName = ClusterNamespace.getFullName(elements[1], elements[0]);
|
||||
authInfo.cluster = elements[1];
|
||||
}
|
||||
authInfo.password = authString.substring(index + 1);
|
||||
authInfo.remoteIp = request.getHostString();
|
||||
} finally {
|
||||
// release the buf and decode buf after using Unpooled.copiedBuffer
|
||||
// or it will get memory leak
|
||||
if (buf != null) {
|
||||
buf.release();
|
||||
}
|
||||
|
||||
if (decodeBuf != null) {
|
||||
decodeBuf.release();
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
protected int checkIntParam(String strParam) {
|
||||
return Integer.parseInt(strParam);
|
||||
}
|
||||
|
||||
protected long checkLongParam(String strParam) {
|
||||
return Long.parseLong(strParam);
|
||||
}
|
||||
}
|
||||
@ -1,158 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http;
|
||||
|
||||
import org.apache.doris.common.DdlException;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.Maps;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
import io.netty.handler.codec.http.FullHttpRequest;
|
||||
import io.netty.handler.codec.http.HttpHeaderNames;
|
||||
import io.netty.handler.codec.http.HttpRequest;
|
||||
import io.netty.handler.codec.http.QueryStringDecoder;
|
||||
import io.netty.handler.codec.http.cookie.ClientCookieDecoder;
|
||||
import io.netty.handler.codec.http.cookie.Cookie;
|
||||
|
||||
public class BaseRequest {
|
||||
protected ChannelHandlerContext context;
|
||||
protected HttpRequest request;
|
||||
protected Map<String, String> params = Maps.newHashMap();
|
||||
|
||||
private boolean isAuthorized = false;
|
||||
private QueryStringDecoder decoder;
|
||||
|
||||
public BaseRequest(ChannelHandlerContext ctx, HttpRequest request) {
|
||||
this.context = ctx;
|
||||
this.request = request;
|
||||
}
|
||||
|
||||
public ChannelHandlerContext getContext() {
|
||||
return context;
|
||||
}
|
||||
|
||||
public void setContext(ChannelHandlerContext context) {
|
||||
this.context = context;
|
||||
}
|
||||
|
||||
public HttpRequest getRequest() {
|
||||
return request;
|
||||
}
|
||||
|
||||
public void setRequest(HttpRequest request) {
|
||||
this.request = request;
|
||||
}
|
||||
|
||||
public Map<String, String> getParams() {
|
||||
return params;
|
||||
}
|
||||
|
||||
public void setParams(Map<String, String> params) {
|
||||
this.params = params;
|
||||
}
|
||||
|
||||
public boolean isAuthorized() {
|
||||
return isAuthorized;
|
||||
}
|
||||
|
||||
public void setAuthorized(boolean isAuthorized) {
|
||||
this.isAuthorized = isAuthorized;
|
||||
}
|
||||
|
||||
public Cookie getCookieByName(String cookieName) {
|
||||
String cookieString = request.headers().get(HttpHeaderNames.COOKIE.toString());
|
||||
if (!Strings.isNullOrEmpty(cookieString)) {
|
||||
Cookie cookie = ClientCookieDecoder.STRICT.decode(cookieString);
|
||||
if (cookie.name().equalsIgnoreCase(cookieName)) {
|
||||
return cookie;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public String getCookieValue(String cookieName) {
|
||||
Cookie cookie = getCookieByName(cookieName);
|
||||
if (cookie != null) {
|
||||
return cookie.value();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// get a single parameter.
|
||||
// return null if key is not exist; return the first value if key is an array
|
||||
public String getSingleParameter(String key) {
|
||||
String uri = request.uri();
|
||||
if (decoder == null) {
|
||||
decoder = new QueryStringDecoder(uri);
|
||||
}
|
||||
|
||||
List<String> values = decoder.parameters().get(key);
|
||||
if (values != null && values.size() > 0) {
|
||||
return values.get(0);
|
||||
}
|
||||
|
||||
return params.get(key);
|
||||
}
|
||||
|
||||
public String getContent() throws DdlException {
|
||||
if (request instanceof FullHttpRequest) {
|
||||
FullHttpRequest fullHttpRequest = (FullHttpRequest) request;
|
||||
return fullHttpRequest.content().toString(Charset.forName("UTF-8"));
|
||||
} else {
|
||||
throw new DdlException("Invalid request");
|
||||
}
|
||||
}
|
||||
|
||||
// get an array parameter.
|
||||
// eg. ?a=1&a=2
|
||||
public List<String> getArrayParameter(String key) {
|
||||
String uri = request.uri();
|
||||
if (decoder == null) {
|
||||
decoder = new QueryStringDecoder(uri);
|
||||
}
|
||||
|
||||
return decoder.parameters().get(key);
|
||||
}
|
||||
|
||||
public Map<String, List<String>> getAllParameters() {
|
||||
String uri = request.uri();
|
||||
if (decoder == null) {
|
||||
decoder = new QueryStringDecoder(uri);
|
||||
}
|
||||
|
||||
return decoder.parameters();
|
||||
}
|
||||
|
||||
public String getAuthorizationHeader() {
|
||||
String authString = request.headers().get("Authorization");
|
||||
return authString;
|
||||
}
|
||||
|
||||
public String getHostString() {
|
||||
// get client host
|
||||
InetSocketAddress clientSocket = (InetSocketAddress) context.channel().remoteAddress();
|
||||
String clientIp = clientSocket.getHostString();
|
||||
return clientIp;
|
||||
}
|
||||
}
|
||||
@ -1,99 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Maps;
|
||||
import com.google.common.collect.Sets;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import io.netty.handler.codec.http.cookie.Cookie;
|
||||
|
||||
public class BaseResponse {
|
||||
private String contentType;
|
||||
protected StringBuilder content = new StringBuilder();
|
||||
|
||||
protected Map<String, List<String>> customHeaders = Maps.newHashMap();
|
||||
private Set<Cookie> cookies = Sets.newHashSet();
|
||||
|
||||
public String getContentType() {
|
||||
return contentType;
|
||||
}
|
||||
|
||||
public void setContentType(String contentType) {
|
||||
this.contentType = contentType;
|
||||
}
|
||||
|
||||
public StringBuilder getContent() {
|
||||
return content;
|
||||
}
|
||||
|
||||
public Set<Cookie> getCookies() {
|
||||
return cookies;
|
||||
}
|
||||
|
||||
public Map<String, List<String>> getCustomHeaders() {
|
||||
return customHeaders;
|
||||
}
|
||||
|
||||
// update old key-value mapping of 'name' if Exist, or add new mapping if not exists.
|
||||
// It will only change the mapping of 'name', other header will not be changed.
|
||||
public void updateHeader(String name, String value) {
|
||||
if (customHeaders == null) {
|
||||
customHeaders = Maps.newHashMap();
|
||||
}
|
||||
customHeaders.remove(name);
|
||||
addHeader(name, value);
|
||||
}
|
||||
|
||||
// Add a custom header.
|
||||
private void addHeader(String name, String value) {
|
||||
if (customHeaders == null) {
|
||||
customHeaders = Maps.newHashMap();
|
||||
}
|
||||
List<String> header = customHeaders.get(name);
|
||||
if (header == null) {
|
||||
header = Lists.newArrayList();
|
||||
customHeaders.put(name, header);
|
||||
}
|
||||
header.add(value);
|
||||
}
|
||||
|
||||
public void appendContent(String buffer) {
|
||||
if (content == null) {
|
||||
content = new StringBuilder();
|
||||
}
|
||||
content.append(buffer);
|
||||
}
|
||||
|
||||
public void addCookie(Cookie cookie) {
|
||||
cookies.add(cookie);
|
||||
}
|
||||
|
||||
public void updateCookieAge(BaseRequest request, String cookieName, long age) {
|
||||
Cookie cookie = request.getCookieByName(cookieName);
|
||||
if (cookie != null) {
|
||||
cookies.remove(cookie);
|
||||
cookie.setMaxAge(age);
|
||||
cookies.add(cookie);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,64 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http;
|
||||
|
||||
import org.apache.doris.analysis.UserIdentity;
|
||||
|
||||
import com.google.common.cache.Cache;
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
// We simulate a simplified session here: only store user-name of clients who already logged in,
|
||||
// and we only have a default admin user for now.
|
||||
public final class HttpAuthManager {
|
||||
private static long SESSION_EXPIRE_TIME = 2; // hour
|
||||
private static long SESSION_MAX_SIZE = 100; // avoid to store too many
|
||||
|
||||
private static HttpAuthManager instance = new HttpAuthManager();
|
||||
|
||||
public static class SessionValue {
|
||||
public UserIdentity currentUser;
|
||||
}
|
||||
|
||||
// session_id => session value
|
||||
private Cache<String, SessionValue> authSessions = CacheBuilder.newBuilder()
|
||||
.maximumSize(SESSION_MAX_SIZE)
|
||||
.expireAfterAccess(SESSION_EXPIRE_TIME, TimeUnit.HOURS)
|
||||
.build();
|
||||
|
||||
private HttpAuthManager() {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
public static HttpAuthManager getInstance() {
|
||||
return instance;
|
||||
}
|
||||
|
||||
public SessionValue getSessionValue(String sessionId) {
|
||||
return authSessions.getIfPresent(sessionId);
|
||||
}
|
||||
|
||||
public void addSessionValue(String key, SessionValue value) {
|
||||
authSessions.put(key, value);
|
||||
}
|
||||
|
||||
public Cache<String, SessionValue> getAuthSessions() {
|
||||
return authSessions;
|
||||
}
|
||||
}
|
||||
@ -1,279 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http;
|
||||
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.http.action.BackendAction;
|
||||
import org.apache.doris.http.action.HaAction;
|
||||
import org.apache.doris.http.action.HelpAction;
|
||||
import org.apache.doris.http.action.IndexAction;
|
||||
import org.apache.doris.http.action.LogAction;
|
||||
import org.apache.doris.http.action.QueryAction;
|
||||
import org.apache.doris.http.action.QueryProfileAction;
|
||||
import org.apache.doris.http.action.SessionAction;
|
||||
import org.apache.doris.http.action.StaticResourceAction;
|
||||
import org.apache.doris.http.action.SystemAction;
|
||||
import org.apache.doris.http.action.VariableAction;
|
||||
import org.apache.doris.http.common.DorisHttpPostObjectAggregator;
|
||||
import org.apache.doris.http.meta.ColocateMetaService;
|
||||
import org.apache.doris.http.meta.MetaService.CheckAction;
|
||||
import org.apache.doris.http.meta.MetaService.DumpAction;
|
||||
import org.apache.doris.http.meta.MetaService.ImageAction;
|
||||
import org.apache.doris.http.meta.MetaService.InfoAction;
|
||||
import org.apache.doris.http.meta.MetaService.JournalIdAction;
|
||||
import org.apache.doris.http.meta.MetaService.PutAction;
|
||||
import org.apache.doris.http.meta.MetaService.RoleAction;
|
||||
import org.apache.doris.http.meta.MetaService.VersionAction;
|
||||
import org.apache.doris.http.rest.BootstrapFinishAction;
|
||||
import org.apache.doris.http.rest.CancelStreamLoad;
|
||||
import org.apache.doris.http.rest.CheckDecommissionAction;
|
||||
import org.apache.doris.http.rest.ConnectionAction;
|
||||
import org.apache.doris.http.rest.GetDdlStmtAction;
|
||||
import org.apache.doris.http.rest.GetLoadInfoAction;
|
||||
import org.apache.doris.http.rest.GetLogFileAction;
|
||||
import org.apache.doris.http.rest.GetSmallFileAction;
|
||||
import org.apache.doris.http.rest.GetStreamLoadState;
|
||||
import org.apache.doris.http.rest.HealthAction;
|
||||
import org.apache.doris.http.rest.LoadAction;
|
||||
import org.apache.doris.http.rest.MetaReplayerCheckAction;
|
||||
import org.apache.doris.http.rest.MetricsAction;
|
||||
import org.apache.doris.http.rest.MigrationAction;
|
||||
import org.apache.doris.http.rest.MultiAbort;
|
||||
import org.apache.doris.http.rest.MultiCommit;
|
||||
import org.apache.doris.http.rest.MultiDesc;
|
||||
import org.apache.doris.http.rest.MultiList;
|
||||
import org.apache.doris.http.rest.MultiStart;
|
||||
import org.apache.doris.http.rest.MultiUnload;
|
||||
import org.apache.doris.http.rest.ProfileAction;
|
||||
import org.apache.doris.http.rest.QueryDetailAction;
|
||||
import org.apache.doris.http.rest.RowCountAction;
|
||||
import org.apache.doris.http.rest.SetConfigAction;
|
||||
import org.apache.doris.http.rest.ShowDataAction;
|
||||
import org.apache.doris.http.rest.ShowMetaInfoAction;
|
||||
import org.apache.doris.http.rest.ShowProcAction;
|
||||
import org.apache.doris.http.rest.ShowRuntimeInfoAction;
|
||||
import org.apache.doris.http.rest.StorageTypeCheckAction;
|
||||
import org.apache.doris.http.rest.TableQueryPlanAction;
|
||||
import org.apache.doris.http.rest.TableRowCountAction;
|
||||
import org.apache.doris.http.rest.TableSchemaAction;
|
||||
import org.apache.doris.master.MetaHelper;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import io.netty.bootstrap.ServerBootstrap;
|
||||
import io.netty.channel.Channel;
|
||||
import io.netty.channel.ChannelInitializer;
|
||||
import io.netty.channel.ChannelOption;
|
||||
import io.netty.channel.EventLoopGroup;
|
||||
import io.netty.channel.nio.NioEventLoopGroup;
|
||||
import io.netty.channel.socket.SocketChannel;
|
||||
import io.netty.channel.socket.nio.NioServerSocketChannel;
|
||||
import io.netty.handler.codec.http.HttpServerCodec;
|
||||
import io.netty.handler.stream.ChunkedWriteHandler;
|
||||
|
||||
public class HttpServer {
|
||||
private static final Logger LOG = LogManager.getLogger(HttpServer.class);
|
||||
|
||||
/**
|
||||
* The default netty param, witch is the same as `HttpServerCodec`.
|
||||
*/
|
||||
public static final int DEFAULT_MAX_LINE_LENGTH = 4096;
|
||||
public static final int DEFAULT_MAX_HEADER_SIZE = 8192;
|
||||
public static final int DEFAULT_MAX_CHUNK_SIZE = 8192;
|
||||
|
||||
private final int port;
|
||||
private final int maxInitialLineLength;
|
||||
private final int maxHeaderSize;
|
||||
private final int maxChunkSize;
|
||||
private ActionController controller;
|
||||
|
||||
private Thread serverThread;
|
||||
|
||||
private AtomicBoolean isStarted = new AtomicBoolean(false);
|
||||
|
||||
public HttpServer(int port) {
|
||||
this(port, DEFAULT_MAX_LINE_LENGTH, DEFAULT_MAX_HEADER_SIZE, DEFAULT_MAX_CHUNK_SIZE);
|
||||
}
|
||||
|
||||
public HttpServer(int port, int maxInitialLineLength, int maxHeaderSize, int maxChunkSize) {
|
||||
this.port = port;
|
||||
this.maxInitialLineLength = maxInitialLineLength;
|
||||
this.maxHeaderSize = maxHeaderSize;
|
||||
this.maxChunkSize = maxChunkSize;
|
||||
controller = new ActionController();
|
||||
}
|
||||
|
||||
public void setup() throws IllegalArgException {
|
||||
registerActions();
|
||||
}
|
||||
|
||||
private void registerActions() throws IllegalArgException {
|
||||
// add rest action
|
||||
LoadAction.registerAction(controller);
|
||||
GetLoadInfoAction.registerAction(controller);
|
||||
SetConfigAction.registerAction(controller);
|
||||
GetDdlStmtAction.registerAction(controller);
|
||||
MigrationAction.registerAction(controller);
|
||||
StorageTypeCheckAction.registerAction(controller);
|
||||
CancelStreamLoad.registerAction(controller);
|
||||
GetStreamLoadState.registerAction(controller);
|
||||
|
||||
// add web action
|
||||
IndexAction.registerAction(controller);
|
||||
SystemAction.registerAction(controller);
|
||||
BackendAction.registerAction(controller);
|
||||
LogAction.registerAction(controller);
|
||||
QueryAction.registerAction(controller);
|
||||
QueryProfileAction.registerAction(controller);
|
||||
SessionAction.registerAction(controller);
|
||||
VariableAction.registerAction(controller);
|
||||
HelpAction.registerAction(controller);
|
||||
StaticResourceAction.registerAction(controller);
|
||||
HaAction.registerAction(controller);
|
||||
|
||||
// Add multi action
|
||||
MultiStart.registerAction(controller);
|
||||
MultiDesc.registerAction(controller);
|
||||
MultiCommit.registerAction(controller);
|
||||
MultiUnload.registerAction(controller);
|
||||
MultiAbort.registerAction(controller);
|
||||
MultiList.registerAction(controller);
|
||||
|
||||
// rest action
|
||||
HealthAction.registerAction(controller);
|
||||
MetricsAction.registerAction(controller);
|
||||
ShowMetaInfoAction.registerAction(controller);
|
||||
ShowProcAction.registerAction(controller);
|
||||
ShowRuntimeInfoAction.registerAction(controller);
|
||||
GetLogFileAction.registerAction(controller);
|
||||
GetSmallFileAction.registerAction(controller);
|
||||
RowCountAction.registerAction(controller);
|
||||
CheckDecommissionAction.registerAction(controller);
|
||||
MetaReplayerCheckAction.registerAction(controller);
|
||||
ColocateMetaService.BucketSeqAction.registerAction(controller);
|
||||
ColocateMetaService.ColocateMetaAction.registerAction(controller);
|
||||
ColocateMetaService.MarkGroupStableAction.registerAction(controller);
|
||||
ProfileAction.registerAction(controller);
|
||||
QueryDetailAction.registerAction(controller);
|
||||
ConnectionAction.registerAction(controller);
|
||||
ShowDataAction.registerAction(controller);
|
||||
|
||||
// meta service action
|
||||
File imageDir = MetaHelper.getMasterImageDir();
|
||||
ImageAction.registerAction(controller, imageDir);
|
||||
InfoAction.registerAction(controller, imageDir);
|
||||
VersionAction.registerAction(controller, imageDir);
|
||||
PutAction.registerAction(controller, imageDir);
|
||||
JournalIdAction.registerAction(controller, imageDir);
|
||||
CheckAction.registerAction(controller, imageDir);
|
||||
DumpAction.registerAction(controller, imageDir);
|
||||
RoleAction.registerAction(controller, imageDir);
|
||||
|
||||
// external usage
|
||||
TableRowCountAction.registerAction(controller);
|
||||
TableSchemaAction.registerAction(controller);
|
||||
TableQueryPlanAction.registerAction(controller);
|
||||
|
||||
BootstrapFinishAction.registerAction(controller);
|
||||
}
|
||||
|
||||
public void start() {
|
||||
serverThread = new Thread(new HttpServerThread(), "FE Http Server");
|
||||
serverThread.start();
|
||||
}
|
||||
|
||||
protected class PaloHttpServerInitializer extends ChannelInitializer<SocketChannel> {
|
||||
@Override
|
||||
protected void initChannel(SocketChannel ch) throws Exception {
|
||||
ch.pipeline().addLast(new HttpServerCodec(maxInitialLineLength, maxHeaderSize, maxChunkSize));
|
||||
ch.pipeline().addLast(new DorisHttpPostObjectAggregator(100 * 65536));
|
||||
ch.pipeline().addLast(new ChunkedWriteHandler());
|
||||
ch.pipeline().addLast(new HttpServerHandler(controller));
|
||||
}
|
||||
}
|
||||
|
||||
ServerBootstrap serverBootstrap;
|
||||
|
||||
private class HttpServerThread implements Runnable {
|
||||
@Override
|
||||
public void run() {
|
||||
// Configure the server.
|
||||
EventLoopGroup bossGroup = new NioEventLoopGroup();
|
||||
EventLoopGroup workerGroup = new NioEventLoopGroup();
|
||||
try {
|
||||
serverBootstrap = new ServerBootstrap();
|
||||
serverBootstrap.option(ChannelOption.SO_BACKLOG, Config.http_backlog_num);
|
||||
// reused address and port to avoid bind already exception
|
||||
serverBootstrap.option(ChannelOption.SO_REUSEADDR, true);
|
||||
serverBootstrap.childOption(ChannelOption.SO_REUSEADDR, true);
|
||||
serverBootstrap.group(bossGroup, workerGroup)
|
||||
.channel(NioServerSocketChannel.class)
|
||||
.childHandler(new PaloHttpServerInitializer());
|
||||
Channel ch = serverBootstrap.bind(port).sync().channel();
|
||||
|
||||
isStarted.set(true);
|
||||
LOG.info("HttpServer started with port {}", port);
|
||||
// block until server is closed
|
||||
ch.closeFuture().sync();
|
||||
} catch (Exception e) {
|
||||
LOG.error("Fail to start FE query http server[port: " + port + "] ", e);
|
||||
System.exit(-1);
|
||||
} finally {
|
||||
bossGroup.shutdownGracefully();
|
||||
workerGroup.shutdownGracefully();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// used for test, release bound port
|
||||
public void shutDown() {
|
||||
if (serverBootstrap != null) {
|
||||
Future future = serverBootstrap.config().group().shutdownGracefully(0, 1, TimeUnit.SECONDS).syncUninterruptibly();
|
||||
try {
|
||||
future.get();
|
||||
isStarted.set(false);
|
||||
LOG.info("HttpServer was closed completely");
|
||||
} catch (Throwable e) {
|
||||
LOG.warn("Exception happened when close HttpServer", e);
|
||||
}
|
||||
serverBootstrap = null;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isStarted() {
|
||||
return isStarted.get();
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
HttpServer httpServer = new HttpServer(8080);
|
||||
httpServer.setup();
|
||||
System.out.println("before start http server.");
|
||||
httpServer.start();
|
||||
System.out.println("after start http server.");
|
||||
|
||||
while (true) {
|
||||
Thread.sleep(2000);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,117 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http;
|
||||
|
||||
import org.apache.doris.http.action.IndexAction;
|
||||
import org.apache.doris.http.action.NotFoundAction;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.net.URISyntaxException;
|
||||
|
||||
import io.netty.buffer.Unpooled;
|
||||
import io.netty.channel.ChannelFutureListener;
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
import io.netty.channel.ChannelInboundHandlerAdapter;
|
||||
import io.netty.handler.codec.http.DefaultFullHttpResponse;
|
||||
import io.netty.handler.codec.http.FullHttpRequest;
|
||||
import io.netty.handler.codec.http.FullHttpResponse;
|
||||
import io.netty.handler.codec.http.HttpHeaderNames;
|
||||
import io.netty.handler.codec.http.HttpRequest;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
import io.netty.handler.codec.http.HttpVersion;
|
||||
import io.netty.util.ReferenceCountUtil;
|
||||
|
||||
public class HttpServerHandler extends ChannelInboundHandlerAdapter {
|
||||
private static final Logger LOG = LogManager.getLogger(HttpServerHandler.class);
|
||||
|
||||
private ActionController controller = null;
|
||||
protected FullHttpRequest fullRequest = null;
|
||||
protected HttpRequest request = null;
|
||||
private BaseAction action = null;
|
||||
|
||||
public HttpServerHandler(ActionController controller) {
|
||||
super();
|
||||
this.controller = controller;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void channelReadComplete(ChannelHandlerContext ctx) {
|
||||
ctx.flush();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
|
||||
if (msg instanceof HttpRequest) {
|
||||
this.request = (HttpRequest) msg;
|
||||
LOG.debug("request: url:[{}]", request.uri());
|
||||
if (!isRequestValid(ctx, request)) {
|
||||
writeResponse(ctx, HttpResponseStatus.BAD_REQUEST, "this is a bad request.");
|
||||
return;
|
||||
}
|
||||
BaseRequest req = new BaseRequest(ctx, request);
|
||||
|
||||
action = getAction(req);
|
||||
if (action != null) {
|
||||
LOG.debug("action: {} ", action.getClass().getName());
|
||||
action.handleRequest(req);
|
||||
}
|
||||
} else {
|
||||
ReferenceCountUtil.release(msg);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isRequestValid(ChannelHandlerContext ctx, HttpRequest request) throws URISyntaxException {
|
||||
return true;
|
||||
}
|
||||
|
||||
private void writeResponse(ChannelHandlerContext context , HttpResponseStatus status, String content) {
|
||||
FullHttpResponse responseObj = new DefaultFullHttpResponse(
|
||||
HttpVersion.HTTP_1_1,
|
||||
status,
|
||||
Unpooled.wrappedBuffer(content.getBytes()));
|
||||
responseObj.headers().set(HttpHeaderNames.CONTENT_TYPE.toString(), "text/html");
|
||||
responseObj.headers().set(HttpHeaderNames.CONTENT_LENGTH.toString(), responseObj.content().readableBytes());
|
||||
context.writeAndFlush(responseObj).addListener(ChannelFutureListener.CLOSE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
|
||||
LOG.warn("Handle http from " + ctx.channel().remoteAddress() + " caught exception", cause);
|
||||
ctx.close();
|
||||
}
|
||||
|
||||
private BaseAction getAction(BaseRequest request) {
|
||||
String uri = request.getRequest().uri();
|
||||
// ignore this request, which is a default request from client's browser.
|
||||
if (uri.endsWith("/favicon.ico")) {
|
||||
return NotFoundAction.getNotFoundAction();
|
||||
} else if (uri.equals("/")) {
|
||||
return new IndexAction(controller);
|
||||
}
|
||||
|
||||
// Map<String, String> params = Maps.newHashMap();
|
||||
BaseAction action = (BaseAction) controller.getHandler(request);
|
||||
if (action == null) {
|
||||
action = NotFoundAction.getNotFoundAction();
|
||||
}
|
||||
|
||||
return action;
|
||||
}
|
||||
}
|
||||
@ -1,25 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http;
|
||||
|
||||
|
||||
public interface IAction {
|
||||
|
||||
public void handleRequest(BaseRequest request) throws Exception;
|
||||
|
||||
}
|
||||
@ -1,26 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http;
|
||||
|
||||
import org.apache.doris.common.DdlException;
|
||||
|
||||
public class UnauthorizedException extends DdlException {
|
||||
public UnauthorizedException(String msg) {
|
||||
super(msg);
|
||||
}
|
||||
}
|
||||
@ -1,230 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http;
|
||||
|
||||
import org.apache.doris.common.path.PathTrie;
|
||||
|
||||
import com.google.common.base.Charsets;
|
||||
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.Map;
|
||||
|
||||
public class WebUtils {
|
||||
public static final String HTTP_CORS_ALLOW_ORIGIN_SETTING = "http.cors.allow-origin";
|
||||
|
||||
public static final PathTrie.Decoder REST_DECODER = new PathTrie.Decoder() {
|
||||
@Override
|
||||
public String decode(String value) {
|
||||
return WebUtils.decodeComponent(value);
|
||||
}
|
||||
};
|
||||
|
||||
public static boolean isBrowser(String userAgent) {
|
||||
if (userAgent == null) {
|
||||
return false;
|
||||
}
|
||||
// chrome, safari, firefox, ie
|
||||
if (userAgent.startsWith("Mozilla")) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public static void decodeQueryString(String s, int fromIndex, Map<String, String> params) {
|
||||
if (fromIndex < 0) {
|
||||
return;
|
||||
}
|
||||
if (fromIndex >= s.length()) {
|
||||
return;
|
||||
}
|
||||
|
||||
String name = null;
|
||||
int pos = fromIndex; // Beginning of the unprocessed region
|
||||
int i; // End of the unprocessed region
|
||||
char c = 0; // Current character
|
||||
for (i = fromIndex; i < s.length(); i++) {
|
||||
c = s.charAt(i);
|
||||
if (c == '=' && name == null) {
|
||||
if (pos != i) {
|
||||
name = decodeComponent(s.substring(pos, i));
|
||||
}
|
||||
pos = i + 1;
|
||||
} else if (c == '&') {
|
||||
if (name == null && pos != i) {
|
||||
// We haven't seen an `=' so far but moved forward.
|
||||
// Must be a param of the form '&a&' so add it with
|
||||
// an empty value.
|
||||
addParam(params, decodeComponent(s.substring(pos, i)), "");
|
||||
} else if (name != null) {
|
||||
addParam(params, name, decodeComponent(s.substring(pos, i)));
|
||||
name = null;
|
||||
}
|
||||
pos = i + 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (pos != i) { // Are there characters we haven't dealt with?
|
||||
if (name == null) { // Yes and we haven't seen any `='.
|
||||
addParam(params, decodeComponent(s.substring(pos, i)), "");
|
||||
} else { // Yes and this must be the last value.
|
||||
addParam(params, name, decodeComponent(s.substring(pos, i)));
|
||||
}
|
||||
} else if (name != null) { // Have we seen a name without value?
|
||||
addParam(params, name, "");
|
||||
}
|
||||
}
|
||||
|
||||
private static void addParam(Map<String, String> params, String name, String value) {
|
||||
params.put(name, value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes a bit of an URL encoded by a browser.
|
||||
* <p/>
|
||||
* This is equivalent to calling {@link #decodeComponent(String, Charset)}
|
||||
* with the UTF-8 charset (recommended to comply with RFC 3986, Section 2).
|
||||
*
|
||||
* @param s The string to decode (can be empty).
|
||||
* @return The decoded string, or {@code s} if there's nothing to decode.
|
||||
* If the string to decode is {@code null}, returns an empty string.
|
||||
* @throws IllegalArgumentException if the string contains a malformed
|
||||
* escape sequence.
|
||||
*/
|
||||
public static String decodeComponent(final String s) {
|
||||
return decodeComponent(s, Charsets.UTF_8);
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes a bit of an URL encoded by a browser.
|
||||
* <p/>
|
||||
* The string is expected to be encoded as per RFC 3986, Section 2.
|
||||
* This is the encoding used by JavaScript functions {@code encodeURI}
|
||||
* and {@code encodeURIComponent}, but not {@code escape}. For example
|
||||
* in this encoding, é (in Unicode {@code U+00E9} or in UTF-8
|
||||
* {@code 0xC3 0xA9}) is encoded as {@code %C3%A9} or {@code %c3%a9}.
|
||||
* <p/>
|
||||
* This is essentially equivalent to calling
|
||||
* <code>{@link java.net.URLDecoder URLDecoder}.{@link
|
||||
* java.net.URLDecoder#decode(String, String)}</code>
|
||||
* except that it's over 2x faster and generates less garbage for the GC.
|
||||
* Actually this function doesn't allocate any memory if there's nothing
|
||||
* to decode, the argument itself is returned.
|
||||
*
|
||||
* @param s The string to decode (can be empty).
|
||||
* @param charset The charset to use to decode the string (should really
|
||||
* be {@link Charsets#UTF_8}.
|
||||
* @return The decoded string, or {@code s} if there's nothing to decode.
|
||||
* If the string to decode is {@code null}, returns an empty string.
|
||||
* @throws IllegalArgumentException if the string contains a malformed
|
||||
* escape sequence.
|
||||
*/
|
||||
@SuppressWarnings("fallthrough")
|
||||
public static String decodeComponent(final String s, final Charset charset) {
|
||||
if (s == null) {
|
||||
return "";
|
||||
}
|
||||
final int size = s.length();
|
||||
boolean modified = false;
|
||||
for (int i = 0; i < size; i++) {
|
||||
final char c = s.charAt(i);
|
||||
switch (c) {
|
||||
case '%':
|
||||
i++; // We can skip at least one char, e.g. `%%'.
|
||||
// Fall through.
|
||||
case '+':
|
||||
modified = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!modified) {
|
||||
return s;
|
||||
}
|
||||
final byte[] buf = new byte[size];
|
||||
int pos = 0; // position in `buf'.
|
||||
for (int i = 0; i < size; i++) {
|
||||
char c = s.charAt(i);
|
||||
switch (c) {
|
||||
case '+':
|
||||
buf[pos++] = ' '; // "+" -> " "
|
||||
break;
|
||||
case '%':
|
||||
if (i == size - 1) {
|
||||
throw new IllegalArgumentException("unterminated escape"
|
||||
+ " sequence at end of string: " + s);
|
||||
}
|
||||
c = s.charAt(++i);
|
||||
if (c == '%') {
|
||||
buf[pos++] = '%'; // "%%" -> "%"
|
||||
break;
|
||||
} else if (i == size - 1) {
|
||||
throw new IllegalArgumentException("partial escape"
|
||||
+ " sequence at end of string: " + s);
|
||||
}
|
||||
c = decodeHexNibble(c);
|
||||
final char c2 = decodeHexNibble(s.charAt(++i));
|
||||
if (c == Character.MAX_VALUE || c2 == Character.MAX_VALUE) {
|
||||
throw new IllegalArgumentException(
|
||||
"invalid escape sequence `%" + s.charAt(i - 1)
|
||||
+ s.charAt(i) + "' at index " + (i - 2)
|
||||
+ " of: " + s);
|
||||
}
|
||||
c = (char) (c * 16 + c2);
|
||||
// Fall through.
|
||||
default:
|
||||
buf[pos++] = (byte) c;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return new String(buf, 0, pos, charset);
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to decode half of a hexadecimal number from a string.
|
||||
*
|
||||
* @param c The ASCII character of the hexadecimal number to decode.
|
||||
* Must be in the range {@code [0-9a-fA-F]}.
|
||||
* @return The hexadecimal value represented in the ASCII character
|
||||
* given, or {@link Character#MAX_VALUE} if the character is invalid.
|
||||
*/
|
||||
private static char decodeHexNibble(final char c) {
|
||||
if ('0' <= c && c <= '9') {
|
||||
return (char) (c - '0');
|
||||
} else if ('a' <= c && c <= 'f') {
|
||||
return (char) (c - 'a' + 10);
|
||||
} else if ('A' <= c && c <= 'F') {
|
||||
return (char) (c - 'A' + 10);
|
||||
} else {
|
||||
return Character.MAX_VALUE;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if CORS setting is a regex
|
||||
*/
|
||||
// public static Pattern getCorsSettingRegex(Settings settings) {
|
||||
// String corsSetting = settings.get(HTTP_CORS_ALLOW_ORIGIN_SETTING, "*");
|
||||
// int len = corsSetting.length();
|
||||
// boolean isRegex = len > 2 && corsSetting.startsWith("/") && corsSetting.endsWith("/");
|
||||
//
|
||||
// if (isRegex) {
|
||||
// return Pattern.compile(corsSetting.substring(1, corsSetting.length()-1));
|
||||
// }
|
||||
//
|
||||
// return null;
|
||||
// }
|
||||
}
|
||||
@ -1,101 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.action;
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.common.util.ListComparator;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.system.Backend;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
public class BackendAction extends WebBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(BackendAction.class);
|
||||
|
||||
public BackendAction(ActionController controller) {
|
||||
super(controller);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/backend", new BackendAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
getPageHeader(request, response.getContent());
|
||||
|
||||
appendKnownBackendsInfo(response.getContent());
|
||||
|
||||
getPageFooter(response.getContent());
|
||||
writeResponse(request, response);
|
||||
}
|
||||
|
||||
private void appendKnownBackendsInfo(StringBuilder buffer) {
|
||||
ImmutableMap<Long, Backend> backendMap = Catalog.getCurrentSystemInfo().getIdToBackend();
|
||||
|
||||
List<List<Comparable>> backendInfos = new ArrayList<List<Comparable>>();
|
||||
for (Backend backend : backendMap.values()) {
|
||||
List<Comparable> backendInfo = new ArrayList<Comparable>();
|
||||
InetAddress address = null;
|
||||
try {
|
||||
address = InetAddress.getByName(backend.getHost());
|
||||
} catch (UnknownHostException e) {
|
||||
LOG.warn("unknown host: " + backend.getHost(), e);
|
||||
continue;
|
||||
}
|
||||
backendInfo.add(address.getHostName());
|
||||
backendInfo.add(backend.getId());
|
||||
backendInfo.add("heart_port: " + backend.getHeartbeatPort()
|
||||
+ ", be_port: " + backend.getBePort()
|
||||
+ ", http_port: " + backend.getHttpPort());
|
||||
|
||||
backendInfos.add(backendInfo);
|
||||
}
|
||||
|
||||
// sort by id
|
||||
ListComparator<List<Comparable>> comparator = new ListComparator<List<Comparable>>(1);
|
||||
Collections.sort(backendInfos, comparator);
|
||||
|
||||
// set result
|
||||
buffer.append("<h2>Known Backends(" + backendMap.size() + ")</h2>");
|
||||
|
||||
buffer.append("<pre>");
|
||||
for (List<Comparable> info : backendInfos) {
|
||||
buffer.append(info.get(0));
|
||||
buffer.append(" [id: " + info.get(1));
|
||||
buffer.append(", " + info.get(2) + "]");
|
||||
buffer.append(System.getProperty("line.separator"));
|
||||
}
|
||||
buffer.append("</pre>");
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,174 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.action;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.ha.HAProtocol;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.persist.Storage;
|
||||
import org.apache.doris.system.Frontend;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
public class HaAction extends WebBaseAction {
|
||||
|
||||
public HaAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/ha", new HaAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
getPageHeader(request, response.getContent());
|
||||
|
||||
appendRoleInfo(response.getContent());
|
||||
appendJournalInfo(response.getContent());
|
||||
appendCanReadInfo(response.getContent());
|
||||
appendNodesInfo(response.getContent());
|
||||
appendImageInfo(response.getContent());
|
||||
appendDbNames(response.getContent());
|
||||
appendFe(response.getContent());
|
||||
appendRemovedFe(response.getContent());
|
||||
|
||||
getPageFooter(response.getContent());
|
||||
writeResponse(request, response);
|
||||
}
|
||||
|
||||
private void appendRoleInfo(StringBuilder buffer) {
|
||||
buffer.append("<h2>Frontend Role</h2>");
|
||||
buffer.append("<pre>");
|
||||
buffer.append("<p>" + Catalog.getCurrentCatalog().getFeType() + "</p>");
|
||||
buffer.append("</pre>");
|
||||
}
|
||||
|
||||
private void appendJournalInfo(StringBuilder buffer) {
|
||||
buffer.append("<h2>Current Journal Id</h2>");
|
||||
buffer.append("<pre>");
|
||||
if (Catalog.getCurrentCatalog().isMaster()) {
|
||||
buffer.append("<p>" + Catalog.getCurrentCatalog().getEditLog().getMaxJournalId() + "</p>");
|
||||
} else {
|
||||
buffer.append("<p>" + Catalog.getCurrentCatalog().getReplayedJournalId() + "</p>");
|
||||
}
|
||||
buffer.append("</pre>");
|
||||
}
|
||||
|
||||
private void appendNodesInfo(StringBuilder buffer) {
|
||||
HAProtocol haProtocol = Catalog.getCurrentCatalog().getHaProtocol();
|
||||
if (haProtocol == null) {
|
||||
return;
|
||||
}
|
||||
List<InetSocketAddress> electableNodes = haProtocol.getElectableNodes(true);
|
||||
if (electableNodes.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
buffer.append("<h2>Electable nodes</h2>");
|
||||
buffer.append("<pre>");
|
||||
for (InetSocketAddress node : electableNodes) {
|
||||
buffer.append("<p>" + node.getAddress() + "</p>");
|
||||
|
||||
}
|
||||
buffer.append("</pre>");
|
||||
|
||||
List<InetSocketAddress> observerNodes = haProtocol.getObserverNodes();
|
||||
if (observerNodes == null) {
|
||||
return;
|
||||
}
|
||||
buffer.append("<h2>Observer nodes</h2>");
|
||||
buffer.append("<pre>");
|
||||
for (InetSocketAddress node : observerNodes) {
|
||||
buffer.append("<p>" + node.getHostString() + "</p>");
|
||||
}
|
||||
buffer.append("</pre>");
|
||||
}
|
||||
|
||||
private void appendCanReadInfo(StringBuilder buffer) {
|
||||
buffer.append("<h2>Can Read</h2>");
|
||||
buffer.append("<pre>");
|
||||
buffer.append("<p>" + Catalog.getCurrentCatalog().canRead() + "</p>");
|
||||
|
||||
buffer.append("</pre>");
|
||||
}
|
||||
|
||||
private void appendImageInfo(StringBuilder buffer) {
|
||||
try {
|
||||
Storage storage = new Storage(Config.meta_dir + "/image");
|
||||
buffer.append("<h2>Checkpoint Info</h2>");
|
||||
buffer.append("<pre>");
|
||||
buffer.append("<p>last checkpoint version:" + storage.getImageSeq() + "</p>");
|
||||
long lastCheckpointTime = storage.getCurrentImageFile().lastModified();
|
||||
Date date = new Date(lastCheckpointTime);
|
||||
buffer.append("<p>last checkpoint time: " + date + "</p>");
|
||||
buffer.append("</pre>");
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
private void appendDbNames(StringBuilder buffer) {
|
||||
List<Long> names = Catalog.getCurrentCatalog().getEditLog().getDatabaseNames();
|
||||
if (names == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
String msg = "";
|
||||
for (long name : names) {
|
||||
msg += name + " ";
|
||||
}
|
||||
buffer.append("<h2>Database names</h2>");
|
||||
buffer.append("<pre>");
|
||||
buffer.append("<p>" + msg + "</p>");
|
||||
buffer.append("</pre>");
|
||||
}
|
||||
|
||||
private void appendFe(StringBuilder buffer) {
|
||||
List<Frontend> fes = Catalog.getCurrentCatalog().getFrontends(null /* all */);
|
||||
if (fes == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
buffer.append("<h2>Allowed Frontends</h2>");
|
||||
buffer.append("<pre>");
|
||||
for (Frontend fe : fes) {
|
||||
buffer.append("<p>" + fe.toString() + "</p>");
|
||||
}
|
||||
buffer.append("</pre>");
|
||||
}
|
||||
|
||||
private void appendRemovedFe(StringBuilder buffer) {
|
||||
List<String> feNames = Catalog.getCurrentCatalog().getRemovedFrontendNames();
|
||||
buffer.append("<h2>Removed Frontends</h2>");
|
||||
buffer.append("<pre>");
|
||||
for (String feName : feNames) {
|
||||
buffer.append("<p>" + feName + "</p>");
|
||||
}
|
||||
buffer.append("</pre>");
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,223 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.action;
|
||||
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.qe.HelpModule;
|
||||
import org.apache.doris.qe.HelpTopic;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
public class HelpAction extends WebBaseAction {
|
||||
private static final String DIV_BACKGROUND_COLOR = "#FCFCFC";
|
||||
|
||||
String queryString = null;
|
||||
|
||||
public HelpAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/help", new HelpAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean needAdmin() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
getPageHeader(request, response.getContent());
|
||||
appendHelpStyle(response.getContent());
|
||||
|
||||
queryString = request.getSingleParameter("query");
|
||||
if (Strings.isNullOrEmpty(queryString)) {
|
||||
// ATTN: according to Mysql protocol, the default query should be "contents"
|
||||
// when you want to get server side help.
|
||||
queryString = "contents";
|
||||
} else {
|
||||
queryString = queryString.trim();
|
||||
}
|
||||
appendHelpInfo(response.getContent());
|
||||
|
||||
getPageFooter(response.getContent());
|
||||
writeResponse(request, response);
|
||||
}
|
||||
|
||||
private void appendHelpInfo(StringBuilder buffer) {
|
||||
buffer.append("<h2>Help Info</h2>");
|
||||
buffer.append("<p>This page lists the help info, "
|
||||
+ "like 'help contents' in Mysql client.</p>");
|
||||
|
||||
appendSearchButton(buffer);
|
||||
appendExactMatchTopic(buffer);
|
||||
appendFuzzyMatchTopic(buffer);
|
||||
appendCategories(buffer);
|
||||
}
|
||||
|
||||
private void appendSearchButton(StringBuilder buffer) {
|
||||
buffer.append("<form class=\"form-search\">"
|
||||
+ "<div class=\"col-lg-3\" style=\"padding-left: 0px;\">"
|
||||
+ " <div class=\"input-group\">"
|
||||
+ " <input name = \"query\" type=\"text\" class=\"form-control\" placeholder=\"input here...\">"
|
||||
+ " <span class=\"input-group-btn\">"
|
||||
+ " <button class=\"btn btn-default\" type=\"submit\">Search</button>"
|
||||
+ " </span>"
|
||||
+ " </div>"
|
||||
+ "</div>"
|
||||
+ "<a href=\"/help\" class=\"btn btn-primary\">Back To Home</a>"
|
||||
+ "</form>");
|
||||
}
|
||||
|
||||
private void appendExactMatchTopic(StringBuilder buffer) {
|
||||
buffer.append("<h3>Exact Matching Topic</h3>");
|
||||
buffer.append("<div style=\"background-color:" + DIV_BACKGROUND_COLOR + ";"
|
||||
+ "padding:0px, 1px, 1px, 0px;"
|
||||
+ "\">");
|
||||
HelpModule module = HelpModule.getInstance();
|
||||
HelpTopic topic = module.getTopic(queryString);
|
||||
if (topic == null) {
|
||||
buffer.append("<pre>No Exact Matching Topic.</pre>");
|
||||
} else {
|
||||
appendOneTopicInfo(buffer, topic);
|
||||
}
|
||||
buffer.append("</div>");
|
||||
}
|
||||
|
||||
private void appendFuzzyMatchTopic(StringBuilder buffer) {
|
||||
buffer.append("<h3>Fuzzy Matching Topic(By Keyword)</h3>");
|
||||
buffer.append("<div style=\"background-color:" + DIV_BACKGROUND_COLOR + ";"
|
||||
+ "padding:0px, 1px, 1px, 0px;"
|
||||
+ "\">");
|
||||
HelpModule module = HelpModule.getInstance();
|
||||
List<String> topics = module.listTopicByKeyword(queryString);
|
||||
if (topics.isEmpty()) {
|
||||
buffer.append("<pre>No Fuzzy Matching Topic.</pre>");
|
||||
} else if (topics.size() == 1) {
|
||||
buffer.append("<p class=\"text-info\"> "
|
||||
+ "Find only one topic, show you the detail info below.</p>");
|
||||
appendOneTopicInfo(buffer, module.getTopic(topics.get(0)));
|
||||
} else {
|
||||
buffer.append("<p class=\"text-info\"> Find " + topics.size() + " topics:</p>");
|
||||
appendNameList(buffer, topics, "Topics");
|
||||
}
|
||||
buffer.append("</div>");
|
||||
}
|
||||
|
||||
private void appendCategories(StringBuilder buffer) {
|
||||
buffer.append("<h3>Category Info</h3>");
|
||||
buffer.append("<div style=\"background-color:" + DIV_BACKGROUND_COLOR + ";"
|
||||
+ "padding:0px, 1px, 1px, 0px;"
|
||||
+ "\">");
|
||||
HelpModule module = HelpModule.getInstance();
|
||||
List<String> categories = module.listCategoryByName(queryString);
|
||||
if (categories.isEmpty()) {
|
||||
buffer.append("<pre>No Matching Category.</pre>");
|
||||
} else if (categories.size() == 1) {
|
||||
buffer.append("<p class=\"text-info\"> "
|
||||
+ "Find only one category, so show you the detail info below. </p>");
|
||||
List<String> topics = module.listTopicByCategory(categories.get(0));
|
||||
if (topics.size() > 0) {
|
||||
buffer.append("<p class=\"text-info\"> Find "
|
||||
+ topics.size()
|
||||
+ " sub topics. </p>");
|
||||
appendNameList(buffer, topics, "Sub Topics");
|
||||
}
|
||||
|
||||
List<String> subCategories = module.listCategoryByCategory(categories.get(0));
|
||||
if (subCategories.size() > 0) {
|
||||
buffer.append("<p class=\"text-info\"> Find "
|
||||
+ subCategories.size()
|
||||
+ " sub categories. </p>");
|
||||
appendNameList(buffer, subCategories, "Sub Categories");
|
||||
}
|
||||
} else {
|
||||
buffer.append("<p> Find " + categories.size() + " category: </p>");
|
||||
appendNameList(buffer, categories, "Categories");
|
||||
}
|
||||
buffer.append("</div>");
|
||||
}
|
||||
|
||||
// The browser will combine continuous whitespace to one, we use <pre> tag to solve this issue.
|
||||
private void appendOneTopicInfo(StringBuilder buffer, HelpTopic topic) {
|
||||
buffer.append("<div style=\""
|
||||
+ "padding:9.5px; "
|
||||
+ "margin: 0 0 10px; "
|
||||
+ "background-color: #f5f5f5;"
|
||||
+ "border: 1px solid rgba(0, 0, 0, 0.15);"
|
||||
+ "-webkit-border-radius: 4px;"
|
||||
+ "-moz-border-radius: 4px;"
|
||||
+ "border-radius: 4px;"
|
||||
+ "font-family: Consolas;"
|
||||
+ "\">");
|
||||
buffer.append("<h4>'" + escapeHtmlInPreTag(topic.getName()) + "'</h4>");
|
||||
|
||||
buffer.append("<strong>Description</strong>");
|
||||
buffer.append("<pre class=\"topic_text\" style=\"border: 0px;\">"
|
||||
+ escapeHtmlInPreTag(topic.getDescription())
|
||||
+ "</pre>");
|
||||
|
||||
buffer.append("<strong>Example</strong>");
|
||||
buffer.append("<pre class=\"topic_text\" style=\"border: 0px\">"
|
||||
+ escapeHtmlInPreTag(topic.getExample())
|
||||
+ "</pre>");
|
||||
|
||||
buffer.append("<strong>Keyword</strong>");
|
||||
buffer.append("<pre class=\"topic_text\" style=\"border: 0px\">"
|
||||
+ escapeHtmlInPreTag(topic.getKeywords().toString())
|
||||
+ "</pre>");
|
||||
|
||||
buffer.append("<strong>Url</strong>");
|
||||
buffer.append("<pre class=\"topic_text\" style=\"border: 0px\">"
|
||||
+ escapeHtmlInPreTag(topic.getUrl())
|
||||
+ "</pre>");
|
||||
buffer.append("</div>");
|
||||
}
|
||||
|
||||
private void appendNameList(StringBuilder buffer, List<String> names, String tableHeadName) {
|
||||
buffer.append("<div style=\"padding:0px, 1px, 1px, 0px\">");
|
||||
buffer.append("<table "
|
||||
+ "class=\"table table-hover table-bordered table-striped table-hover\"><tr>");
|
||||
buffer.append("<th>" + tableHeadName + "</th>");
|
||||
|
||||
final String href = "?query=";
|
||||
for (String name : names) {
|
||||
buffer.append("<tr><td><a href=\"" + href + name + "\">" + name + "</a><br/></td></tr>");
|
||||
}
|
||||
buffer.append("</table>");
|
||||
buffer.append("</div>");
|
||||
}
|
||||
|
||||
private void appendHelpStyle(StringBuilder buffer) {
|
||||
buffer.append("<style type=\"text/css\">"
|
||||
+ ".topic_text {"
|
||||
+ " font-family: \"Consolas\";"
|
||||
+ " margin-left: 5px;"
|
||||
+ "}"
|
||||
+ "</style>");
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,275 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.action;
|
||||
|
||||
import org.apache.doris.common.Version;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import oshi.SystemInfo;
|
||||
import oshi.hardware.CentralProcessor;
|
||||
import oshi.hardware.GlobalMemory;
|
||||
import oshi.hardware.HWDiskStore;
|
||||
import oshi.hardware.HWPartition;
|
||||
import oshi.hardware.HardwareAbstractionLayer;
|
||||
import oshi.hardware.NetworkIF;
|
||||
import oshi.hardware.VirtualMemory;
|
||||
import oshi.software.os.FileSystem;
|
||||
import oshi.software.os.NetworkParams;
|
||||
import oshi.software.os.OSFileStore;
|
||||
import oshi.software.os.OSProcess;
|
||||
import oshi.software.os.OperatingSystem;
|
||||
import oshi.util.FormatUtil;
|
||||
import oshi.util.Util;
|
||||
|
||||
public class IndexAction extends WebBaseAction {
|
||||
|
||||
public IndexAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/index", new IndexAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
getPageHeader(request, response.getContent());
|
||||
appendVersionInfo(response.getContent());
|
||||
appendHardwareInfo(response.getContent());
|
||||
getPageFooter(response.getContent());
|
||||
writeResponse(request, response);
|
||||
}
|
||||
|
||||
private void appendVersionInfo(StringBuilder buffer) {
|
||||
buffer.append("<h2>Version</h2>");
|
||||
buffer.append("<pre>version info<br/>");
|
||||
buffer.append("Version: " + Version.DORIS_BUILD_VERSION + "<br/>");
|
||||
buffer.append("Git: " + Version.DORIS_BUILD_HASH + "<br/>");
|
||||
buffer.append("Build Info: " + Version.DORIS_BUILD_INFO + "<br/>");
|
||||
buffer.append("Build Time: " + Version.DORIS_BUILD_TIME + "<br/>");
|
||||
buffer.append("</pre>");
|
||||
}
|
||||
|
||||
private void appendHardwareInfo(StringBuilder buffer) {
|
||||
List<String> hardwareInfo = new ArrayList<>();
|
||||
SystemInfo si = new SystemInfo();
|
||||
OperatingSystem os = si.getOperatingSystem();
|
||||
HardwareAbstractionLayer hal = si.getHardware();
|
||||
CentralProcessor processor = hal.getProcessor();
|
||||
GlobalMemory memory = hal.getMemory();
|
||||
buffer.append("<h2>Hardware Info</h2>");
|
||||
buffer.append("<pre>");
|
||||
buffer.append(String.join("<br/>", getOperatingSystem(os)) + "<hr>");
|
||||
buffer.append(String.join("<br/>", getProcessor(processor)) + "<hr>");
|
||||
buffer.append(String.join("<br/>", getMemory(memory)) + "<hr>");
|
||||
buffer.append(String.join("<br/>", getProcesses(os, memory)) + "<hr>");
|
||||
buffer.append(String.join("<br/>", getDisks(hal.getDiskStores())) + "<hr>");
|
||||
buffer.append(String.join("<br/>", getFileSystem(os.getFileSystem())) + "<hr>");
|
||||
buffer.append(String.join("<br/>", getNetworkInterfaces(hal.getNetworkIFs())) + "<hr>");
|
||||
buffer.append(String.join("<br/>", getNetworkParameters(os.getNetworkParams())) + "<hr>");
|
||||
buffer.append("</pre>");
|
||||
}
|
||||
private List<String> getOperatingSystem(OperatingSystem os) {
|
||||
List<String> osInfo = new ArrayList<>();
|
||||
osInfo.add(String.valueOf(os));
|
||||
osInfo.add("Booted: " + Instant.ofEpochSecond(os.getSystemBootTime()));
|
||||
osInfo.add("Uptime: " + FormatUtil.formatElapsedSecs(os.getSystemUptime()));
|
||||
osInfo.add("Running with" + (os.isElevated() ? "" : "out") + " elevated permissions.");
|
||||
return osInfo;
|
||||
}
|
||||
private List<String> getProcessor(CentralProcessor processor) {
|
||||
List<String> processorInfo = new ArrayList<>();
|
||||
processorInfo.add(String.valueOf(processor));
|
||||
processorInfo.add(" " + processor.getPhysicalPackageCount() + " physical CPU package(s)");
|
||||
processorInfo.add(" " + processor.getPhysicalProcessorCount() + " physical CPU core(s)");
|
||||
processorInfo.add(" " + processor.getLogicalProcessorCount() + " logical CPU(s)");
|
||||
|
||||
processorInfo.add("Identifier: " + processor.getIdentifier());
|
||||
processorInfo.add("ProcessorID: " + processor.getProcessorID());
|
||||
processorInfo.add("Context Switches/Interrupts: " + processor.getContextSwitches()
|
||||
+ " / " + processor.getInterrupts() + "\n");
|
||||
|
||||
long[] prevTicks = processor.getSystemCpuLoadTicks();
|
||||
long[][] prevProcTicks = processor.getProcessorCpuLoadTicks();
|
||||
processorInfo.add("CPU, IOWait, and IRQ ticks @ 0 sec:" + Arrays.toString(prevTicks));
|
||||
// Wait a second...
|
||||
Util.sleep(1000);
|
||||
long[] ticks = processor.getSystemCpuLoadTicks();
|
||||
processorInfo.add("CPU, IOWait, and IRQ ticks @ 1 sec:" + Arrays.toString(ticks));
|
||||
long user = ticks[CentralProcessor.TickType.USER.getIndex()] - prevTicks[CentralProcessor.TickType.USER.getIndex()];
|
||||
long nice = ticks[CentralProcessor.TickType.NICE.getIndex()] - prevTicks[CentralProcessor.TickType.NICE.getIndex()];
|
||||
long sys = ticks[CentralProcessor.TickType.SYSTEM.getIndex()] - prevTicks[CentralProcessor.TickType.SYSTEM.getIndex()];
|
||||
long idle = ticks[CentralProcessor.TickType.IDLE.getIndex()] - prevTicks[CentralProcessor.TickType.IDLE.getIndex()];
|
||||
long iowait = ticks[CentralProcessor.TickType.IOWAIT.getIndex()] - prevTicks[CentralProcessor.TickType.IOWAIT.getIndex()];
|
||||
long irq = ticks[CentralProcessor.TickType.IRQ.getIndex()] - prevTicks[CentralProcessor.TickType.IRQ.getIndex()];
|
||||
long softirq = ticks[CentralProcessor.TickType.SOFTIRQ.getIndex()] - prevTicks[CentralProcessor.TickType.SOFTIRQ.getIndex()];
|
||||
long steal = ticks[CentralProcessor.TickType.STEAL.getIndex()] - prevTicks[CentralProcessor.TickType.STEAL.getIndex()];
|
||||
long totalCpu = user + nice + sys + idle + iowait + irq + softirq + steal;
|
||||
|
||||
processorInfo.add(String.format(
|
||||
"User: %.1f%% Nice: %.1f%% System: %.1f%% Idle: %.1f%% IOwait: %.1f%% IRQ: %.1f%% SoftIRQ: %.1f%% Steal: %.1f%%",
|
||||
100d * user / totalCpu, 100d * nice / totalCpu, 100d * sys / totalCpu, 100d * idle / totalCpu,
|
||||
100d * iowait / totalCpu, 100d * irq / totalCpu, 100d * softirq / totalCpu, 100d * steal / totalCpu));
|
||||
processorInfo.add(String.format("CPU load: %.1f%%",
|
||||
processor.getSystemCpuLoadBetweenTicks(prevTicks) * 100));
|
||||
double[] loadAverage = processor.getSystemLoadAverage(3);
|
||||
processorInfo.add("CPU load averages:" + (loadAverage[0] < 0 ? " N/A" : String.format(" %.2f", loadAverage[0]))
|
||||
+ (loadAverage[1] < 0 ? " N/A" : String.format(" %.2f", loadAverage[1]))
|
||||
+ (loadAverage[2] < 0 ? " N/A" : String.format(" %.2f", loadAverage[2])));
|
||||
// per core CPU
|
||||
StringBuilder procCpu = new StringBuilder("CPU load per processor:");
|
||||
double[] load = processor.getProcessorCpuLoadBetweenTicks(prevProcTicks);
|
||||
for (double avg : load) {
|
||||
procCpu.append(String.format(" %.1f%%", avg * 100));
|
||||
}
|
||||
processorInfo.add(procCpu.toString());
|
||||
long freq = processor.getVendorFreq();
|
||||
if (freq > 0) {
|
||||
processorInfo.add("Vendor Frequency: " + FormatUtil.formatHertz(freq));
|
||||
}
|
||||
freq = processor.getMaxFreq();
|
||||
if (freq > 0) {
|
||||
processorInfo.add("Max Frequency: " + FormatUtil.formatHertz(freq));
|
||||
}
|
||||
long[] freqs = processor.getCurrentFreq();
|
||||
if (freqs[0] > 0) {
|
||||
StringBuilder sb = new StringBuilder("Current Frequencies: ");
|
||||
for (int i = 0; i < freqs.length; i++) {
|
||||
if (i > 0) {
|
||||
sb.append(", ");
|
||||
}
|
||||
sb.append(FormatUtil.formatHertz(freqs[i]));
|
||||
}
|
||||
processorInfo.add(sb.toString());
|
||||
}
|
||||
return processorInfo;
|
||||
}
|
||||
|
||||
private List<String> getMemory(GlobalMemory memory) {
|
||||
List<String> memoryInfo = new ArrayList<>();
|
||||
memoryInfo.add("Memory: " + FormatUtil.formatBytes(memory.getAvailable()) + "/"
|
||||
+ FormatUtil.formatBytes(memory.getTotal()));
|
||||
VirtualMemory vm = memory.getVirtualMemory();
|
||||
memoryInfo.add("Swap used: " + FormatUtil.formatBytes(vm.getSwapUsed()) + "/"
|
||||
+ FormatUtil.formatBytes(vm.getSwapTotal()));
|
||||
return memoryInfo;
|
||||
}
|
||||
|
||||
private List<String> getProcesses(OperatingSystem os, GlobalMemory memory) {
|
||||
List<String> processInfo = new ArrayList<>();
|
||||
processInfo.add("Processes: " + os.getProcessCount() + ", Threads: " + os.getThreadCount());
|
||||
// Sort by highest CPU
|
||||
List<OSProcess> procs = Arrays.asList(os.getProcesses(5, OperatingSystem.ProcessSort.CPU));
|
||||
|
||||
processInfo.add(" PID %CPU %MEM VSZ RSS Name");
|
||||
for (int i = 0; i < procs.size() && i < 5; i++) {
|
||||
OSProcess p = procs.get(i);
|
||||
processInfo.add(String.format(" %5d %5.1f %4.1f %9s %9s %s", p.getProcessID(),
|
||||
100d * (p.getKernelTime() + p.getUserTime()) / p.getUpTime(),
|
||||
100d * p.getResidentSetSize() / memory.getTotal(), FormatUtil.formatBytes(p.getVirtualSize()),
|
||||
FormatUtil.formatBytes(p.getResidentSetSize()), p.getName()));
|
||||
}
|
||||
return processInfo;
|
||||
}
|
||||
private List<String> getDisks(HWDiskStore[] diskStores) {
|
||||
List<String> diskInfo = new ArrayList<>();
|
||||
diskInfo.add("Disks:");
|
||||
for (HWDiskStore disk : diskStores) {
|
||||
boolean readwrite = disk.getReads() > 0 || disk.getWrites() > 0;
|
||||
diskInfo.add(String.format(" %s: (model: %s - S/N: %s) size: %s, reads: %s (%s), writes: %s (%s), xfer: %s ms",
|
||||
disk.getName(), disk.getModel(), disk.getSerial(),
|
||||
disk.getSize() > 0 ? FormatUtil.formatBytesDecimal(disk.getSize()) : "?",
|
||||
readwrite ? disk.getReads() : "?", readwrite ? FormatUtil.formatBytes(disk.getReadBytes()) : "?",
|
||||
readwrite ? disk.getWrites() : "?", readwrite ? FormatUtil.formatBytes(disk.getWriteBytes()) : "?",
|
||||
readwrite ? disk.getTransferTime() : "?"));
|
||||
HWPartition[] partitions = disk.getPartitions();
|
||||
for (HWPartition part : partitions) {
|
||||
diskInfo.add(String.format(" |-- %s: %s (%s) Maj:Min=%d:%d, size: %s%s", part.getIdentification(),
|
||||
part.getName(), part.getType(), part.getMajor(), part.getMinor(),
|
||||
FormatUtil.formatBytesDecimal(part.getSize()),
|
||||
part.getMountPoint().isEmpty() ? "" : " @ " + part.getMountPoint()));
|
||||
}
|
||||
}
|
||||
return diskInfo;
|
||||
}
|
||||
|
||||
private List<String> getFileSystem(FileSystem fileSystem) {
|
||||
List<String> fsInfo = new ArrayList<>();
|
||||
fsInfo.add("File System:");
|
||||
|
||||
fsInfo.add(String.format(" File Descriptors: %d/%d", fileSystem.getOpenFileDescriptors(),
|
||||
fileSystem.getMaxFileDescriptors()));
|
||||
|
||||
OSFileStore[] fsArray = fileSystem.getFileStores();
|
||||
for (OSFileStore fs : fsArray) {
|
||||
long usable = fs.getUsableSpace();
|
||||
long total = fs.getTotalSpace();
|
||||
fsInfo.add(String.format(" %s (%s) [%s] %s of %s free (%.1f%%), %s of %s files free (%.1f%%) is %s " +
|
||||
(fs.getLogicalVolume() != null && fs.getLogicalVolume().length() > 0 ? "[%s]" : "%s") +
|
||||
" and is mounted at %s",
|
||||
fs.getName(), fs.getDescription().isEmpty() ? "file system" : fs.getDescription(), fs.getType(),
|
||||
FormatUtil.formatBytes(usable), FormatUtil.formatBytes(fs.getTotalSpace()), 100d * usable / total,
|
||||
FormatUtil.formatValue(fs.getFreeInodes(), ""), FormatUtil.formatValue(fs.getTotalInodes(), ""),
|
||||
100d * fs.getFreeInodes() / fs.getTotalInodes(), fs.getVolume(), fs.getLogicalVolume(),
|
||||
fs.getMount()));
|
||||
}
|
||||
return fsInfo;
|
||||
}
|
||||
|
||||
private List<String> getNetworkInterfaces(NetworkIF[] networkIFs) {
|
||||
List<String> getNetwork = new ArrayList<>();
|
||||
getNetwork.add("Network interfaces:");
|
||||
for (NetworkIF net : networkIFs) {
|
||||
getNetwork.add(String.format(" Name: %s (%s)", net.getName(), net.getDisplayName()));
|
||||
getNetwork.add(String.format(" MAC Address: %s", net.getMacaddr()));
|
||||
getNetwork.add(String.format(" MTU: %s, Speed: %s", net.getMTU(), FormatUtil.formatValue(net.getSpeed(), "bps")));
|
||||
getNetwork.add(String.format(" IPv4: %s", Arrays.toString(net.getIPv4addr())));
|
||||
getNetwork.add(String.format(" IPv6: %s", Arrays.toString(net.getIPv6addr())));
|
||||
boolean hasData = net.getBytesRecv() > 0 || net.getBytesSent() > 0 || net.getPacketsRecv() > 0
|
||||
|| net.getPacketsSent() > 0;
|
||||
getNetwork.add(String.format(" Traffic: received %s/%s%s; transmitted %s/%s%s",
|
||||
hasData ? net.getPacketsRecv() + " packets" : "?",
|
||||
hasData ? FormatUtil.formatBytes(net.getBytesRecv()) : "?",
|
||||
hasData ? " (" + net.getInErrors() + " err)" : "",
|
||||
hasData ? net.getPacketsSent() + " packets" : "?",
|
||||
hasData ? FormatUtil.formatBytes(net.getBytesSent()) : "?",
|
||||
hasData ? " (" + net.getOutErrors() + " err)" : ""));
|
||||
}
|
||||
return getNetwork;
|
||||
}
|
||||
|
||||
private List<String> getNetworkParameters(NetworkParams networkParams) {
|
||||
List<String> networkParameterInfo = new ArrayList<>();
|
||||
networkParameterInfo.add("Network parameters:");
|
||||
networkParameterInfo.add(String.format(" Host name: %s", networkParams.getHostName()));
|
||||
networkParameterInfo.add(String.format(" Domain name: %s", networkParams.getDomainName()));
|
||||
networkParameterInfo.add(String.format(" DNS servers: %s", Arrays.toString(networkParams.getDnsServers())));
|
||||
networkParameterInfo.add(String.format(" IPv4 Gateway: %s", networkParams.getIpv4DefaultGateway()));
|
||||
networkParameterInfo.add(String.format(" IPv6 Gateway: %s", networkParams.getIpv6DefaultGateway()));
|
||||
return networkParameterInfo;
|
||||
}
|
||||
}
|
||||
@ -1,168 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.action;
|
||||
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.common.Log4jConfig;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.Lists;
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.util.List;
|
||||
|
||||
public class LogAction extends WebBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(LogAction.class);
|
||||
private static long WEB_LOG_BYTES = 1024 * 1024; // 1MB
|
||||
|
||||
private String addVerboseName;
|
||||
private String delVerboseName;
|
||||
|
||||
public LogAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/log", new LogAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
getPageHeader(request, response.getContent());
|
||||
|
||||
// get parameters
|
||||
addVerboseName = request.getSingleParameter("add_verbose");
|
||||
delVerboseName = request.getSingleParameter("del_verbose");
|
||||
LOG.info("add verbose name: {}, del verbose name: {}", addVerboseName, delVerboseName);
|
||||
|
||||
appendLogConf(response.getContent());
|
||||
appendLogInfo(response.getContent());
|
||||
|
||||
getPageFooter(response.getContent());
|
||||
writeResponse(request, response);
|
||||
}
|
||||
|
||||
private void appendLogConf(StringBuilder buffer) {
|
||||
buffer.append("<h2>Log Configuration</h2>");
|
||||
try {
|
||||
Log4jConfig.Tuple<String, String[], String[]> configs = Log4jConfig.updateLogging(null, null, null);
|
||||
if (!Strings.isNullOrEmpty(addVerboseName)) {
|
||||
addVerboseName = addVerboseName.trim();
|
||||
List<String> verboseNames = Lists.newArrayList(configs.y);
|
||||
if (!verboseNames.contains(addVerboseName)) {
|
||||
verboseNames.add(addVerboseName);
|
||||
configs = Log4jConfig.updateLogging(null, verboseNames.toArray(new String[verboseNames.size()]),
|
||||
null);
|
||||
}
|
||||
}
|
||||
if (!Strings.isNullOrEmpty(delVerboseName)) {
|
||||
delVerboseName = delVerboseName.trim();
|
||||
List<String> verboseNames = Lists.newArrayList(configs.y);
|
||||
if (verboseNames.contains(delVerboseName)) {
|
||||
verboseNames.remove(delVerboseName);
|
||||
configs = Log4jConfig.updateLogging(null, verboseNames.toArray(new String[verboseNames.size()]),
|
||||
null);
|
||||
}
|
||||
}
|
||||
|
||||
buffer.append("Level: " + configs.x + "<br/>");
|
||||
buffer.append("Verbose Names: " + StringUtils.join(configs.y, ",") + "<br/>");
|
||||
buffer.append("Audit Names: " + StringUtils.join(configs.z, ",") + "<br/>");
|
||||
appendUpdateVerboseButton(buffer, "add_verbose");
|
||||
appendUpdateVerboseButton(buffer, "del_verbose");
|
||||
} catch (IOException e) {
|
||||
LOG.error(e);
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
private void appendUpdateVerboseButton(StringBuilder buffer, String type) {
|
||||
String placeHolder = "";
|
||||
String buttonName = "";
|
||||
if (type.equals("add_verbose")) {
|
||||
placeHolder = "new verbose name";
|
||||
buttonName = "Add";
|
||||
} else if (type.equals("del_verbose")) {
|
||||
placeHolder = "del verbose name";
|
||||
buttonName = "Delete";
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
|
||||
buffer.append("<form>"
|
||||
+ "<div class=\"col-lg-3\" style=\"padding-left: 0px;\">"
|
||||
+ " <div class=\"input-group\">"
|
||||
+ " <input name = \"" + type + "\" type=\"text\" class=\"form-control\" placeholder=\""
|
||||
+ placeHolder + "\">"
|
||||
+ " <span class=\"input-group-btn\" style=\"padding-left: 0px;\">"
|
||||
+ " <button class=\"btn btn-default\" type=\"submit\">" + buttonName + "</button>"
|
||||
+ " </span>\n"
|
||||
+ " </div>\n"
|
||||
+ "</div>"
|
||||
+ "</form>");
|
||||
}
|
||||
|
||||
private void appendLogInfo(StringBuilder buffer) {
|
||||
buffer.append("<br/><h2>Log Contents</h2>");
|
||||
|
||||
final String logPath = Config.sys_log_dir + "/fe.warn.log";
|
||||
buffer.append("Log path is: " + logPath + "<br/>");
|
||||
|
||||
RandomAccessFile raf = null;
|
||||
try {
|
||||
raf = new RandomAccessFile(logPath, "r");
|
||||
long fileSize = raf.length();
|
||||
long startPos = fileSize < WEB_LOG_BYTES ? 0L : fileSize - WEB_LOG_BYTES;
|
||||
long webContentLength = fileSize < WEB_LOG_BYTES ? fileSize : WEB_LOG_BYTES;
|
||||
raf.seek(startPos);
|
||||
buffer.append("<p>Showing last " + webContentLength + " bytes of log</p>");
|
||||
buffer.append("<pre>");
|
||||
String fileBuffer = null;
|
||||
while ((fileBuffer = raf.readLine()) != null) {
|
||||
buffer.append(fileBuffer).append("\n");
|
||||
}
|
||||
buffer.append("</pre>");
|
||||
} catch (FileNotFoundException e) {
|
||||
buffer.append("<p class=\"text-error\">Couldn't open log file: "
|
||||
+ logPath + "</p>");
|
||||
} catch (IOException e) {
|
||||
buffer.append("<p class=\"text-error\">Failed to read log file: "
|
||||
+ logPath + "</p>");
|
||||
} finally {
|
||||
try {
|
||||
if (raf != null) {
|
||||
raf.close();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
LOG.warn("fail to close log file: " + logPath, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,52 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.action;
|
||||
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
|
||||
public class NotFoundAction extends WebBaseAction {
|
||||
|
||||
public NotFoundAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean needAdmin() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
getPageHeader(request, response.getContent());
|
||||
|
||||
response.appendContent("this is 404 page.");
|
||||
|
||||
getPageFooter(response.getContent());
|
||||
writeResponse(request, response, HttpResponseStatus.NOT_FOUND);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/404", new NotFoundAction(controller));
|
||||
}
|
||||
}
|
||||
@ -1,127 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.action;
|
||||
|
||||
import org.apache.doris.common.util.ProfileManager;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class QueryAction extends WebBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(QueryAction.class);
|
||||
|
||||
public QueryAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/query", new QueryAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
getPageHeader(request, response.getContent());
|
||||
|
||||
addFinishedQueryInfo(response.getContent());
|
||||
|
||||
getPageFooter(response.getContent());
|
||||
|
||||
writeResponse(request, response);
|
||||
}
|
||||
|
||||
// Note: we do not show 'Query ID' column in web page
|
||||
private void addFinishedQueryInfo(StringBuilder buffer) {
|
||||
buffer.append("<h2>Finished Queries</h2>");
|
||||
buffer.append("<p>You need to set session variable by executing 'set is_report_success=true' before executing the SQL, to view the profile</p>");
|
||||
buffer.append("<p>This table lists the latest 100 queries</p>");
|
||||
|
||||
List<List<String>> finishedQueries = ProfileManager.getInstance().getAllQueries();
|
||||
List<String> columnHeaders = ProfileManager.PROFILE_HEADERS;
|
||||
int queryIdIndex = 0; // the first column is 'Query ID' by default
|
||||
for (int i = 0; i < columnHeaders.size(); ++i) {
|
||||
if (columnHeaders.get(i).equals(ProfileManager.QUERY_ID)) {
|
||||
queryIdIndex = i;
|
||||
}
|
||||
}
|
||||
appendFinishedQueryTableHeader(buffer, columnHeaders, queryIdIndex);
|
||||
appendFinishedQueryTableBody(buffer, finishedQueries, columnHeaders, queryIdIndex);
|
||||
appendTableFooter(buffer);
|
||||
}
|
||||
|
||||
private void appendFinishedQueryTableHeader(
|
||||
StringBuilder buffer,
|
||||
final List<String> columnHeaders,
|
||||
int queryIdIndex) {
|
||||
buffer.append("<table "
|
||||
+ "class=\"table table-hover table-bordered table-striped table-hover\"><tr>");
|
||||
for (int i = 0; i < columnHeaders.size(); ++i) {
|
||||
if (i == queryIdIndex) {
|
||||
continue;
|
||||
}
|
||||
buffer.append("<th>" + columnHeaders.get(i) + "</th>");
|
||||
}
|
||||
|
||||
buffer.append("<th>Profile</th>");
|
||||
buffer.append("</tr>");
|
||||
}
|
||||
|
||||
private void appendFinishedQueryTableBody(
|
||||
StringBuilder buffer,
|
||||
List<List<String>> bodies,
|
||||
List<String> columnHeaders,
|
||||
int queryIdIndex) {
|
||||
for ( List<String> row : bodies) {
|
||||
buffer.append("<tr>");
|
||||
String queryId = row.get(queryIdIndex);
|
||||
|
||||
for (int i = 0; i < row.size(); ++i) {
|
||||
if (i == queryIdIndex) {
|
||||
continue;
|
||||
}
|
||||
buffer.append("<td>");
|
||||
buffer.append(row.get(i));
|
||||
buffer.append("</td>");
|
||||
}
|
||||
|
||||
// add 'Profile' column
|
||||
if (Strings.isNullOrEmpty(queryId)) {
|
||||
LOG.warn("query id is null or empty, maybe we forget to push it "
|
||||
+ "into array when generate profile info.");
|
||||
buffer.append("<td>Empty Query ID</td>");
|
||||
} else {
|
||||
buffer.append("<td>");
|
||||
buffer.append("<a href=\"/query_profile?query_id="
|
||||
+ queryId
|
||||
+ "\">Profile</a>");
|
||||
buffer.append("</td>");
|
||||
}
|
||||
|
||||
buffer.append("</tr>");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,68 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.action;
|
||||
|
||||
import org.apache.doris.common.util.ProfileManager;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
|
||||
public class QueryProfileAction extends WebBaseAction {
|
||||
|
||||
public QueryProfileAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/query_profile", new QueryProfileAction(controller));
|
||||
}
|
||||
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
getPageHeader(request, response.getContent());
|
||||
|
||||
String queryId = request.getSingleParameter("query_id");
|
||||
if (Strings.isNullOrEmpty(queryId)) {
|
||||
response.appendContent("");
|
||||
response.appendContent("<p class=\"text-error\"> Must specify a query_id[]</p>");
|
||||
}
|
||||
|
||||
String queryProfileStr = ProfileManager.getInstance().getProfile(queryId);
|
||||
if (queryProfileStr != null) {
|
||||
appendQueryProfile(response.getContent(), queryProfileStr);
|
||||
getPageFooter(response.getContent());
|
||||
writeResponse(request, response);
|
||||
} else {
|
||||
appendQueryProfile(response.getContent(), "query id " + queryId + " not found.");
|
||||
getPageFooter(response.getContent());
|
||||
writeResponse(request, response, HttpResponseStatus.NOT_FOUND);
|
||||
}
|
||||
}
|
||||
|
||||
private void appendQueryProfile(StringBuilder buffer, String queryProfileStr) {
|
||||
buffer.append("<pre>");
|
||||
buffer.append(queryProfileStr);
|
||||
buffer.append("</pre>");
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,85 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.action;
|
||||
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.service.ExecuteEnv;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
public class SessionAction extends WebBaseAction {
|
||||
// we make
|
||||
private static final ArrayList<String> SESSION_TABLE_HEADER = Lists.newArrayList();
|
||||
|
||||
static {
|
||||
SESSION_TABLE_HEADER.add("Id");
|
||||
SESSION_TABLE_HEADER.add("User");
|
||||
SESSION_TABLE_HEADER.add("Host");
|
||||
SESSION_TABLE_HEADER.add("Cluster");
|
||||
SESSION_TABLE_HEADER.add("Db");
|
||||
SESSION_TABLE_HEADER.add("Command");
|
||||
SESSION_TABLE_HEADER.add("Time");
|
||||
SESSION_TABLE_HEADER.add("State");
|
||||
SESSION_TABLE_HEADER.add("Info");
|
||||
}
|
||||
|
||||
public SessionAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/session", new SessionAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
getPageHeader(request, response.getContent());
|
||||
appendSessionInfo(response.getContent());
|
||||
getPageFooter(response.getContent());
|
||||
writeResponse(request, response);
|
||||
}
|
||||
|
||||
private void appendSessionInfo(StringBuilder buffer) {
|
||||
buffer.append("<h2>Session Info</h2>");
|
||||
|
||||
List<ConnectContext.ThreadInfo> threadInfos = ExecuteEnv.getInstance().getScheduler().listConnection("root");
|
||||
List<List<String>> rowSet = Lists.newArrayList();
|
||||
long nowMs = System.currentTimeMillis();
|
||||
for (ConnectContext.ThreadInfo info : threadInfos) {
|
||||
rowSet.add(info.toRow(nowMs));
|
||||
}
|
||||
|
||||
buffer.append("<p>This page lists the session info, there are "
|
||||
+ rowSet.size()
|
||||
+ " active sessions.</p>");
|
||||
|
||||
appendTableHeader(buffer, SESSION_TABLE_HEADER);
|
||||
appendTableBody(buffer, rowSet);
|
||||
appendTableFooter(buffer);
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,242 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.action;
|
||||
|
||||
import org.apache.doris.PaloFe;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.File;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.GregorianCalendar;
|
||||
import java.util.Locale;
|
||||
import java.util.TimeZone;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import javax.activation.MimetypesFileTypeMap;
|
||||
|
||||
import io.netty.handler.codec.http.HttpHeaderNames;
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
|
||||
/**
|
||||
* A simple handler that serves incoming HTTP requests to send their respective
|
||||
* HTTP responses. It also implements {@code 'If-Modified-Since'} header to
|
||||
* take advantage of browser cache, as described in
|
||||
* <a href="http://tools.ietf.org/html/rfc2616#section-14.25">RFC 2616</a>.
|
||||
*
|
||||
* <h3>How Browser Caching Works</h3>
|
||||
*
|
||||
* Web browser caching works with HTTP headers as illustrated by the following
|
||||
* sample:
|
||||
* <ol>
|
||||
* <li>Request #1 returns the content of {@code /file1.txt}.</li>
|
||||
* <li>Contents of {@code /file1.txt} is cached by the browser.</li>
|
||||
* <li>Request #2 for {@code /file1.txt} does return the contents of the
|
||||
* file again. Rather, a 304 Not Modified is returned. This tells the
|
||||
* browser to use the contents stored in its cache.</li>
|
||||
* <li>The server knows the file has not been modified because the
|
||||
* {@code If-Modified-Since} date is the same as the file's last
|
||||
* modified date.</li>
|
||||
* </ol>
|
||||
*
|
||||
* <pre>
|
||||
* Request #1 Headers
|
||||
* ===================
|
||||
* GET /file1.txt HTTP/1.1
|
||||
*
|
||||
* Response #1 Headers
|
||||
* ===================
|
||||
* HTTP/1.1 200 OK
|
||||
* Date: Tue, 01 Mar 2011 22:44:26 GMT
|
||||
* Last-Modified: Wed, 30 Jun 2010 21:36:48 GMT
|
||||
* Expires: Tue, 01 Mar 2012 22:44:26 GMT
|
||||
* Cache-Control: private, max-age=31536000
|
||||
*
|
||||
* Request #2 Headers
|
||||
* ===================
|
||||
* GET /file1.txt HTTP/1.1
|
||||
* If-Modified-Since: Wed, 30 Jun 2010 21:36:48 GMT
|
||||
*
|
||||
* Response #2 Headers
|
||||
* ===================
|
||||
* HTTP/1.1 304 Not Modified
|
||||
* Date: Tue, 01 Mar 2011 22:44:28 GMT
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
* We use parameter named 'res' to specify the static resource path, it relative to the
|
||||
* root path of http server.
|
||||
*/
|
||||
public class StaticResourceAction extends WebBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(StaticResourceAction.class);
|
||||
|
||||
public static final String HTTP_DATE_FORMAT = "EEE, dd MMM yyyy HH:mm:ss zzz";
|
||||
public static final String HTTP_DATE_GMT_TIMEZONE = "GMT";
|
||||
public static final int HTTP_CACHE_SECONDS = 2592000;
|
||||
|
||||
public MimetypesFileTypeMap mimeTypesMap;
|
||||
public String rootDir;
|
||||
|
||||
public StaticResourceAction(ActionController controller, String rootDir) {
|
||||
super(controller);
|
||||
mimeTypesMap = new MimetypesFileTypeMap();
|
||||
mimeTypesMap.addMimeTypes("text/html html htm");
|
||||
// According to RFC 4329(http://tools.ietf.org/html/rfc4329#section-7.2), the MIME type of
|
||||
// javascript script is 'application/javascript'
|
||||
mimeTypesMap.addMimeTypes("application/javascript js");
|
||||
// mimeTypesMap.addMimeTypes("text/javascript js");
|
||||
mimeTypesMap.addMimeTypes("text/css css");
|
||||
|
||||
this.rootDir = rootDir;
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
String httpDir = PaloFe.DORIS_HOME_DIR + "/webroot";
|
||||
StaticResourceAction action = new StaticResourceAction(controller, httpDir + "/static");
|
||||
controller.registerHandler(HttpMethod.GET, "/static/js", action);
|
||||
controller.registerHandler(HttpMethod.GET, "/static/css", action);
|
||||
controller.registerHandler(HttpMethod.GET, "/static", action);
|
||||
controller.registerHandler(HttpMethod.GET, "/static/resource", action);
|
||||
controller.registerHandler(HttpMethod.GET, "/static/images", action);
|
||||
controller.registerHandler(HttpMethod.GET, "/static/Bootstrap-3.3.7/fonts/", action);
|
||||
|
||||
StaticResourceAction action2 = new StaticResourceAction(controller, "webroot");
|
||||
controller.registerHandler(HttpMethod.GET, "/static_test", action2);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean needAdmin() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
String resourcePath = request.getSingleParameter("res");
|
||||
if (Strings.isNullOrEmpty(resourcePath)) {
|
||||
LOG.error("Wrong request without 'res' parameter. url: {}",
|
||||
request.getRequest().uri());
|
||||
writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
resourcePath = sanitizePath(resourcePath);
|
||||
if (resourcePath == null) {
|
||||
LOG.error("Close this request because of risk factor in 'res' parameter.url: {}",
|
||||
request.getRequest().uri());
|
||||
writeResponse(request, response, HttpResponseStatus.FORBIDDEN);
|
||||
return;
|
||||
}
|
||||
|
||||
String resourceAbsolutePath = rootDir + File.separator + resourcePath;
|
||||
File resFile = new File(resourceAbsolutePath);
|
||||
LOG.debug("resAbsolutePath: {}", resourceAbsolutePath);
|
||||
|
||||
if (!resFile.exists() || resFile.isHidden() || resFile.isDirectory()) {
|
||||
LOG.error("Request with wrong path. url: {}", request.getRequest().uri());
|
||||
writeResponse(request, response, HttpResponseStatus.NOT_FOUND);
|
||||
return;
|
||||
}
|
||||
if (!resFile.isFile()) {
|
||||
LOG.error("Wrong request: not normal file. url: {}", request.getRequest().uri());
|
||||
writeResponse(request, response, HttpResponseStatus.FORBIDDEN);
|
||||
return;
|
||||
}
|
||||
|
||||
// Cache validation
|
||||
String ifModifiedSince = request.getRequest()
|
||||
.headers().get(HttpHeaderNames.IF_MODIFIED_SINCE.toString());
|
||||
if (!Strings.isNullOrEmpty(ifModifiedSince)) {
|
||||
SimpleDateFormat dateFormatter = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US);
|
||||
Date ifModifiedSinceDate;
|
||||
try {
|
||||
ifModifiedSinceDate = dateFormatter.parse(ifModifiedSince);
|
||||
// Only compare up to the second because the datetime format we send to the client
|
||||
// does not have milliseconds
|
||||
long ifModifiedSinceDateSeconds = ifModifiedSinceDate.getTime() / 1000;
|
||||
long fileLastModifiedSeconds = resFile.lastModified() / 1000;
|
||||
if (ifModifiedSinceDateSeconds == fileLastModifiedSeconds) {
|
||||
writeResponse(request, response, HttpResponseStatus.NOT_MODIFIED);
|
||||
return;
|
||||
}
|
||||
} catch (ParseException e) {
|
||||
LOG.error("Fail to analyse IF_MODIFIED_SINCE header: ", e);
|
||||
writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
response.updateHeader(HttpHeaderNames.CONTENT_TYPE.toString(), getContentType(resourceAbsolutePath));
|
||||
setDateAndCacheHeaders(response, resFile);
|
||||
|
||||
writeObjectResponse(request, response, HttpResponseStatus.OK, resFile, resFile.getName(), false);
|
||||
}
|
||||
|
||||
|
||||
|
||||
// Gets the content type header for the HTTP Response
|
||||
private String getContentType(String filename) {
|
||||
return mimeTypesMap.getContentType(filename);
|
||||
}
|
||||
|
||||
private void setDateAndCacheHeaders(BaseResponse response, File fileToCache) {
|
||||
SimpleDateFormat dateFormatter = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US);
|
||||
dateFormatter.setTimeZone(TimeZone.getTimeZone(HTTP_DATE_GMT_TIMEZONE));
|
||||
|
||||
// Date header
|
||||
Calendar time = new GregorianCalendar();
|
||||
response.updateHeader(HttpHeaderNames.DATE.toString(), dateFormatter.format(time.getTime()));
|
||||
|
||||
// Add cache headers
|
||||
time.add(Calendar.SECOND, HTTP_CACHE_SECONDS);
|
||||
response.updateHeader(HttpHeaderNames.EXPIRES.toString(), dateFormatter.format(time.getTime()));
|
||||
response.updateHeader(HttpHeaderNames.CACHE_CONTROL.toString(), "private, max-age=" + HTTP_CACHE_SECONDS);
|
||||
response.updateHeader(HttpHeaderNames.LAST_MODIFIED.toString(),
|
||||
dateFormatter.format(new Date(fileToCache.lastModified())));
|
||||
}
|
||||
|
||||
private static final Pattern INSECURE_URI = Pattern.compile(".*[<>&\"].*");
|
||||
|
||||
private String sanitizePath(String path) {
|
||||
if (Strings.isNullOrEmpty(path)) {
|
||||
return null;
|
||||
}
|
||||
// Convert file separators
|
||||
String newPath = path.replace('/', File.separatorChar);
|
||||
|
||||
// Simplistic dumb security check.
|
||||
if (newPath.contains(File.separator + '.')
|
||||
|| newPath.contains('.' + File.separator)
|
||||
|| newPath.charAt(0) == '.'
|
||||
|| newPath.charAt(newPath.length() - 1) == '.'
|
||||
|| INSECURE_URI.matcher(newPath).matches()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return newPath;
|
||||
}
|
||||
}
|
||||
@ -1,184 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.action;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
import org.apache.doris.analysis.RedirectStatus;
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.common.AnalysisException;
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.common.proc.ProcDirInterface;
|
||||
import org.apache.doris.common.proc.ProcNodeInterface;
|
||||
import org.apache.doris.common.proc.ProcResult;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.qe.MasterOpExecutor;
|
||||
import org.apache.doris.qe.OriginStatement;
|
||||
import org.apache.doris.qe.ShowResultSet;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import org.apache.commons.validator.routines.UrlValidator;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class SystemAction extends WebBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(SystemAction.class);
|
||||
|
||||
public SystemAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/system", new SystemAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
getPageHeader(request, response.getContent());
|
||||
|
||||
String currentPath = request.getSingleParameter("path");
|
||||
if (Strings.isNullOrEmpty(currentPath)) {
|
||||
currentPath = "/";
|
||||
}
|
||||
appendSystemInfo(response.getContent(), currentPath, currentPath);
|
||||
|
||||
getPageFooter(response.getContent());
|
||||
writeResponse(request, response);
|
||||
}
|
||||
|
||||
private void appendSystemInfo(StringBuilder buffer, String procPath, String path) {
|
||||
buffer.append("<h2>System Info</h2>");
|
||||
buffer.append("<p>This page lists the system info, like /proc in Linux.</p>");
|
||||
buffer.append("<p class=\"text-info\"> Current path: " + path + "<a href=\"?path=" + getParentPath(path)
|
||||
+ "\" class=\"btn btn-primary\" style=\"float: right;\">"
|
||||
+ "Parent Dir</a></p><br/>");
|
||||
|
||||
ProcNodeInterface procNode = getProcNode(procPath);
|
||||
if (procNode == null) {
|
||||
buffer.append("<p class=\"text-error\"> No such proc path[" + path + "]</p>");
|
||||
return;
|
||||
}
|
||||
boolean isDir = (procNode instanceof ProcDirInterface);
|
||||
|
||||
List<String> columnNames = null;
|
||||
List<List<String>> rows = null;
|
||||
if (!Catalog.getCurrentCatalog().isMaster() && !Config.enable_bdbje_debug_mode) {
|
||||
// forward to master
|
||||
String showProcStmt = "SHOW PROC \"" + procPath + "\"";
|
||||
MasterOpExecutor masterOpExecutor = new MasterOpExecutor(new OriginStatement(showProcStmt, 0),
|
||||
ConnectContext.get(), RedirectStatus.FORWARD_NO_SYNC, true);
|
||||
try {
|
||||
masterOpExecutor.execute();
|
||||
} catch (Exception e) {
|
||||
LOG.warn("Fail to forward. ", e);
|
||||
buffer.append("<p class=\"text-error\"> Failed to forward request to master</p>");
|
||||
return;
|
||||
}
|
||||
|
||||
ShowResultSet resultSet = masterOpExecutor.getProxyResultSet();
|
||||
if (resultSet == null) {
|
||||
buffer.append("<p class=\"text-error\"> Failed to get result from master</p>");
|
||||
return;
|
||||
}
|
||||
|
||||
columnNames = resultSet.getMetaData().getColumns().stream().map(c -> c.getName()).collect(
|
||||
Collectors.toList());
|
||||
rows = resultSet.getResultRows();
|
||||
} else {
|
||||
ProcResult result;
|
||||
try {
|
||||
result = procNode.fetchResult();
|
||||
} catch (AnalysisException e) {
|
||||
buffer.append("<p class=\"text-error\"> The result is null, "
|
||||
+ "maybe haven't be implemented completely[" + e.getMessage() + "], please check.</p>");
|
||||
buffer.append("<p class=\"text-info\"> "
|
||||
+ "INFO: ProcNode type is [" + procNode.getClass().getName()
|
||||
+ "]</p>");
|
||||
return;
|
||||
}
|
||||
|
||||
columnNames = result.getColumnNames();
|
||||
rows = result.getRows();
|
||||
}
|
||||
|
||||
Preconditions.checkNotNull(columnNames);
|
||||
Preconditions.checkNotNull(rows);
|
||||
|
||||
appendTableHeader(buffer, columnNames);
|
||||
appendSystemTableBody(buffer, rows, isDir, path);
|
||||
appendTableFooter(buffer);
|
||||
}
|
||||
|
||||
private void appendSystemTableBody(StringBuilder buffer, List<List<String>> rows, boolean isDir, String path) {
|
||||
UrlValidator validator = new UrlValidator();
|
||||
for ( List<String> strList : rows) {
|
||||
buffer.append("<tr>");
|
||||
int columnIndex = 1;
|
||||
for (String str : strList) {
|
||||
buffer.append("<td>");
|
||||
if (isDir && columnIndex == 1) {
|
||||
String escapeStr = str.replace("%", "%25");
|
||||
buffer.append("<a href=\"?path=" + path + "/" + escapeStr + "\">");
|
||||
buffer.append(str);
|
||||
buffer.append("</a>");
|
||||
} else if (validator.isValid(str)) {
|
||||
buffer.append("<a href=\"" + str + "\">");
|
||||
buffer.append("URL");
|
||||
buffer.append("</a>");
|
||||
} else {
|
||||
buffer.append(str.replaceAll("\\n", "<br/>"));
|
||||
}
|
||||
buffer.append("</td>");
|
||||
++columnIndex;
|
||||
}
|
||||
buffer.append("</tr>");
|
||||
}
|
||||
}
|
||||
|
||||
// some example:
|
||||
// '/' => '/'
|
||||
// '///aaa' => '///'
|
||||
// '/aaa/bbb///' => '/aaa'
|
||||
// '/aaa/bbb/ccc' => '/aaa/bbb'
|
||||
// ATTN: the root path's parent is itself.
|
||||
private String getParentPath(String path) {
|
||||
int lastSlashIndex = path.length() - 1;
|
||||
while (lastSlashIndex > 0) {
|
||||
int tempIndex = path.lastIndexOf('/', lastSlashIndex);
|
||||
if (tempIndex > 0) {
|
||||
if (tempIndex == lastSlashIndex) {
|
||||
lastSlashIndex = tempIndex - 1;
|
||||
continue;
|
||||
} else if (tempIndex < lastSlashIndex) { // '//aaa/bbb'
|
||||
lastSlashIndex = tempIndex;
|
||||
return path.substring(0, lastSlashIndex);
|
||||
}
|
||||
}
|
||||
}
|
||||
return "/";
|
||||
}
|
||||
}
|
||||
@ -1,84 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.action;
|
||||
|
||||
import org.apache.doris.analysis.SetType;
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.qe.VariableMgr;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
||||
public class VariableAction extends WebBaseAction {
|
||||
|
||||
public VariableAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/variable", new VariableAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
getPageHeader(request, response.getContent());
|
||||
|
||||
appendConfigureInfo(response.getContent());
|
||||
appendVariableInfo(response.getContent());
|
||||
|
||||
getPageFooter(response.getContent());
|
||||
writeResponse(request, response);
|
||||
}
|
||||
|
||||
public void appendConfigureInfo(StringBuilder buffer) {
|
||||
buffer.append("<h2>Configure Info</h2>");
|
||||
buffer.append("<pre>");
|
||||
HashMap<String, String> confmap;
|
||||
try {
|
||||
confmap = Config.dump();
|
||||
List<String> keyList = Lists.newArrayList(confmap.keySet());
|
||||
Collections.sort(keyList);
|
||||
for (String key : keyList) {
|
||||
buffer.append(key + "=" + confmap.get(key) + "\n");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
buffer.append("read conf exception" + e.toString());
|
||||
}
|
||||
buffer.append("</pre>");
|
||||
}
|
||||
|
||||
private void appendVariableInfo(StringBuilder buffer) {
|
||||
buffer.append("<h2>Variable Info</h2>");
|
||||
buffer.append("<pre>");
|
||||
List<List<String>> variableInfo = VariableMgr.dump(SetType.GLOBAL, null, null);
|
||||
for (List<String> list : variableInfo) {
|
||||
buffer.append(list.get(0) + "=" + list.get(1) + "\n");
|
||||
}
|
||||
buffer.append("</pre>");
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,374 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.action;
|
||||
|
||||
import org.apache.doris.analysis.CompoundPredicate.Operator;
|
||||
import org.apache.doris.analysis.UserIdentity;
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.common.AnalysisException;
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.common.proc.ProcNodeInterface;
|
||||
import org.apache.doris.common.proc.ProcService;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseAction;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.HttpAuthManager;
|
||||
import org.apache.doris.http.HttpAuthManager.SessionValue;
|
||||
import org.apache.doris.http.UnauthorizedException;
|
||||
import org.apache.doris.http.rest.RestBaseResult;
|
||||
import org.apache.doris.mysql.privilege.PaloPrivilege;
|
||||
import org.apache.doris.mysql.privilege.PrivBitSet;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.system.SystemInfoService;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
import io.netty.handler.codec.http.HttpHeaderNames;
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
import io.netty.handler.codec.http.cookie.DefaultCookie;
|
||||
|
||||
public class WebBaseAction extends BaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(WebBaseAction.class);
|
||||
|
||||
protected static final String PALO_SESSION_ID = "PALO_SESSION_ID";
|
||||
private static final long PALO_SESSION_EXPIRED_TIME = 3600 * 24; // one day
|
||||
|
||||
protected static final String PAGE_HEADER = "<!DOCTYPE html>"
|
||||
+ "<html>"
|
||||
+ "<head>"
|
||||
+ " <title>Apache Doris(Incubating)</title>"
|
||||
+ " <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" >"
|
||||
+ " <link rel=\"shortcut icon\" href=\"/static/images?res=favicon.ico\">"
|
||||
+ " <link href=\"/static/css?res=Bootstrap-3.3.7/css/bootstrap.css\" "
|
||||
+ " rel=\"stylesheet\" media=\"screen\"/>"
|
||||
+ " <link href=\"/static/css?res=Bootstrap-3.3.7/css/bootstrap-theme.css\" "
|
||||
+ " rel=\"stylesheet\" media=\"screen\"/>"
|
||||
+ " <link href=\"/static/css?res=DataTables-1.10.25/css/dataTables.bootstrap.css\" "
|
||||
+ " rel=\"stylesheet\" media=\"screen\"/>"
|
||||
|
||||
+ " <script type=\"text/javascript\" src=\"/static?res=jQuery-3.3.1/jquery-3.3.1.min.js\"></script>"
|
||||
+ " <script type=\"text/javascript\" src=\"/static?res=DataTables-1.10.25/js/jquery.dataTables.js\"></script>"
|
||||
+ " <script type=\"text/javascript\" src=\"/static?res=DataTables-1.10.25/js/dataTables.bootstrap.js\"></script>"
|
||||
|
||||
+ " <script type=\"text/javascript\"> "
|
||||
+ " $(document).ready(function() { "
|
||||
+ " $('#table_id').dataTable({ "
|
||||
+ " \"aaSorting\": [],"
|
||||
+ " \"lengthMenu\": [[10, 25, 50, 100,-1], [10, 25, 50, 100, \"All\"]],"
|
||||
+ " \"iDisplayLength\": 50,"
|
||||
+ " });"
|
||||
+ " }); "
|
||||
+ " $(document).ready(function () {"
|
||||
+ " var location = window.location.pathname;"
|
||||
+ " var id = location.substring(location.lastIndexOf('/') + 1);"
|
||||
+ " if (id != null || $.trim(id) != \"\") {"
|
||||
+ " $(\"#nav_\" + id).addClass(\"active\");"
|
||||
+ " }"
|
||||
+ " });"
|
||||
+ " </script> "
|
||||
|
||||
+ " <style>"
|
||||
+ " body {"
|
||||
+ " padding-top: 60px;"
|
||||
+ " }"
|
||||
+ " </style>"
|
||||
+ "</head>"
|
||||
+ "<body>";
|
||||
protected static final String PAGE_FOOTER = "</div></body></html>";
|
||||
protected static final String NAVIGATION_BAR_PREFIX =
|
||||
" <nav class=\"navbar navbar-default navbar-fixed-top\" role=\"navigation\">"
|
||||
+ " <div class=\"container-fluid\">"
|
||||
+ " <div class=\"navbar-header\">"
|
||||
+ " <a class=\"navbar-brand\" href=\"/\" style=\"padding: unset;\">"
|
||||
+ " <img alt=\"Doris\" style=\"height: inherit;\" src=\"/static/images?res=doris-logo.png\">"
|
||||
+ " </a>"
|
||||
+ " </div>"
|
||||
+ " <div>"
|
||||
+ " <ul class=\"nav navbar-nav\" role=\"tablist\">";
|
||||
protected static final String NAVIGATION_BAR_SUFFIX =
|
||||
" </ul>"
|
||||
+ " </div>"
|
||||
+ " </nav>"
|
||||
+ " <div class=\"container\">";
|
||||
|
||||
public WebBaseAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(BaseRequest request, BaseResponse response) {
|
||||
if (!checkAuthWithCookie(request, response)) {
|
||||
return;
|
||||
}
|
||||
|
||||
HttpMethod method = request.getRequest().method();
|
||||
if (method.equals(HttpMethod.GET)) {
|
||||
executeGet(request, response);
|
||||
} else if (method.equals(HttpMethod.POST)) {
|
||||
executePost(request, response);
|
||||
} else {
|
||||
response.appendContent(new RestBaseResult("HTTP method is not allowed: " + method.name()).toJson());
|
||||
writeResponse(request, response, HttpResponseStatus.METHOD_NOT_ALLOWED);
|
||||
}
|
||||
}
|
||||
|
||||
// Sub Action class should override this method
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
response.appendContent(new RestBaseResult("Not implemented").toJson());
|
||||
writeResponse(request, response, HttpResponseStatus.NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
// Sub Action class should override this method
|
||||
public void executePost(BaseRequest request, BaseResponse response) {
|
||||
response.appendContent(new RestBaseResult("Not implemented").toJson());
|
||||
writeResponse(request, response, HttpResponseStatus.NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
// We first check cookie, if not admin, we check http's authority header
|
||||
private boolean checkAuthWithCookie(BaseRequest request, BaseResponse response) {
|
||||
if (!needPassword()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (checkCookie(request, response)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// cookie is invalid.
|
||||
ActionAuthorizationInfo authInfo;
|
||||
try {
|
||||
authInfo = getAuthorizationInfo(request);
|
||||
UserIdentity currentUser = checkPassword(authInfo);
|
||||
if (needAdmin()) {
|
||||
checkGlobalAuth(currentUser, PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV,
|
||||
PaloPrivilege.NODE_PRIV), Operator.OR));
|
||||
}
|
||||
request.setAuthorized(true);
|
||||
SessionValue value = new SessionValue();
|
||||
value.currentUser = currentUser;
|
||||
addSession(request, response, value);
|
||||
|
||||
ConnectContext ctx = new ConnectContext(null);
|
||||
ctx.setQualifiedUser(authInfo.fullUserName);
|
||||
ctx.setRemoteIP(authInfo.remoteIp);
|
||||
ctx.setCurrentUserIdentity(currentUser);
|
||||
ctx.setCatalog(Catalog.getCurrentCatalog());
|
||||
ctx.setCluster(SystemInfoService.DEFAULT_CLUSTER);
|
||||
ctx.setThreadLocalInfo();
|
||||
|
||||
return true;
|
||||
} catch (UnauthorizedException e) {
|
||||
response.appendContent("Authentication Failed. <br/> " + e.getMessage());
|
||||
writeAuthResponse(request, response);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean checkCookie(BaseRequest request, BaseResponse response) {
|
||||
String sessionId = request.getCookieValue(PALO_SESSION_ID);
|
||||
HttpAuthManager authMgr = HttpAuthManager.getInstance();
|
||||
if (!Strings.isNullOrEmpty(sessionId)) {
|
||||
SessionValue sessionValue = authMgr.getSessionValue(sessionId);
|
||||
if (sessionValue == null) {
|
||||
return false;
|
||||
}
|
||||
if (Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(sessionValue.currentUser,
|
||||
PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV,
|
||||
PaloPrivilege.NODE_PRIV),
|
||||
Operator.OR))) {
|
||||
response.updateCookieAge(request, PALO_SESSION_ID, PALO_SESSION_EXPIRED_TIME);
|
||||
request.setAuthorized(true);
|
||||
|
||||
ConnectContext ctx = new ConnectContext(null);
|
||||
ctx.setQualifiedUser(sessionValue.currentUser.getQualifiedUser());
|
||||
ctx.setRemoteIP(request.getHostString());
|
||||
ctx.setCurrentUserIdentity(sessionValue.currentUser);
|
||||
ctx.setCatalog(Catalog.getCurrentCatalog());
|
||||
ctx.setCluster(SystemInfoService.DEFAULT_CLUSTER);
|
||||
ctx.setThreadLocalInfo();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// return true if this Action need to check password.
|
||||
// Currently, all sub actions need to check password except for MetaBaseAction.
|
||||
// if needPassword() is false, then needAdmin() should also return false
|
||||
public boolean needPassword() {
|
||||
return true;
|
||||
}
|
||||
|
||||
// return true if this Action need Admin privilege.
|
||||
public boolean needAdmin() {
|
||||
return true;
|
||||
}
|
||||
|
||||
protected void writeAuthResponse(BaseRequest request, BaseResponse response) {
|
||||
response.updateHeader(HttpHeaderNames.WWW_AUTHENTICATE.toString(), "Basic realm=\"\"");
|
||||
writeResponse(request, response, HttpResponseStatus.UNAUTHORIZED);
|
||||
}
|
||||
|
||||
protected void writeResponse(BaseRequest request, BaseResponse response) {
|
||||
writeResponse(request, response, HttpResponseStatus.OK);
|
||||
}
|
||||
|
||||
protected void addSession(BaseRequest request, BaseResponse response, SessionValue value) {
|
||||
String key = UUID.randomUUID().toString();
|
||||
DefaultCookie cookie = new DefaultCookie(PALO_SESSION_ID, key);
|
||||
cookie.setMaxAge(PALO_SESSION_EXPIRED_TIME);
|
||||
response.addCookie(cookie);
|
||||
HttpAuthManager.getInstance().addSessionValue(key, value);
|
||||
}
|
||||
|
||||
protected void getPageHeader(BaseRequest request, StringBuilder sb) {
|
||||
String newPageHeaderString = PAGE_HEADER;
|
||||
newPageHeaderString = newPageHeaderString.replaceAll("<title>Apache Doris</title>",
|
||||
"<title>" + Config.cluster_name + "</title>");
|
||||
|
||||
sb.append(newPageHeaderString);
|
||||
sb.append(NAVIGATION_BAR_PREFIX);
|
||||
|
||||
if (request.isAuthorized()) {
|
||||
sb.append("<li id=\"nav_system\"><a href=\"/system\">")
|
||||
.append("system")
|
||||
.append("</a></li>");
|
||||
sb.append("<li id=\"nav_backend\"><a href=\"/backend\">")
|
||||
.append("backends")
|
||||
.append("</a></li>");
|
||||
sb.append("<li id=\"nav_log\"><a href=\"/log\">")
|
||||
.append("logs")
|
||||
.append("</a></li>");
|
||||
sb.append("<li id=\"nav_query\"><a href=\"/query\">")
|
||||
.append("queries")
|
||||
.append("</a></li>");
|
||||
sb.append("<li id=\"nav_session\"><a href=\"/session\">")
|
||||
.append("sessions")
|
||||
.append("</a></li>");
|
||||
sb.append("<li id=\"nav_variable\"><a href=\"/variable\">")
|
||||
.append("variables")
|
||||
.append("</a></li>");
|
||||
sb.append("<li id=\"nav_ha\"><a href=\"/ha\">")
|
||||
.append("ha")
|
||||
.append("</a></li>");
|
||||
}
|
||||
sb.append("<li id=\"nav_help\"><a href=\"/help\">")
|
||||
.append("help")
|
||||
.append("</a></li></tr>");
|
||||
|
||||
sb.append(NAVIGATION_BAR_SUFFIX);
|
||||
}
|
||||
|
||||
protected void getPageFooter(StringBuilder sb) {
|
||||
sb.append(PAGE_FOOTER);
|
||||
}
|
||||
|
||||
// Note: DO NOT omit '<thead>', because it is useful in 'datatable' plugin.
|
||||
protected void appendTableHeader(StringBuilder buffer, List<String> columnHeaders) {
|
||||
buffer.append("<table id=\"table_id\" "
|
||||
+ "class=\"table table-hover table-bordered table-striped table-condensed\">");
|
||||
buffer.append("<thead><tr> ");
|
||||
for (String str : columnHeaders) {
|
||||
buffer.append("<th>" + str + "</th>");
|
||||
}
|
||||
buffer.append(" </tr></thead>");
|
||||
}
|
||||
|
||||
protected void appendTableBody(StringBuilder buffer, List<List<String>> bodies) {
|
||||
buffer.append("<tbody>");
|
||||
for ( List<String> row : bodies) {
|
||||
buffer.append("<tr>");
|
||||
for (String column : row) {
|
||||
buffer.append("<td>");
|
||||
buffer.append(column);
|
||||
buffer.append("</td>");
|
||||
}
|
||||
buffer.append("</tr>");
|
||||
}
|
||||
buffer.append("</tbody>");
|
||||
}
|
||||
|
||||
protected void appendTableFooter(StringBuilder buffer) {
|
||||
buffer.append("</table>");
|
||||
}
|
||||
|
||||
protected ProcNodeInterface getProcNode(String path) {
|
||||
ProcService instance = ProcService.getInstance();
|
||||
ProcNodeInterface node;
|
||||
try {
|
||||
if (Strings.isNullOrEmpty(path)) {
|
||||
node = instance.open("/");
|
||||
} else {
|
||||
node = instance.open(path);
|
||||
}
|
||||
} catch (AnalysisException e) {
|
||||
LOG.warn(e.getMessage());
|
||||
return null;
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
// Because org.apache.commons.lang.StringEscapeUtils.escapeHtml() not only escape tags in html,
|
||||
// but also escape Chinese character code, which may cause Chinese garbled in browser, so we
|
||||
// define our own simplified escape method here.
|
||||
// ATTN: we should make sure file-encoding of help files is utf-8
|
||||
protected String escapeHtmlInPreTag(String oriStr) {
|
||||
if (oriStr == null) {
|
||||
return "";
|
||||
}
|
||||
|
||||
StringBuilder buff = new StringBuilder();
|
||||
char[] chars = oriStr.toCharArray();
|
||||
for (int i = 0; i < chars.length; ++i) {
|
||||
switch (chars[i]) {
|
||||
case '<':
|
||||
buff.append("<");
|
||||
break;
|
||||
case '>':
|
||||
buff.append("<");
|
||||
break;
|
||||
case '"':
|
||||
buff.append(""");
|
||||
break;
|
||||
case '&':
|
||||
buff.append("&");
|
||||
break;
|
||||
default:
|
||||
buff.append(chars[i]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return buff.toString();
|
||||
}
|
||||
|
||||
private static final NotFoundAction NOT_FOUND_ACTION = new NotFoundAction(null);
|
||||
|
||||
public static NotFoundAction getNotFoundAction() {
|
||||
return NOT_FOUND_ACTION;
|
||||
}
|
||||
}
|
||||
@ -1,61 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.common;
|
||||
|
||||
import io.netty.channel.ChannelPipeline;
|
||||
import io.netty.handler.codec.http.HttpContent;
|
||||
import io.netty.handler.codec.http.HttpMessage;
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpObject;
|
||||
import io.netty.handler.codec.http.HttpObjectAggregator;
|
||||
import io.netty.handler.codec.http.HttpRequest;
|
||||
|
||||
/*
|
||||
* only handle post request, avoid conflicting with {@link LoadAction}
|
||||
* don't handle 100-continue header
|
||||
*/
|
||||
public class DorisHttpPostObjectAggregator extends HttpObjectAggregator {
|
||||
private boolean startAggregated = false;
|
||||
|
||||
public DorisHttpPostObjectAggregator(int maxContentLength) {
|
||||
super(maxContentLength, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean isStartMessage(HttpObject msg) throws Exception {
|
||||
if (msg instanceof HttpMessage) {
|
||||
HttpRequest request = (HttpRequest) msg;
|
||||
if (request.method().equals(HttpMethod.POST)) {
|
||||
startAggregated = true;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean isContentMessage(HttpObject msg) throws Exception {
|
||||
return msg instanceof HttpContent && startAggregated;
|
||||
}
|
||||
|
||||
// Doris FE needn't handle 100-continue header
|
||||
@Override
|
||||
protected Object newContinueResponse(HttpMessage start, int maxContentLength, ChannelPipeline pipeline) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -1,229 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.meta;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.catalog.ColocateGroupSchema;
|
||||
import org.apache.doris.catalog.ColocateTableIndex;
|
||||
import org.apache.doris.catalog.ColocateTableIndex.GroupId;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.http.rest.RestBaseAction;
|
||||
import org.apache.doris.http.rest.RestBaseResult;
|
||||
import org.apache.doris.http.rest.RestResult;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.reflect.TypeToken;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.lang.reflect.Type;
|
||||
import java.util.List;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
|
||||
/*
|
||||
* the colocate meta define in {@link ColocateTableIndex}
|
||||
* The actions in ColocateMetaService is for modifying or showing colocate group info manually.
|
||||
*
|
||||
* ColocateMetaAction:
|
||||
* get all information in ColocateTableIndex, as a json string
|
||||
* eg:
|
||||
* GET /api/colocate
|
||||
* return:
|
||||
* {"colocate_meta":{"groupName2Id":{...},"group2Tables":{}, ...},"status":"OK"}
|
||||
*
|
||||
* eg:
|
||||
* POST /api/colocate/group_stable?db_id=123&group_id=456 (mark group[123.456] as unstable)
|
||||
* DELETE /api/colocate/group_stable?db_id=123&group_id=456 (mark group[123.456] as stable)
|
||||
*
|
||||
* BucketSeqAction:
|
||||
* change the backends per bucket sequence of a group
|
||||
* eg:
|
||||
* POST /api/colocate/bucketseq?db_id=123&group_id=456
|
||||
*/
|
||||
public class ColocateMetaService {
|
||||
private static final Logger LOG = LogManager.getLogger(ColocateMetaService.class);
|
||||
private static final String GROUP_ID = "group_id";
|
||||
private static final String TABLE_ID = "table_id";
|
||||
private static final String DB_ID = "db_id";
|
||||
|
||||
private static ColocateTableIndex colocateIndex = Catalog.getCurrentColocateIndex();
|
||||
|
||||
private static GroupId checkAndGetGroupId(BaseRequest request) throws DdlException {
|
||||
long grpId = Long.valueOf(request.getSingleParameter(GROUP_ID).trim());
|
||||
long dbId = Long.valueOf(request.getSingleParameter(DB_ID).trim());
|
||||
GroupId groupId = new GroupId(dbId, grpId);
|
||||
|
||||
if (!colocateIndex.isGroupExist(groupId)) {
|
||||
throw new DdlException("the group " + groupId + "isn't exist");
|
||||
}
|
||||
return groupId;
|
||||
}
|
||||
|
||||
public static class ColocateMetaBaseAction extends RestBaseAction {
|
||||
ColocateMetaBaseAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeWithoutPassword(BaseRequest request, BaseResponse response)
|
||||
throws DdlException {
|
||||
if (redirectToMaster(request, response)) {
|
||||
return;
|
||||
}
|
||||
checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
|
||||
executeInMasterWithAdmin(request, response);
|
||||
}
|
||||
|
||||
// implement in derived classes
|
||||
protected void executeInMasterWithAdmin(BaseRequest request, BaseResponse response)
|
||||
throws DdlException {
|
||||
throw new DdlException("Not implemented");
|
||||
}
|
||||
}
|
||||
|
||||
// get all colocate meta
|
||||
public static class ColocateMetaAction extends ColocateMetaBaseAction {
|
||||
ColocateMetaAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
ColocateMetaAction action = new ColocateMetaAction(controller);
|
||||
controller.registerHandler(HttpMethod.GET, "/api/colocate", action);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeInMasterWithAdmin(BaseRequest request, BaseResponse response)
|
||||
throws DdlException {
|
||||
response.setContentType("application/json");
|
||||
RestResult result = new RestResult();
|
||||
result.addResultEntry("colocate_meta", Catalog.getCurrentColocateIndex());
|
||||
sendResult(request, response, result);
|
||||
}
|
||||
}
|
||||
|
||||
// mark a colocate group as stable or unstable
|
||||
public static class MarkGroupStableAction extends ColocateMetaBaseAction {
|
||||
MarkGroupStableAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
MarkGroupStableAction action = new MarkGroupStableAction(controller);
|
||||
controller.registerHandler(HttpMethod.POST, "/api/colocate/group_stable", action);
|
||||
controller.registerHandler(HttpMethod.DELETE, "/api/colocate/group_stable", action);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeInMasterWithAdmin(BaseRequest request, BaseResponse response)
|
||||
throws DdlException {
|
||||
GroupId groupId = checkAndGetGroupId(request);
|
||||
|
||||
HttpMethod method = request.getRequest().method();
|
||||
if (method.equals(HttpMethod.POST)) {
|
||||
colocateIndex.markGroupUnstable(groupId, "mark unstable via http api", true);
|
||||
} else if (method.equals(HttpMethod.DELETE)) {
|
||||
colocateIndex.markGroupStable(groupId, true);
|
||||
} else {
|
||||
response.appendContent(new RestBaseResult("HTTP method is not allowed.").toJson());
|
||||
writeResponse(request, response, HttpResponseStatus.METHOD_NOT_ALLOWED);
|
||||
}
|
||||
|
||||
sendResult(request, response);
|
||||
}
|
||||
}
|
||||
|
||||
// update a backendsPerBucketSeq meta for a colocate group
|
||||
public static class BucketSeqAction extends ColocateMetaBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(BucketSeqAction.class);
|
||||
|
||||
BucketSeqAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
BucketSeqAction action = new BucketSeqAction(controller);
|
||||
controller.registerHandler(HttpMethod.POST, "/api/colocate/bucketseq", action);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeInMasterWithAdmin(BaseRequest request, BaseResponse response)
|
||||
throws DdlException {
|
||||
final String clusterName = ConnectContext.get().getClusterName();
|
||||
if (Strings.isNullOrEmpty(clusterName)) {
|
||||
throw new DdlException("No cluster selected.");
|
||||
}
|
||||
GroupId groupId = checkAndGetGroupId(request);
|
||||
|
||||
String meta = request.getContent();
|
||||
Type type = new TypeToken<List<List<Long>>>() {}.getType();
|
||||
List<List<Long>> backendsPerBucketSeq = new Gson().fromJson(meta, type);
|
||||
LOG.info("get buckets sequence: {}", backendsPerBucketSeq);
|
||||
|
||||
ColocateGroupSchema groupSchema = Catalog.getCurrentColocateIndex().getGroupSchema(groupId);
|
||||
if (backendsPerBucketSeq.size() != groupSchema.getBucketsNum()) {
|
||||
throw new DdlException("Invalid bucket num. expected: " + groupSchema.getBucketsNum() + ", actual: "
|
||||
+ backendsPerBucketSeq.size());
|
||||
}
|
||||
|
||||
List<Long> clusterBackendIds = Catalog.getCurrentSystemInfo().getClusterBackendIds(clusterName, true);
|
||||
//check the Backend id
|
||||
for (List<Long> backendIds : backendsPerBucketSeq) {
|
||||
if (backendIds.size() != groupSchema.getReplicaAlloc().getTotalReplicaNum()) {
|
||||
throw new DdlException("Invalid backend num per bucket. expected: "
|
||||
+ groupSchema.getReplicaAlloc().getTotalReplicaNum() + ", actual: " + backendIds.size());
|
||||
}
|
||||
for (Long beId : backendIds) {
|
||||
if (!clusterBackendIds.contains(beId)) {
|
||||
throw new DdlException("The backend " + beId + " does not exist or not available");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int bucketsNum = colocateIndex.getBackendsPerBucketSeq(groupId).size();
|
||||
Preconditions.checkState(backendsPerBucketSeq.size() == bucketsNum,
|
||||
backendsPerBucketSeq.size() + " vs. " + bucketsNum);
|
||||
updateBackendPerBucketSeq(groupId, backendsPerBucketSeq);
|
||||
LOG.info("the group {} backendsPerBucketSeq meta has been changed to {}", groupId, backendsPerBucketSeq);
|
||||
|
||||
sendResult(request, response);
|
||||
}
|
||||
|
||||
private void updateBackendPerBucketSeq(GroupId groupId, List<List<Long>> backendsPerBucketSeq)
|
||||
throws DdlException {
|
||||
throw new DdlException("Currently not support");
|
||||
/*
|
||||
colocateIndex.addBackendsPerBucketSeq(groupId, backendsPerBucketSeq);
|
||||
ColocatePersistInfo info2 = ColocatePersistInfo.createForBackendsPerBucketSeq(groupId, backendsPerBucketSeq);
|
||||
Catalog.getCurrentCatalog().getEditLog().logColocateBackendsPerBucketSeq(info2);
|
||||
*/
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,26 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.meta;
|
||||
|
||||
import org.apache.doris.common.DdlException;
|
||||
|
||||
public class InvalidClientException extends DdlException {
|
||||
public InvalidClientException(String msg) {
|
||||
super(msg);
|
||||
}
|
||||
}
|
||||
@ -1,108 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.meta;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.action.WebBaseAction;
|
||||
import org.apache.doris.master.MetaHelper;
|
||||
import org.apache.doris.system.Frontend;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
|
||||
public class MetaBaseAction extends WebBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(MetaBaseAction.class);
|
||||
private static String CONTENT_DISPOSITION = "Content-disposition";
|
||||
|
||||
public static final String CLUSTER_ID = "cluster_id";
|
||||
public static final String TOKEN = "token";
|
||||
|
||||
protected File imageDir;
|
||||
|
||||
public MetaBaseAction(ActionController controller, File imageDir) {
|
||||
super(controller);
|
||||
this.imageDir = imageDir;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean needAdmin() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean needPassword() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(BaseRequest request, BaseResponse response) {
|
||||
if (needCheckClientIsFe()) {
|
||||
try {
|
||||
checkFromValidFe(request, response);
|
||||
} catch (InvalidClientException e) {
|
||||
response.appendContent("invalid client host.");
|
||||
writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
}
|
||||
super.execute(request, response);
|
||||
}
|
||||
|
||||
protected boolean needCheckClientIsFe() {
|
||||
return true;
|
||||
}
|
||||
|
||||
protected void writeFileResponse(BaseRequest request, BaseResponse response, File file) {
|
||||
if (file == null || !file.exists()) {
|
||||
response.appendContent("File does not exist.");
|
||||
writeResponse(request, response, HttpResponseStatus.NOT_FOUND);
|
||||
return;
|
||||
}
|
||||
|
||||
// add custom header
|
||||
response.updateHeader(CONTENT_DISPOSITION, "attachment; filename=" + file.getName());
|
||||
response.updateHeader(MetaHelper.X_IMAGE_SIZE, String.valueOf(file.length()));
|
||||
|
||||
writeObjectResponse(request, response, HttpResponseStatus.OK, file, file.getName(), true);
|
||||
return;
|
||||
}
|
||||
|
||||
private boolean isFromValidFe(BaseRequest request) {
|
||||
String clientHost = request.getHostString();
|
||||
Frontend fe = Catalog.getCurrentCatalog().getFeByHost(clientHost);
|
||||
if (fe == null) {
|
||||
LOG.warn("request is not from valid FE. client: {}", clientHost);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private void checkFromValidFe(BaseRequest request, BaseResponse response)
|
||||
throws InvalidClientException {
|
||||
if (!isFromValidFe(request)) {
|
||||
throw new InvalidClientException("invalid client host");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,356 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.meta;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.ha.FrontendNodeType;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.master.MetaHelper;
|
||||
import org.apache.doris.persist.MetaCleaner;
|
||||
import org.apache.doris.persist.Storage;
|
||||
import org.apache.doris.persist.StorageInfo;
|
||||
import org.apache.doris.system.Frontend;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.gson.Gson;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
|
||||
public class MetaService {
|
||||
private static final int TIMEOUT_SECOND = 10;
|
||||
|
||||
public static class ImageAction extends MetaBaseAction {
|
||||
private static final String VERSION = "version";
|
||||
|
||||
public ImageAction(ActionController controller, File imageDir) {
|
||||
super(controller, imageDir);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller, File imageDir)
|
||||
throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/image", new ImageAction(controller, imageDir));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
String versionStr = request.getSingleParameter(VERSION);
|
||||
if (Strings.isNullOrEmpty(versionStr)) {
|
||||
response.appendContent("Miss version parameter");
|
||||
writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
|
||||
long version = checkLongParam(versionStr);
|
||||
if (version < 0) {
|
||||
writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
|
||||
File imageFile = Storage.getImageFile(imageDir, version);
|
||||
if (!imageFile.exists()) {
|
||||
writeResponse(request, response, HttpResponseStatus.NOT_FOUND);
|
||||
return;
|
||||
}
|
||||
|
||||
writeFileResponse(request, response, imageFile);
|
||||
}
|
||||
}
|
||||
|
||||
public static class InfoAction extends MetaBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(InfoAction.class);
|
||||
|
||||
public InfoAction(ActionController controller, File imageDir) {
|
||||
super(controller, imageDir);
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller, File imageDir)
|
||||
throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/info", new InfoAction(controller, imageDir));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
try {
|
||||
Storage currentStorageInfo = new Storage(imageDir.getAbsolutePath());
|
||||
StorageInfo storageInfo = new StorageInfo(currentStorageInfo.getClusterID(),
|
||||
currentStorageInfo.getImageSeq(), currentStorageInfo.getEditsSeq());
|
||||
|
||||
response.setContentType("application/json");
|
||||
Gson gson = new Gson();
|
||||
response.appendContent(gson.toJson(storageInfo));
|
||||
writeResponse(request, response);
|
||||
return;
|
||||
} catch (IOException e) {
|
||||
LOG.warn("IO error.", e);
|
||||
response.appendContent("failed to get master info.");
|
||||
writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static class VersionAction extends MetaBaseAction {
|
||||
public VersionAction(ActionController controller, File imageDir) {
|
||||
super(controller, imageDir);
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller, File imageDir)
|
||||
throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/version", new VersionAction(controller, imageDir));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
File versionFile = new File(imageDir, Storage.VERSION_FILE);
|
||||
writeFileResponse(request, response, versionFile);
|
||||
}
|
||||
}
|
||||
|
||||
public static class PutAction extends MetaBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(PutAction.class);
|
||||
|
||||
private static final String VERSION = "version";
|
||||
private static final String PORT = "port";
|
||||
|
||||
public PutAction(ActionController controller, File imageDir) {
|
||||
super(controller, imageDir);
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller, File imageDir)
|
||||
throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/put", new PutAction(controller, imageDir));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
String machine = request.getHostString();
|
||||
String portStr = request.getSingleParameter(PORT);
|
||||
// check port to avoid SSRF(Server-Side Request Forgery)
|
||||
if (Strings.isNullOrEmpty(portStr)) {
|
||||
writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
{
|
||||
int port = Integer.parseInt(portStr);
|
||||
if (port < 0 || port > 65535) {
|
||||
LOG.warn("port is invalid. port={}", port);
|
||||
writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
String versionStr = request.getSingleParameter(VERSION);
|
||||
if (Strings.isNullOrEmpty(versionStr)) {
|
||||
response.appendContent("Miss version parameter");
|
||||
writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
|
||||
long version = checkLongParam(versionStr);
|
||||
|
||||
// for master node, reject image put
|
||||
if (Catalog.getCurrentCatalog().isMaster()) {
|
||||
response.appendContent("this node is master, reject image put");
|
||||
writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
LOG.error("this node is master, but receive image put from host {}, reject it", machine);
|
||||
return;
|
||||
}
|
||||
|
||||
// do not accept image whose version is bigger than max journalId
|
||||
// if accepted, newly added log will not be replayed when restart
|
||||
long maxJournalId = Catalog.getCurrentCatalog().getMaxJournalId();
|
||||
if (version > maxJournalId) {
|
||||
response.appendContent("image version [" + version + "] is bigger than local max journal id ["
|
||||
+ maxJournalId + "], reject image put");
|
||||
writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
LOG.error("receive image whose version [{}] is bigger than local max journal id [{}], reject it",
|
||||
version, maxJournalId);
|
||||
return;
|
||||
}
|
||||
|
||||
String url = "http://" + machine + ":" + portStr
|
||||
+ "/image?version=" + versionStr;
|
||||
String filename = Storage.IMAGE + "." + versionStr;
|
||||
|
||||
File dir = new File(Catalog.getCurrentCatalog().getImageDir());
|
||||
try {
|
||||
OutputStream out = MetaHelper.getOutputStream(filename, dir);
|
||||
MetaHelper.getRemoteFile(url, TIMEOUT_SECOND * 1000, out);
|
||||
MetaHelper.complete(filename, dir);
|
||||
writeResponse(request, response);
|
||||
} catch (FileNotFoundException e) {
|
||||
LOG.warn("file not found. file: {}", filename, e);
|
||||
writeResponse(request, response, HttpResponseStatus.NOT_FOUND);
|
||||
return;
|
||||
} catch (IOException e) {
|
||||
LOG.warn("failed to get remote file. url: {}", url, e);
|
||||
writeResponse(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR);
|
||||
return;
|
||||
}
|
||||
|
||||
// Delete old image files
|
||||
MetaCleaner cleaner = new MetaCleaner(Config.meta_dir + "/image");
|
||||
try {
|
||||
cleaner.clean();
|
||||
} catch (IOException e) {
|
||||
LOG.error("Follower/Observer delete old image file fail.", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static class JournalIdAction extends MetaBaseAction {
|
||||
public JournalIdAction(ActionController controller, File imageDir) {
|
||||
super(controller, imageDir);
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller, File imageDir)
|
||||
throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/journal_id", new JournalIdAction(controller, imageDir));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
long id = Catalog.getCurrentCatalog().getReplayedJournalId();
|
||||
response.updateHeader("id", Long.toString(id));
|
||||
writeResponse(request, response);
|
||||
}
|
||||
}
|
||||
|
||||
public static class RoleAction extends MetaBaseAction {
|
||||
private static final String HOST = "host";
|
||||
private static final String PORT = "port";
|
||||
|
||||
public RoleAction(ActionController controller, File imageDir) {
|
||||
super(controller, imageDir);
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller, File imageDir)
|
||||
throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/role", new RoleAction(controller, imageDir));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
String host = request.getSingleParameter(HOST);
|
||||
String portString = request.getSingleParameter(PORT);
|
||||
|
||||
if (!Strings.isNullOrEmpty(host) && !Strings.isNullOrEmpty(portString)) {
|
||||
int port = Integer.parseInt(portString);
|
||||
Frontend fe = Catalog.getCurrentCatalog().checkFeExist(host, port);
|
||||
if (fe == null) {
|
||||
response.updateHeader("role", FrontendNodeType.UNKNOWN.name());
|
||||
} else {
|
||||
response.updateHeader("role", fe.getRole().name());
|
||||
response.updateHeader("name", fe.getNodeName());
|
||||
}
|
||||
writeResponse(request, response);
|
||||
} else {
|
||||
response.appendContent("Miss parameter");
|
||||
writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* This action is used to get the electable_nodes config and the cluster id of
|
||||
* the fe with the given ip and port. When one frontend start, it should check
|
||||
* the local electable_nodes config and local cluster id with other frontends.
|
||||
* If there is any difference, local fe will exit. This is designed to protect
|
||||
* the consistency of the cluster.
|
||||
*/
|
||||
public static class CheckAction extends MetaBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(CheckAction.class);
|
||||
|
||||
public CheckAction(ActionController controller, File imageDir) {
|
||||
super(controller, imageDir);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller, File imageDir)
|
||||
throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/check",
|
||||
new CheckAction(controller, imageDir));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
try {
|
||||
Storage storage = new Storage(imageDir.getAbsolutePath());
|
||||
response.updateHeader(MetaBaseAction.CLUSTER_ID, Integer.toString(storage.getClusterID()));
|
||||
response.updateHeader(MetaBaseAction.TOKEN, storage.getToken());
|
||||
} catch (IOException e) {
|
||||
LOG.error(e);
|
||||
}
|
||||
writeResponse(request, response);
|
||||
}
|
||||
}
|
||||
|
||||
public static class DumpAction extends MetaBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(CheckAction.class);
|
||||
|
||||
public DumpAction(ActionController controller, File imageDir) {
|
||||
super(controller, imageDir);
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller, File imageDir)
|
||||
throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/dump", new DumpAction(controller, imageDir));
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean needAdmin() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean needCheckClientIsFe() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeGet(BaseRequest request, BaseResponse response) {
|
||||
/*
|
||||
* Before dump, we acquired the catalog read lock and all databases' read lock and all
|
||||
* the jobs' read lock. This will guarantee the consistency of database and job queues.
|
||||
* But Backend may still inconsistent.
|
||||
*/
|
||||
|
||||
// TODO: Still need to lock ClusterInfoService to prevent add or drop Backends
|
||||
String dumpFilePath = Catalog.getCurrentCatalog().dumpImage();
|
||||
if (dumpFilePath == null) {
|
||||
response.appendContent("dump failed. " + dumpFilePath);
|
||||
}
|
||||
|
||||
response.appendContent("dump finished. " + dumpFilePath);
|
||||
writeResponse(request, response);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,162 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.gson.Gson;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import org.apache.doris.common.Version;
|
||||
|
||||
/*
|
||||
* fe_host:fe_http_port/api/bootstrap
|
||||
* return:
|
||||
* {"status":"OK","msg":"Success","replayedJournal"=123456, "queryPort"=9000, "rpcPort"=9001}
|
||||
* {"status":"FAILED","msg":"err info..."}
|
||||
*/
|
||||
public class BootstrapFinishAction extends RestBaseAction {
|
||||
private static final String CLUSTER_ID = "cluster_id";
|
||||
private static final String TOKEN = "token";
|
||||
|
||||
public static final String REPLAYED_JOURNAL_ID = "replayedJournalId";
|
||||
public static final String QUERY_PORT = "queryPort";
|
||||
public static final String RPC_PORT = "rpcPort";
|
||||
public static final String VERSION = "version";
|
||||
|
||||
public BootstrapFinishAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/api/bootstrap", new BootstrapFinishAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(BaseRequest request, BaseResponse response) throws DdlException {
|
||||
boolean isReady = Catalog.getCurrentCatalog().isReady();
|
||||
|
||||
// to json response
|
||||
BootstrapResult result = null;
|
||||
if (isReady) {
|
||||
result = new BootstrapResult();
|
||||
String clusterIdStr = request.getSingleParameter(CLUSTER_ID);
|
||||
String token = request.getSingleParameter(TOKEN);
|
||||
if (!Strings.isNullOrEmpty(clusterIdStr) && !Strings.isNullOrEmpty(token)) {
|
||||
// cluster id or token is provided, return more info
|
||||
int clusterId = 0;
|
||||
try {
|
||||
clusterId = Integer.valueOf(clusterIdStr);
|
||||
} catch (NumberFormatException e) {
|
||||
result.status = ActionStatus.FAILED;
|
||||
result.msg = "invalid cluster id format: " + clusterIdStr;
|
||||
}
|
||||
|
||||
if (result.status == ActionStatus.OK) {
|
||||
if (clusterId != Catalog.getCurrentCatalog().getClusterId()) {
|
||||
result.status = ActionStatus.FAILED;
|
||||
result.msg = "invalid cluster id: " + Catalog.getCurrentCatalog().getClusterId();
|
||||
}
|
||||
}
|
||||
|
||||
if (result.status == ActionStatus.OK) {
|
||||
if (!token.equals(Catalog.getCurrentCatalog().getToken())) {
|
||||
result.status = ActionStatus.FAILED;
|
||||
result.msg = "invalid token: " + Catalog.getCurrentCatalog().getToken();
|
||||
}
|
||||
}
|
||||
|
||||
if (result.status == ActionStatus.OK) {
|
||||
// cluster id and token are valid, return replayed journal id
|
||||
long replayedJournalId = Catalog.getCurrentCatalog().getReplayedJournalId();
|
||||
result.setMaxReplayedJournal(replayedJournalId);
|
||||
result.setQueryPort(Config.query_port);
|
||||
result.setRpcPort(Config.rpc_port);
|
||||
result.setVersion(Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
result = new BootstrapResult("not ready");
|
||||
}
|
||||
|
||||
// send result
|
||||
response.setContentType("application/json");
|
||||
response.getContent().append(result.toJson());
|
||||
sendResult(request, response);
|
||||
}
|
||||
|
||||
public static class BootstrapResult extends RestBaseResult {
|
||||
private long replayedJournalId = 0;
|
||||
private int queryPort = 0;
|
||||
private int rpcPort = 0;
|
||||
private String version = "";
|
||||
|
||||
public BootstrapResult() {
|
||||
super();
|
||||
}
|
||||
|
||||
public BootstrapResult(String msg) {
|
||||
super(msg);
|
||||
}
|
||||
|
||||
public void setMaxReplayedJournal(long replayedJournalId) {
|
||||
this.replayedJournalId = replayedJournalId;
|
||||
}
|
||||
|
||||
public long getMaxReplayedJournal() {
|
||||
return replayedJournalId;
|
||||
}
|
||||
|
||||
public void setQueryPort(int queryPort) {
|
||||
this.queryPort = queryPort;
|
||||
}
|
||||
|
||||
public int getQueryPort() {
|
||||
return queryPort;
|
||||
}
|
||||
|
||||
public void setRpcPort(int rpcPort) {
|
||||
this.rpcPort = rpcPort;
|
||||
}
|
||||
|
||||
public int getRpcPort() {
|
||||
return rpcPort;
|
||||
}
|
||||
|
||||
public String getVersion() {
|
||||
return version;
|
||||
}
|
||||
|
||||
public void setVersion(String version) {
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toJson() {
|
||||
Gson gson = new Gson();
|
||||
return gson.toJson(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,83 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.catalog.Database;
|
||||
import org.apache.doris.cluster.ClusterNamespace;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.common.UserException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
public class CancelStreamLoad extends RestBaseAction {
|
||||
public CancelStreamLoad(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller)
|
||||
throws IllegalArgException {
|
||||
CancelStreamLoad action = new CancelStreamLoad(controller);
|
||||
controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_cancel", action);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
|
||||
|
||||
if (redirectToMaster(request, response)) {
|
||||
return;
|
||||
}
|
||||
|
||||
final String clusterName = ConnectContext.get().getClusterName();
|
||||
if (Strings.isNullOrEmpty(clusterName)) {
|
||||
throw new DdlException("No cluster selected.");
|
||||
}
|
||||
|
||||
String dbName = request.getSingleParameter(DB_KEY);
|
||||
if (Strings.isNullOrEmpty(dbName)) {
|
||||
throw new DdlException("No database selected.");
|
||||
}
|
||||
|
||||
String fullDbName = ClusterNamespace.getFullName(clusterName, dbName);
|
||||
|
||||
String label = request.getSingleParameter(LABEL_KEY);
|
||||
if (Strings.isNullOrEmpty(label)) {
|
||||
throw new DdlException("No label selected.");
|
||||
}
|
||||
|
||||
// FIXME(cmy)
|
||||
// checkWritePriv(authInfo.fullUserName, fullDbName);
|
||||
|
||||
Database db = Catalog.getCurrentCatalog().getDbOrDdlException(fullDbName);
|
||||
|
||||
try {
|
||||
Catalog.getCurrentGlobalTransactionMgr().abortTransaction(db.getId(), label, "user cancel");
|
||||
} catch (UserException e) {
|
||||
throw new DdlException(e.getMessage());
|
||||
}
|
||||
|
||||
sendResult(request, response, new RestBaseResult());
|
||||
}
|
||||
}
|
||||
@ -1,93 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.alter.SystemHandler;
|
||||
import org.apache.doris.common.AnalysisException;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.common.Pair;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.system.SystemInfoService;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
/*
|
||||
* calc row count from replica to table
|
||||
* fe_host:fe_http_port/api/check_decommission?host_ports=host:port,host2:port2...
|
||||
* return:
|
||||
* {"status":"OK","msg":"Success"}
|
||||
* {"status":"FAILED","msg":"err info..."}
|
||||
*/
|
||||
public class CheckDecommissionAction extends RestBaseAction {
|
||||
public static final String HOST_PORTS = "host_ports";
|
||||
|
||||
public CheckDecommissionAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/api/check_decommission", new CheckDecommissionAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeWithoutPassword(BaseRequest request, BaseResponse response)
|
||||
throws DdlException {
|
||||
checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.OPERATOR);
|
||||
|
||||
String hostPorts = request.getSingleParameter(HOST_PORTS);
|
||||
if (Strings.isNullOrEmpty(hostPorts)) {
|
||||
throw new DdlException("No host:port specified.");
|
||||
}
|
||||
|
||||
String[] hostPortArr = hostPorts.split(",");
|
||||
if (hostPortArr.length == 0) {
|
||||
throw new DdlException("No host:port specified.");
|
||||
}
|
||||
|
||||
List<Pair<String, Integer>> hostPortPairs = Lists.newArrayList();
|
||||
for (String hostPort : hostPortArr) {
|
||||
Pair<String, Integer> pair;
|
||||
try {
|
||||
pair = SystemInfoService.validateHostAndPort(hostPort);
|
||||
} catch (AnalysisException e) {
|
||||
throw new DdlException(e.getMessage());
|
||||
}
|
||||
hostPortPairs.add(pair);
|
||||
}
|
||||
|
||||
SystemHandler.checkDecommission(hostPortPairs);
|
||||
|
||||
// to json response
|
||||
RestBaseResult result = new RestBaseResult();
|
||||
|
||||
// send result
|
||||
response.setContentType("application/json");
|
||||
response.getContent().append(result.toJson());
|
||||
sendResult(request, response);
|
||||
}
|
||||
}
|
||||
@ -1,62 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
import org.apache.doris.common.util.DebugUtil;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.service.ExecuteEnv;
|
||||
|
||||
// This class is used to get current query_id of connection_id.
|
||||
// Every connection holds at most one query at every point.
|
||||
// Some we can get query_id firstly, and get query by query_id.
|
||||
public class ConnectionAction extends RestBaseAction {
|
||||
public ConnectionAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/api/connection", new ConnectionAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(BaseRequest request, BaseResponse response) {
|
||||
String connStr = request.getSingleParameter("connection_id");
|
||||
if (connStr == null) {
|
||||
response.getContent().append("not valid parameter");
|
||||
sendResult(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
int connectionId = Integer.valueOf(connStr.trim());
|
||||
ConnectContext context = ExecuteEnv.getInstance().getScheduler().getContext(connectionId);
|
||||
if (context == null || context.queryId() == null) {
|
||||
response.getContent().append("connection id " + connectionId + " not found.");
|
||||
sendResult(request, response, HttpResponseStatus.NOT_FOUND);
|
||||
return;
|
||||
}
|
||||
String queryId = DebugUtil.printId(context.queryId());
|
||||
response.setContentType("application/json");
|
||||
response.getContent().append("{\"query_id\" : \"" + queryId + "\"}");
|
||||
sendResult(request, response);
|
||||
}
|
||||
}
|
||||
@ -1,108 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.catalog.Database;
|
||||
import org.apache.doris.catalog.Table;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Maps;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.codehaus.jackson.map.ObjectMapper;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
/*
|
||||
* used to get a table's ddl stmt
|
||||
* eg:
|
||||
* fe_host:http_port/api/_get_ddl?db=xxx&tbl=yyy
|
||||
*/
|
||||
public class GetDdlStmtAction extends RestBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(GetDdlStmtAction.class);
|
||||
private static final String DB_PARAM = "db";
|
||||
private static final String TABLE_PARAM = "tbl";
|
||||
|
||||
public GetDdlStmtAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
GetDdlStmtAction action = new GetDdlStmtAction(controller);
|
||||
controller.registerHandler(HttpMethod.GET, "/api/_get_ddl", action);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeWithoutPassword(BaseRequest request, BaseResponse response)
|
||||
throws DdlException {
|
||||
checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
|
||||
|
||||
String dbName = request.getSingleParameter(DB_PARAM);
|
||||
String tableName = request.getSingleParameter(TABLE_PARAM);
|
||||
|
||||
if (Strings.isNullOrEmpty(dbName) || Strings.isNullOrEmpty(tableName)) {
|
||||
throw new DdlException("Missing params. Need database name and Table name");
|
||||
}
|
||||
|
||||
Database db = Catalog.getCurrentCatalog().getDbOrDdlException(dbName);
|
||||
Table table = db.getTableOrDdlException(tableName);
|
||||
|
||||
List<String> createTableStmt = Lists.newArrayList();
|
||||
List<String> addPartitionStmt = Lists.newArrayList();
|
||||
List<String> createRollupStmt = Lists.newArrayList();
|
||||
|
||||
table.readLock();
|
||||
try {
|
||||
Catalog.getDdlStmt(table, createTableStmt, addPartitionStmt, createRollupStmt, true, false /* show password */);
|
||||
} finally {
|
||||
table.readUnlock();
|
||||
}
|
||||
|
||||
Map<String, List<String>> results = Maps.newHashMap();
|
||||
results.put("TABLE", createTableStmt);
|
||||
results.put("PARTITION", addPartitionStmt);
|
||||
results.put("ROLLUP", createRollupStmt);
|
||||
|
||||
// to json response
|
||||
String result = "";
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
try {
|
||||
result = mapper.writeValueAsString(results);
|
||||
} catch (Exception e) {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
// send result
|
||||
response.setContentType("application/json");
|
||||
response.getContent().append(result);
|
||||
sendResult(request, response);
|
||||
}
|
||||
}
|
||||
@ -1,88 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.common.MetaNotFoundException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.load.Load;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
// Get load information of one load job
|
||||
public class GetLoadInfoAction extends RestBaseAction {
|
||||
public GetLoadInfoAction(ActionController controller, boolean isStreamLoad) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller)
|
||||
throws IllegalArgException {
|
||||
GetLoadInfoAction action = new GetLoadInfoAction(controller, false);
|
||||
controller.registerHandler(HttpMethod.GET, "/api/{" + DB_KEY + "}/_load_info", action);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeWithoutPassword(BaseRequest request, BaseResponse response)
|
||||
throws DdlException {
|
||||
Load.JobInfo info = new Load.JobInfo(request.getSingleParameter(DB_KEY),
|
||||
request.getSingleParameter(LABEL_KEY),
|
||||
ConnectContext.get().getClusterName());
|
||||
if (Strings.isNullOrEmpty(info.dbName)) {
|
||||
throw new DdlException("No database selected");
|
||||
}
|
||||
if (Strings.isNullOrEmpty(info.label)) {
|
||||
throw new DdlException("No label selected");
|
||||
}
|
||||
if (Strings.isNullOrEmpty(info.clusterName)) {
|
||||
throw new DdlException("No cluster name selected");
|
||||
}
|
||||
|
||||
if (redirectToMaster(request, response)) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
catalog.getLoadInstance().getJobInfo(info);
|
||||
if (info.tblNames.isEmpty()) {
|
||||
checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), info.dbName, PrivPredicate.LOAD);
|
||||
} else {
|
||||
for (String tblName : info.tblNames) {
|
||||
checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), info.dbName, tblName,
|
||||
PrivPredicate.LOAD);
|
||||
}
|
||||
}
|
||||
} catch (DdlException | MetaNotFoundException e) {
|
||||
catalog.getLoadManager().getLoadJobInfo(info);
|
||||
}
|
||||
sendResult(request, response, new Result(info));
|
||||
}
|
||||
|
||||
private static class Result extends RestBaseResult {
|
||||
private Load.JobInfo jobInfo;
|
||||
public Result(Load.JobInfo info) {
|
||||
jobInfo = info;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,131 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.Maps;
|
||||
import com.google.common.collect.Sets;
|
||||
|
||||
import org.codehaus.jackson.map.ObjectMapper;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
|
||||
/*
|
||||
* get log file infos:
|
||||
* curl -I http://fe_host:http_port/api/get_log_file?type=fe.audit.log
|
||||
* return:
|
||||
* HTTP/1.1 200 OK
|
||||
* file_infos: {"fe.audit.log":24759,"fe.audit.log.20190528.1":132934}
|
||||
* content-type: text/html
|
||||
* connection: keep-alive
|
||||
*
|
||||
* get log file:
|
||||
* curl -X GET http://fe_host:http_port/api/get_log_file?type=fe.audit.log&file=fe.audit.log.20190528.1
|
||||
*/
|
||||
public class GetLogFileAction extends RestBaseAction {
|
||||
private final Set<String> logFileTypes = Sets.newHashSet("fe.audit.log");
|
||||
|
||||
public GetLogFileAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/api/get_log_file", new GetLogFileAction(controller));
|
||||
controller.registerHandler(HttpMethod.HEAD, "/api/get_log_file", new GetLogFileAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeWithoutPassword(BaseRequest request, BaseResponse response) {
|
||||
String logType = request.getSingleParameter("type");
|
||||
String logFile = request.getSingleParameter("file");
|
||||
|
||||
// check param empty
|
||||
if (Strings.isNullOrEmpty(logType)) {
|
||||
response.appendContent("Miss type parameter");
|
||||
writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
|
||||
// check type valid or not
|
||||
if (!logFileTypes.contains(logType)) {
|
||||
response.appendContent("log type: " + logType + " is invalid!");
|
||||
writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
|
||||
HttpMethod method = request.getRequest().method();
|
||||
if (method.equals(HttpMethod.HEAD)) {
|
||||
String fileInfos = getFileInfos(logType);
|
||||
response.updateHeader("file_infos", fileInfos);
|
||||
writeResponse(request, response, HttpResponseStatus.OK);
|
||||
return;
|
||||
} else if (method.equals(HttpMethod.GET)) {
|
||||
File log = getLogFile(logType, logFile);
|
||||
if (!log.exists() || !log.isFile()) {
|
||||
response.appendContent("Log file not exist: " + log.getName());
|
||||
writeResponse(request, response, HttpResponseStatus.NOT_FOUND);
|
||||
return;
|
||||
}
|
||||
writeObjectResponse(request, response, HttpResponseStatus.OK, log, log.getName(), true);
|
||||
} else {
|
||||
response.appendContent(new RestBaseResult("HTTP method is not allowed.").toJson());
|
||||
writeResponse(request, response, HttpResponseStatus.METHOD_NOT_ALLOWED);
|
||||
}
|
||||
}
|
||||
|
||||
private String getFileInfos(String logType) {
|
||||
Map<String, Long> fileInfos = Maps.newTreeMap();
|
||||
if (logType.equals("fe.audit.log")) {
|
||||
File logDir = new File(Config.audit_log_dir);
|
||||
File[] files = logDir.listFiles();
|
||||
for (int i = 0; i < files.length; i++) {
|
||||
if (files[i].isFile() && files[i].getName().startsWith("fe.audit.log")) {
|
||||
fileInfos.put(files[i].getName(), files[i].length());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
String result = "";
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
try {
|
||||
result = mapper.writeValueAsString(fileInfos);
|
||||
} catch (Exception e) {
|
||||
// do nothing
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private File getLogFile(String logType, String logFile) {
|
||||
String logPath = "";
|
||||
if ("fe.audit.log".equals(logType)) {
|
||||
logPath = Config.audit_log_dir + "/" + logFile;
|
||||
}
|
||||
return new File(logPath);
|
||||
}
|
||||
}
|
||||
@ -1,92 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.common.util.SmallFileMgr;
|
||||
import org.apache.doris.common.util.SmallFileMgr.SmallFile;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
|
||||
public class GetSmallFileAction extends RestBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(GetSmallFileAction.class);
|
||||
|
||||
public GetSmallFileAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/api/get_small_file", new GetSmallFileAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(BaseRequest request, BaseResponse response) {
|
||||
String token = request.getSingleParameter("token");
|
||||
String fileIdStr = request.getSingleParameter("file_id");
|
||||
|
||||
// check param empty
|
||||
if (Strings.isNullOrEmpty(token) || Strings.isNullOrEmpty(fileIdStr)) {
|
||||
response.appendContent("Missing parameter");
|
||||
writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
|
||||
// check token
|
||||
if (!token.equals(Catalog.getCurrentCatalog().getToken())) {
|
||||
response.appendContent("Invalid token");
|
||||
writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
|
||||
long fileId = -1;
|
||||
try {
|
||||
fileId = Long.valueOf(fileIdStr);
|
||||
} catch (NumberFormatException e) {
|
||||
response.appendContent("Invalid file id format: " + fileIdStr);
|
||||
writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
|
||||
SmallFileMgr fileMgr = Catalog.getCurrentCatalog().getSmallFileMgr();
|
||||
SmallFile smallFile = fileMgr.getSmallFile(fileId);
|
||||
if (smallFile == null || !smallFile.isContent) {
|
||||
response.appendContent("File not found or is not content");
|
||||
writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
|
||||
HttpMethod method = request.getRequest().method();
|
||||
if (method.equals(HttpMethod.GET)) {
|
||||
writeObjectResponse(request, response, HttpResponseStatus.OK, smallFile.getContentBytes(),
|
||||
smallFile.name, true);
|
||||
} else {
|
||||
response.appendContent(new RestBaseResult("HTTP method is not allowed.").toJson());
|
||||
writeResponse(request, response, HttpResponseStatus.METHOD_NOT_ALLOWED);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,86 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.catalog.Database;
|
||||
import org.apache.doris.cluster.ClusterNamespace;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
public class GetStreamLoadState extends RestBaseAction {
|
||||
public GetStreamLoadState(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller)
|
||||
throws IllegalArgException {
|
||||
GetStreamLoadState action = new GetStreamLoadState(controller);
|
||||
controller.registerHandler(HttpMethod.GET, "/api/{" + DB_KEY + "}/get_load_state", action);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeWithoutPassword(BaseRequest request, BaseResponse response)
|
||||
throws DdlException {
|
||||
|
||||
if (redirectToMaster(request, response)) {
|
||||
return;
|
||||
}
|
||||
|
||||
final String clusterName = ConnectContext.get().getClusterName();
|
||||
if (Strings.isNullOrEmpty(clusterName)) {
|
||||
throw new DdlException("No cluster selected.");
|
||||
}
|
||||
|
||||
String dbName = request.getSingleParameter(DB_KEY);
|
||||
if (Strings.isNullOrEmpty(dbName)) {
|
||||
throw new DdlException("No database selected.");
|
||||
}
|
||||
|
||||
String fullDbName = ClusterNamespace.getFullName(clusterName, dbName);
|
||||
|
||||
String label = request.getSingleParameter(LABEL_KEY);
|
||||
if (Strings.isNullOrEmpty(label)) {
|
||||
throw new DdlException("No label selected.");
|
||||
}
|
||||
|
||||
// FIXME(cmy)
|
||||
// checkReadPriv(authInfo.fullUserName, fullDbName);
|
||||
|
||||
Database db = Catalog.getCurrentCatalog().getDbOrDdlException(fullDbName);
|
||||
|
||||
String state = Catalog.getCurrentGlobalTransactionMgr().getLabelState(db.getId(), label).toString();
|
||||
|
||||
sendResult(request, response, new Result(state));
|
||||
}
|
||||
|
||||
private static class Result extends RestBaseResult {
|
||||
private String state;
|
||||
public Result(String state) {
|
||||
this.state = state;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,47 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
public class HealthAction extends RestBaseAction {
|
||||
public HealthAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller)
|
||||
throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/api/health", new HealthAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(BaseRequest request, BaseResponse response) {
|
||||
response.setContentType("application/json");
|
||||
|
||||
RestResult result = new RestResult();
|
||||
result.addResultEntry("total_backend_num", Catalog.getCurrentSystemInfo().getBackendIds(false).size());
|
||||
result.addResultEntry("online_backend_num", Catalog.getCurrentSystemInfo().getBackendIds(true).size());
|
||||
sendResult(request, response, result);
|
||||
}
|
||||
}
|
||||
@ -1,141 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.cluster.ClusterNamespace;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.service.ExecuteEnv;
|
||||
import org.apache.doris.system.Backend;
|
||||
import org.apache.doris.system.SystemInfoService;
|
||||
import org.apache.doris.thrift.TNetworkAddress;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import io.netty.handler.codec.http.HttpHeaders;
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
public class LoadAction extends RestBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(LoadAction.class);
|
||||
|
||||
public static final String SUB_LABEL_NAME_PARAM = "sub_label";
|
||||
|
||||
private ExecuteEnv execEnv;
|
||||
private boolean isStreamLoad = false;
|
||||
|
||||
public LoadAction(ActionController controller, ExecuteEnv execEnv) {
|
||||
this(controller, execEnv, false);
|
||||
}
|
||||
|
||||
public LoadAction(ActionController controller, ExecuteEnv execEnv, boolean isStreamLoad) {
|
||||
super(controller);
|
||||
this.execEnv = execEnv;
|
||||
this.isStreamLoad = isStreamLoad;
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
ExecuteEnv execEnv = ExecuteEnv.getInstance();
|
||||
LoadAction action = new LoadAction(controller, execEnv);
|
||||
controller.registerHandler(HttpMethod.PUT,
|
||||
"/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_load", action);
|
||||
|
||||
controller.registerHandler(HttpMethod.PUT,
|
||||
"/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_stream_load",
|
||||
new LoadAction(controller, execEnv, true));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
|
||||
|
||||
// A 'Load' request must have 100-continue header
|
||||
if (!request.getRequest().headers().contains(HttpHeaders.Names.EXPECT)) {
|
||||
throw new DdlException("There is no 100-continue header");
|
||||
}
|
||||
|
||||
final String clusterName = ConnectContext.get().getClusterName();
|
||||
if (Strings.isNullOrEmpty(clusterName)) {
|
||||
throw new DdlException("No cluster selected.");
|
||||
}
|
||||
|
||||
String dbName = request.getSingleParameter(DB_KEY);
|
||||
if (Strings.isNullOrEmpty(dbName)) {
|
||||
throw new DdlException("No database selected.");
|
||||
}
|
||||
|
||||
String tableName = request.getSingleParameter(TABLE_KEY);
|
||||
if (Strings.isNullOrEmpty(tableName)) {
|
||||
throw new DdlException("No table selected.");
|
||||
}
|
||||
|
||||
String fullDbName = ClusterNamespace.getFullName(clusterName, dbName);
|
||||
|
||||
String label = request.getSingleParameter(LABEL_KEY);
|
||||
if (!isStreamLoad) {
|
||||
if (Strings.isNullOrEmpty(label)) {
|
||||
throw new DdlException("No label selected.");
|
||||
}
|
||||
} else {
|
||||
label = request.getRequest().headers().get(LABEL_KEY);
|
||||
}
|
||||
|
||||
// check auth
|
||||
checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tableName, PrivPredicate.LOAD);
|
||||
|
||||
TNetworkAddress redirectAddr;
|
||||
if (!isStreamLoad && !Strings.isNullOrEmpty(request.getSingleParameter(SUB_LABEL_NAME_PARAM))) {
|
||||
// only multi mini load need to redirect to Master, because only Master has the info of table to
|
||||
// the Backend which the file exists.
|
||||
if (redirectToMaster(request, response)) {
|
||||
return;
|
||||
}
|
||||
redirectAddr = execEnv.getMultiLoadMgr().redirectAddr(fullDbName, label);
|
||||
} else {
|
||||
// Choose a backend sequentially.
|
||||
SystemInfoService.BeAvailablePredicate beAvailablePredicate =
|
||||
new SystemInfoService.BeAvailablePredicate(false, false, true);
|
||||
List<Long> backendIds = Catalog.getCurrentSystemInfo().seqChooseBackendIdsByStorageMediumAndTag(
|
||||
1, beAvailablePredicate, false, clusterName, null, null);
|
||||
if (backendIds == null) {
|
||||
throw new DdlException(SystemInfoService.NO_BACKEND_LOAD_AVAILABLE_MSG);
|
||||
}
|
||||
|
||||
Backend backend = Catalog.getCurrentSystemInfo().getBackend(backendIds.get(0));
|
||||
if (backend == null) {
|
||||
throw new DdlException(SystemInfoService.NO_BACKEND_LOAD_AVAILABLE_MSG);
|
||||
}
|
||||
|
||||
redirectAddr = new TNetworkAddress(backend.getHost(), backend.getHttpPort());
|
||||
}
|
||||
|
||||
LOG.info("redirect load action to destination={}, stream: {}, db: {}, tbl: {}, label: {}",
|
||||
redirectAddr.toString(), isStreamLoad, dbName, tableName, label);
|
||||
redirectTo(request, response, redirectAddr);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,70 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
|
||||
import org.codehaus.jackson.map.ObjectMapper;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
/*
|
||||
* used to get meta replay info
|
||||
* eg:
|
||||
* fe_host:http_port/api/_meta_replay_state
|
||||
*/
|
||||
public class MetaReplayerCheckAction extends RestBaseAction {
|
||||
public MetaReplayerCheckAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
MetaReplayerCheckAction action = new MetaReplayerCheckAction(controller);
|
||||
controller.registerHandler(HttpMethod.GET, "/api/_meta_replay_state", action);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
|
||||
checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
|
||||
|
||||
Map<String, String> resultMap = Catalog.getCurrentCatalog().getMetaReplayState().getInfo();
|
||||
|
||||
// to json response
|
||||
String result = "";
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
try {
|
||||
result = mapper.writeValueAsString(resultMap);
|
||||
} catch (Exception e) {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
// send result
|
||||
response.setContentType("application/json");
|
||||
response.getContent().append(result);
|
||||
sendResult(request, response);
|
||||
}
|
||||
}
|
||||
@ -1,65 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.metric.MetricRepo;
|
||||
import org.apache.doris.metric.MetricVisitor;
|
||||
import org.apache.doris.metric.JsonMetricVisitor;
|
||||
import org.apache.doris.metric.PrometheusMetricVisitor;
|
||||
import org.apache.doris.metric.SimpleCoreMetricVisitor;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
//fehost:port/metrics
|
||||
//fehost:port/metrics?type=core
|
||||
//fehost:port/metrics?type=json
|
||||
public class MetricsAction extends RestBaseAction {
|
||||
|
||||
private static final String TYPE_PARAM = "type";
|
||||
|
||||
public MetricsAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/metrics", new MetricsAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(BaseRequest request, BaseResponse response) {
|
||||
String type = request.getSingleParameter(TYPE_PARAM);
|
||||
MetricVisitor visitor = null;
|
||||
if (!Strings.isNullOrEmpty(type) && type.equalsIgnoreCase("core")) {
|
||||
visitor = new SimpleCoreMetricVisitor("doris_fe");
|
||||
} else if (!Strings.isNullOrEmpty(type) && type.equalsIgnoreCase("json")) {
|
||||
visitor = new JsonMetricVisitor("doris_fe");
|
||||
} else {
|
||||
visitor = new PrometheusMetricVisitor("doris_fe");
|
||||
}
|
||||
|
||||
response.setContentType("text/plain");
|
||||
response.getContent().append(MetricRepo.getMetric(visitor));
|
||||
sendResult(request, response);
|
||||
}
|
||||
}
|
||||
@ -1,164 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.catalog.Database;
|
||||
import org.apache.doris.catalog.MaterializedIndex;
|
||||
import org.apache.doris.catalog.OlapTable;
|
||||
import org.apache.doris.catalog.Partition;
|
||||
import org.apache.doris.catalog.Replica;
|
||||
import org.apache.doris.catalog.Table;
|
||||
import org.apache.doris.catalog.Table.TableType;
|
||||
import org.apache.doris.catalog.Tablet;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.common.util.ListComparator;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.codehaus.jackson.map.ObjectMapper;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
/*
|
||||
* used to get table's sorted tablet info
|
||||
* eg:
|
||||
* fe_host:http_port/api/_migration?db=xxx&tbl=yyy
|
||||
*/
|
||||
public class MigrationAction extends RestBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(MigrationAction.class);
|
||||
private static final String DB_PARAM = "db";
|
||||
private static final String TABLE_PARAM = "tbl";
|
||||
|
||||
public MigrationAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
MigrationAction action = new MigrationAction(controller);
|
||||
controller.registerHandler(HttpMethod.GET, "/api/_migration", action);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
|
||||
checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
|
||||
|
||||
String dbName = request.getSingleParameter(DB_PARAM);
|
||||
String tableName = request.getSingleParameter(TABLE_PARAM);
|
||||
|
||||
if (Strings.isNullOrEmpty(dbName)) {
|
||||
throw new DdlException("Missing params. Need database name");
|
||||
}
|
||||
|
||||
Database db = Catalog.getCurrentCatalog().getDbOrDdlException(dbName);
|
||||
|
||||
List<List<Comparable>> rows = Lists.newArrayList();
|
||||
|
||||
|
||||
|
||||
if (!Strings.isNullOrEmpty(tableName)) {
|
||||
OlapTable olapTable = db.getOlapTableOrDdlException(tableName);
|
||||
olapTable.readLock();
|
||||
try {
|
||||
for (Partition partition : olapTable.getPartitions()) {
|
||||
String partitionName = partition.getName();
|
||||
MaterializedIndex baseIndex = partition.getBaseIndex();
|
||||
for (Tablet tablet : baseIndex.getTablets()) {
|
||||
List<Comparable> row = Lists.newArrayList();
|
||||
row.add(tableName);
|
||||
row.add(partitionName);
|
||||
row.add(tablet.getId());
|
||||
row.add(olapTable.getSchemaHashByIndexId(baseIndex.getId()));
|
||||
for (Replica replica : tablet.getReplicas()) {
|
||||
row.add(replica.getBackendId());
|
||||
break;
|
||||
}
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
olapTable.readUnlock();
|
||||
}
|
||||
} else {
|
||||
List<Table> tableList = db.getTables();
|
||||
|
||||
// get all olap table
|
||||
for (Table table : tableList) {
|
||||
if (table.getType() != TableType.OLAP) {
|
||||
continue;
|
||||
}
|
||||
|
||||
OlapTable olapTable = (OlapTable) table;
|
||||
table.readLock();
|
||||
try {
|
||||
tableName = table.getName();
|
||||
for (Partition partition : olapTable.getPartitions()) {
|
||||
String partitionName = partition.getName();
|
||||
MaterializedIndex baseIndex = partition.getBaseIndex();
|
||||
for (Tablet tablet : baseIndex.getTablets()) {
|
||||
List<Comparable> row = Lists.newArrayList();
|
||||
row.add(tableName);
|
||||
row.add(partitionName);
|
||||
row.add(tablet.getId());
|
||||
row.add(olapTable.getSchemaHashByIndexId(baseIndex.getId()));
|
||||
for (Replica replica : tablet.getReplicas()) {
|
||||
row.add(replica.getBackendId());
|
||||
break;
|
||||
}
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
table.readUnlock();
|
||||
}
|
||||
}
|
||||
}
|
||||
ListComparator<List<Comparable>> comparator = new ListComparator<List<Comparable>>(0, 1, 2);
|
||||
Collections.sort(rows, comparator);
|
||||
|
||||
// to json response
|
||||
String result = "";
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
try {
|
||||
result = mapper.writeValueAsString(rows);
|
||||
} catch (Exception e) {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
// send result
|
||||
response.setContentType("application/json");
|
||||
response.getContent().append(result);
|
||||
sendResult(request, response);
|
||||
}
|
||||
|
||||
public static void print(String msg) {
|
||||
System.out.println(System.currentTimeMillis() + " " + msg);
|
||||
}
|
||||
}
|
||||
@ -1,72 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.cluster.ClusterNamespace;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.service.ExecuteEnv;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
public class MultiAbort extends RestBaseAction {
|
||||
private ExecuteEnv execEnv;
|
||||
|
||||
public MultiAbort(ActionController controller, ExecuteEnv execEnv) {
|
||||
super(controller);
|
||||
this.execEnv = execEnv;
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
ExecuteEnv executeEnv = ExecuteEnv.getInstance();
|
||||
MultiAbort action = new MultiAbort(controller, executeEnv);
|
||||
controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_abort", action);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeWithoutPassword(BaseRequest request, BaseResponse response)
|
||||
throws DdlException {
|
||||
String db = request.getSingleParameter(DB_KEY);
|
||||
if (Strings.isNullOrEmpty(db)) {
|
||||
throw new DdlException("No database selected");
|
||||
}
|
||||
String label = request.getSingleParameter(LABEL_KEY);
|
||||
if (Strings.isNullOrEmpty(label)) {
|
||||
throw new DdlException("No label selected");
|
||||
}
|
||||
|
||||
String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db);
|
||||
checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD);
|
||||
|
||||
// only Master has these load info
|
||||
if (redirectToMaster(request, response)) {
|
||||
return;
|
||||
}
|
||||
|
||||
execEnv.getMultiLoadMgr().abort(fullDbName, label);
|
||||
sendResult(request, response, RestBaseResult.getOk());
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,77 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.cluster.ClusterNamespace;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.service.ExecuteEnv;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
public class MultiCommit extends RestBaseAction {
|
||||
private ExecuteEnv execEnv;
|
||||
|
||||
public MultiCommit(ActionController controller, ExecuteEnv execEnv) {
|
||||
super(controller);
|
||||
this.execEnv = execEnv;
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
ExecuteEnv executeEnv = ExecuteEnv.getInstance();
|
||||
MultiCommit action = new MultiCommit(controller, executeEnv);
|
||||
controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_commit", action);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeWithoutPassword(BaseRequest request, BaseResponse response)
|
||||
throws DdlException {
|
||||
String db = request.getSingleParameter(DB_KEY);
|
||||
if (Strings.isNullOrEmpty(db)) {
|
||||
throw new DdlException("No database selected");
|
||||
}
|
||||
String label = request.getSingleParameter(LABEL_KEY);
|
||||
if (Strings.isNullOrEmpty(label)) {
|
||||
throw new DdlException("No label selected");
|
||||
}
|
||||
|
||||
String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db);
|
||||
checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD);
|
||||
|
||||
// only Master has these load info
|
||||
if (redirectToMaster(request, response)) {
|
||||
return;
|
||||
}
|
||||
RestBaseResult result = new RestBaseResult();
|
||||
try {
|
||||
execEnv.getMultiLoadMgr().commit(fullDbName, label);
|
||||
} catch (Exception e) {
|
||||
result.msg = e.getMessage();
|
||||
result.status = ActionStatus.FAILED;
|
||||
}
|
||||
sendResult(request, response, result);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,85 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.cluster.ClusterNamespace;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.service.ExecuteEnv;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
// List all labels of one multi-load
|
||||
public class MultiDesc extends RestBaseAction {
|
||||
private ExecuteEnv execEnv;
|
||||
|
||||
public MultiDesc(ActionController controller, ExecuteEnv execEnv) {
|
||||
super(controller);
|
||||
this.execEnv = execEnv;
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
ExecuteEnv executeEnv = ExecuteEnv.getInstance();
|
||||
MultiDesc action = new MultiDesc(controller, executeEnv);
|
||||
controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_desc", action);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeWithoutPassword(BaseRequest request, BaseResponse response)
|
||||
throws DdlException {
|
||||
String db = request.getSingleParameter(DB_KEY);
|
||||
if (Strings.isNullOrEmpty(db)) {
|
||||
throw new DdlException("No database selected");
|
||||
}
|
||||
String label = request.getSingleParameter(LABEL_KEY);
|
||||
if (Strings.isNullOrEmpty(label)) {
|
||||
throw new DdlException("No label selected");
|
||||
}
|
||||
|
||||
String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db);
|
||||
checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD);
|
||||
|
||||
// only Master has these load info
|
||||
if (redirectToMaster(request, response)) {
|
||||
return;
|
||||
}
|
||||
|
||||
final List<String> labels = Lists.newArrayList();
|
||||
execEnv.getMultiLoadMgr().desc(fullDbName, label, labels);
|
||||
sendResult(request, response, new Result(labels));
|
||||
}
|
||||
|
||||
private static class Result extends RestBaseResult {
|
||||
private List<String> labels;
|
||||
|
||||
public Result(List<String> labels) {
|
||||
this.labels = labels;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,82 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.cluster.ClusterNamespace;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.service.ExecuteEnv;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
// list all multi load before commit
|
||||
public class MultiList extends RestBaseAction {
|
||||
|
||||
private ExecuteEnv execEnv;
|
||||
|
||||
public MultiList(ActionController controller, ExecuteEnv execEnv) {
|
||||
super(controller);
|
||||
this.execEnv = execEnv;
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
ExecuteEnv executeEnv = ExecuteEnv.getInstance();
|
||||
MultiList action = new MultiList(controller, executeEnv);
|
||||
controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_list", action);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeWithoutPassword(BaseRequest request, BaseResponse response)
|
||||
throws DdlException {
|
||||
String db = request.getSingleParameter(DB_KEY);
|
||||
if (Strings.isNullOrEmpty(db)) {
|
||||
throw new DdlException("No database selected");
|
||||
}
|
||||
|
||||
String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db);
|
||||
checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD);
|
||||
|
||||
// only Master has these load info
|
||||
if (redirectToMaster(request, response)) {
|
||||
return;
|
||||
}
|
||||
|
||||
final List<String> labels = Lists.newArrayList();
|
||||
execEnv.getMultiLoadMgr().list(fullDbName, labels);
|
||||
sendResult(request, response, new Result(labels));
|
||||
}
|
||||
|
||||
private static class Result extends RestBaseResult {
|
||||
private List<String> labels;
|
||||
|
||||
public Result(List<String> labels) {
|
||||
this.labels = labels;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,92 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.analysis.LoadStmt;
|
||||
import org.apache.doris.cluster.ClusterNamespace;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.service.ExecuteEnv;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.Maps;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
// Start multi action
|
||||
public class MultiStart extends RestBaseAction {
|
||||
private ExecuteEnv execEnv;
|
||||
|
||||
public MultiStart(ActionController controller, ExecuteEnv execEnv) {
|
||||
super(controller);
|
||||
this.execEnv = execEnv;
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
ExecuteEnv executeEnv = ExecuteEnv.getInstance();
|
||||
MultiStart action = new MultiStart(controller, executeEnv);
|
||||
controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_start", action);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeWithoutPassword(BaseRequest request, BaseResponse response)
|
||||
throws DdlException {
|
||||
String db = request.getSingleParameter(DB_KEY);
|
||||
if (Strings.isNullOrEmpty(db)) {
|
||||
throw new DdlException("No database selected");
|
||||
}
|
||||
String label = request.getSingleParameter(LABEL_KEY);
|
||||
if (Strings.isNullOrEmpty(label)) {
|
||||
throw new DdlException("No label selected");
|
||||
}
|
||||
|
||||
String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db);
|
||||
checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD);
|
||||
|
||||
// Multi start request must redirect to master, because all following sub requests will be handled
|
||||
// on Master
|
||||
if (redirectToMaster(request, response)) {
|
||||
return;
|
||||
}
|
||||
|
||||
Map<String, String> properties = Maps.newHashMap();
|
||||
String[] keys = {LoadStmt.TIMEOUT_PROPERTY, LoadStmt.MAX_FILTER_RATIO_PROPERTY};
|
||||
for (String key : keys) {
|
||||
String value = request.getSingleParameter(key);
|
||||
if (!Strings.isNullOrEmpty(value)) {
|
||||
properties.put(key, value);
|
||||
}
|
||||
}
|
||||
for (String key : keys) {
|
||||
String value = request.getRequest().headers().get(key);
|
||||
if (!Strings.isNullOrEmpty(value)) {
|
||||
properties.put(key, value);
|
||||
}
|
||||
}
|
||||
execEnv.getMultiLoadMgr().startMulti(fullDbName, label, properties);
|
||||
sendResult(request, response, RestBaseResult.getOk());
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,76 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.cluster.ClusterNamespace;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.service.ExecuteEnv;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
public class MultiUnload extends RestBaseAction {
|
||||
private static final String SUB_LABEL_KEY = "sub_label";
|
||||
|
||||
private ExecuteEnv execEnv;
|
||||
|
||||
public MultiUnload(ActionController controller, ExecuteEnv execEnv) {
|
||||
super(controller);
|
||||
this.execEnv = execEnv;
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
ExecuteEnv executeEnv = ExecuteEnv.getInstance();
|
||||
MultiUnload action = new MultiUnload(controller, executeEnv);
|
||||
controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_unload", action);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeWithoutPassword(BaseRequest request, BaseResponse response)
|
||||
throws DdlException {
|
||||
String db = request.getSingleParameter(DB_KEY);
|
||||
if (Strings.isNullOrEmpty(db)) {
|
||||
throw new DdlException("No database selected");
|
||||
}
|
||||
String label = request.getSingleParameter(LABEL_KEY);
|
||||
if (Strings.isNullOrEmpty(label)) {
|
||||
throw new DdlException("No label selected");
|
||||
}
|
||||
String subLabel = request.getSingleParameter(SUB_LABEL_KEY);
|
||||
if (Strings.isNullOrEmpty(subLabel)) {
|
||||
throw new DdlException("No sub_label selected");
|
||||
}
|
||||
|
||||
String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db);
|
||||
checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD);
|
||||
|
||||
if (redirectToMaster(request, response)) {
|
||||
return;
|
||||
}
|
||||
|
||||
execEnv.getMultiLoadMgr().unload(fullDbName, label, subLabel);
|
||||
sendResult(request, response, RestBaseResult.getOk());
|
||||
}
|
||||
}
|
||||
@ -1,60 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.common.util.ProfileManager;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
|
||||
// This class is a RESTFUL interface to get query profile.
|
||||
// It will be used in query monitor to collect profiles.
|
||||
// Usage:
|
||||
// wget http://fe_host:fe_http_port/api/profile?query_id=123456
|
||||
public class ProfileAction extends RestBaseAction {
|
||||
|
||||
public ProfileAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/api/profile", new ProfileAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(BaseRequest request, BaseResponse response) {
|
||||
String queryId = request.getSingleParameter("query_id");
|
||||
if (queryId == null) {
|
||||
response.getContent().append("not valid parameter");
|
||||
sendResult(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
String queryProfileStr = ProfileManager.getInstance().getProfile(queryId);
|
||||
if (queryProfileStr != null) {
|
||||
response.getContent().append(queryProfileStr);
|
||||
sendResult(request, response);
|
||||
} else {
|
||||
response.getContent().append("query id " + queryId + " not found.");
|
||||
sendResult(request, response, HttpResponseStatus.NOT_FOUND);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,56 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
import java.util.List;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.qe.QueryDetail;
|
||||
import org.apache.doris.qe.QueryDetailQueue;
|
||||
|
||||
public class QueryDetailAction extends RestBaseAction {
|
||||
|
||||
public QueryDetailAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/api/query_detail", new QueryDetailAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(BaseRequest request, BaseResponse response) {
|
||||
String eventTimeStr = request.getSingleParameter("event_time");
|
||||
if (eventTimeStr == null) {
|
||||
response.getContent().append("not valid parameter");
|
||||
sendResult(request, response, HttpResponseStatus.BAD_REQUEST);
|
||||
return;
|
||||
}
|
||||
long eventTime = Long.valueOf(eventTimeStr.trim());
|
||||
List<QueryDetail> queryDetails = QueryDetailQueue.getQueryDetails(eventTime);
|
||||
Gson gson = new Gson();
|
||||
String json_string = gson.toJson(queryDetails);
|
||||
response.getContent().append(json_string);
|
||||
sendResult(request, response);
|
||||
}
|
||||
}
|
||||
@ -1,127 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.analysis.UserIdentity;
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseAction;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.UnauthorizedException;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.thrift.TNetworkAddress;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
|
||||
import io.netty.handler.codec.http.HttpHeaderNames;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
|
||||
public class RestBaseAction extends BaseAction {
|
||||
protected static final String DB_KEY = "db";
|
||||
protected static final String TABLE_KEY = "table";
|
||||
protected static final String LABEL_KEY = "label";
|
||||
private static final Logger LOG = LogManager.getLogger(RestBaseAction.class);
|
||||
|
||||
public RestBaseAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handleRequest(BaseRequest request) throws Exception {
|
||||
LOG.debug("receive http request. url={}", request.getRequest().uri());
|
||||
BaseResponse response = new BaseResponse();
|
||||
try {
|
||||
execute(request, response);
|
||||
} catch (DdlException e) {
|
||||
if (e instanceof UnauthorizedException) {
|
||||
response.appendContent(e.getMessage());
|
||||
response.updateHeader(HttpHeaderNames.WWW_AUTHENTICATE.toString(), "Basic realm=\"\"");
|
||||
writeResponse(request, response, HttpResponseStatus.UNAUTHORIZED);
|
||||
} else {
|
||||
sendResult(request, response, new RestBaseResult(e.getMessage()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(BaseRequest request, BaseResponse response) throws DdlException {
|
||||
ActionAuthorizationInfo authInfo = getAuthorizationInfo(request);
|
||||
// check password
|
||||
UserIdentity currentUser = checkPassword(authInfo);
|
||||
ConnectContext ctx = new ConnectContext(null);
|
||||
ctx.setCatalog(Catalog.getCurrentCatalog());
|
||||
ctx.setQualifiedUser(authInfo.fullUserName);
|
||||
ctx.setRemoteIP(authInfo.remoteIp);
|
||||
ctx.setCurrentUserIdentity(currentUser);
|
||||
ctx.setCluster(authInfo.cluster);
|
||||
ctx.setThreadLocalInfo();
|
||||
executeWithoutPassword(request, response);
|
||||
}
|
||||
|
||||
// If user password should be checked, the derived class should implement this method, NOT 'execute()',
|
||||
// otherwise, override 'execute()' directly
|
||||
protected void executeWithoutPassword(BaseRequest request, BaseResponse response)
|
||||
throws DdlException {
|
||||
throw new DdlException("Not implemented");
|
||||
}
|
||||
|
||||
public void sendResult(BaseRequest request, BaseResponse response, RestBaseResult result) {
|
||||
response.appendContent(result.toJson());
|
||||
writeResponse(request, response, HttpResponseStatus.OK);
|
||||
}
|
||||
|
||||
public void sendResult(BaseRequest request, BaseResponse response, HttpResponseStatus status) {
|
||||
writeResponse(request, response, status);
|
||||
}
|
||||
|
||||
public void sendResult(BaseRequest request, BaseResponse response) {
|
||||
writeResponse(request, response, HttpResponseStatus.OK);
|
||||
}
|
||||
|
||||
public void redirectTo(BaseRequest request, BaseResponse response, TNetworkAddress addr)
|
||||
throws DdlException {
|
||||
String urlStr = request.getRequest().uri();
|
||||
URI urlObj = null;
|
||||
URI resultUriObj = null;
|
||||
try {
|
||||
urlObj = new URI(urlStr);
|
||||
resultUriObj = new URI("http", null, addr.getHostname(),
|
||||
addr.getPort(), urlObj.getPath(), urlObj.getQuery(), null);
|
||||
} catch (URISyntaxException e) {
|
||||
LOG.warn(e.getMessage());
|
||||
throw new DdlException(e.getMessage());
|
||||
}
|
||||
response.updateHeader(HttpHeaderNames.LOCATION.toString(), resultUriObj.toString());
|
||||
writeResponse(request, response, HttpResponseStatus.TEMPORARY_REDIRECT);
|
||||
}
|
||||
|
||||
public boolean redirectToMaster(BaseRequest request, BaseResponse response) throws DdlException {
|
||||
Catalog catalog = Catalog.getCurrentCatalog();
|
||||
if (catalog.isMaster()) {
|
||||
return false;
|
||||
}
|
||||
redirectTo(request, response, new TNetworkAddress(catalog.getMasterIp(), catalog.getMasterHttpPort()));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -1,115 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.catalog.Database;
|
||||
import org.apache.doris.catalog.MaterializedIndex;
|
||||
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
|
||||
import org.apache.doris.catalog.OlapTable;
|
||||
import org.apache.doris.catalog.Partition;
|
||||
import org.apache.doris.catalog.Replica;
|
||||
import org.apache.doris.catalog.Tablet;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.Maps;
|
||||
|
||||
import org.codehaus.jackson.map.ObjectMapper;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
/*
|
||||
* calc row count from replica to table
|
||||
* fe_host:fe_http_port/api/rowcount?db=dbname&table=tablename
|
||||
*/
|
||||
public class RowCountAction extends RestBaseAction {
|
||||
|
||||
public RowCountAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/api/rowcount", new RowCountAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
|
||||
checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
|
||||
|
||||
String dbName = request.getSingleParameter(DB_KEY);
|
||||
if (Strings.isNullOrEmpty(dbName)) {
|
||||
throw new DdlException("No database selected.");
|
||||
}
|
||||
|
||||
String tableName = request.getSingleParameter(TABLE_KEY);
|
||||
if (Strings.isNullOrEmpty(tableName)) {
|
||||
throw new DdlException("No table selected.");
|
||||
}
|
||||
|
||||
Map<String, Long> indexRowCountMap = Maps.newHashMap();
|
||||
Catalog catalog = Catalog.getCurrentCatalog();
|
||||
Database db = catalog.getDbOrDdlException(dbName);
|
||||
OlapTable olapTable = db.getOlapTableOrDdlException(tableName);
|
||||
olapTable.writeLockOrDdlException();
|
||||
try {
|
||||
for (Partition partition : olapTable.getAllPartitions()) {
|
||||
long version = partition.getVisibleVersion();
|
||||
for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
|
||||
long indexRowCount = 0L;
|
||||
for (Tablet tablet : index.getTablets()) {
|
||||
long tabletRowCount = 0L;
|
||||
for (Replica replica : tablet.getReplicas()) {
|
||||
if (replica.checkVersionCatchUp(version, false)
|
||||
&& replica.getRowCount() > tabletRowCount) {
|
||||
tabletRowCount = replica.getRowCount();
|
||||
}
|
||||
}
|
||||
indexRowCount += tabletRowCount;
|
||||
} // end for tablets
|
||||
index.setRowCount(indexRowCount);
|
||||
indexRowCountMap.put(olapTable.getIndexNameById(index.getId()), indexRowCount);
|
||||
} // end for indices
|
||||
} // end for partitions
|
||||
} finally {
|
||||
olapTable.writeUnlock();
|
||||
}
|
||||
|
||||
// to json response
|
||||
String result = "";
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
try {
|
||||
result = mapper.writeValueAsString(indexRowCountMap);
|
||||
} catch (Exception e) {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
// send result
|
||||
response.setContentType("application/json");
|
||||
response.getContent().append(result);
|
||||
sendResult(request, response);
|
||||
}
|
||||
}
|
||||
@ -1,136 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
import org.apache.doris.common.ConfigBase;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
|
||||
import com.google.common.collect.Maps;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.codehaus.jackson.map.ObjectMapper;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/*
|
||||
* used to set fe config
|
||||
* eg:
|
||||
* fe_host:http_port/api/_set_config?config_key1=config_value1&config_key2=config_value2&...
|
||||
*/
|
||||
public class SetConfigAction extends RestBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(SetConfigAction.class);
|
||||
|
||||
private static final String PERSIST_PARAM = "persist";
|
||||
private static final String RESET_PERSIST = "reset_persist";
|
||||
|
||||
public SetConfigAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
SetConfigAction action = new SetConfigAction(controller);
|
||||
controller.registerHandler(HttpMethod.GET, "/api/_set_config", action);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
|
||||
checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
|
||||
|
||||
boolean needPersist = false;
|
||||
boolean resetPersist = true;
|
||||
Map<String, List<String>> configs = request.getAllParameters();
|
||||
if (configs.containsKey(PERSIST_PARAM)) {
|
||||
List<String> val = configs.remove(PERSIST_PARAM);
|
||||
if (val.size() == 1 && val.get(0).equals("true")) {
|
||||
needPersist = true;
|
||||
}
|
||||
}
|
||||
if (configs.containsKey(RESET_PERSIST)) {
|
||||
List<String> val = configs.remove(RESET_PERSIST);
|
||||
if (val.size() == 1 && val.get(0).equals("false")) {
|
||||
resetPersist = false;
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, String> setConfigs = Maps.newHashMap();
|
||||
Map<String, String> errConfigs = Maps.newHashMap();
|
||||
|
||||
LOG.debug("get config from url: {}, need persist: {}", configs, needPersist);
|
||||
|
||||
for (Map.Entry<String, List<String>> config : configs.entrySet()) {
|
||||
String confKey = config.getKey();
|
||||
List<String> confValue = config.getValue();
|
||||
try {
|
||||
if (confValue != null && confValue.size() == 1) {
|
||||
ConfigBase.setMutableConfig(confKey, confValue.get(0));
|
||||
setConfigs.put(confKey, confValue.get(0));
|
||||
} else {
|
||||
throw new DdlException("conf value size != 1");
|
||||
}
|
||||
} catch (DdlException e) {
|
||||
LOG.warn("failed to set config {}:{}", confKey, confValue, e);
|
||||
errConfigs.put(confKey, String.valueOf(confValue));
|
||||
}
|
||||
}
|
||||
|
||||
String persistMsg = "";
|
||||
if (needPersist) {
|
||||
try {
|
||||
ConfigBase.persistConfig(setConfigs, resetPersist);
|
||||
persistMsg = "ok";
|
||||
} catch (IOException e) {
|
||||
LOG.warn("failed to persist config", e);
|
||||
persistMsg = e.getMessage();
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, Object> resultMap = Maps.newHashMap();
|
||||
resultMap.put("set", setConfigs);
|
||||
resultMap.put("err", errConfigs);
|
||||
resultMap.put("persist", persistMsg);
|
||||
|
||||
// to json response
|
||||
String result = "";
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
try {
|
||||
result = mapper.writeValueAsString(resultMap);
|
||||
} catch (Exception e) {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
// send result
|
||||
response.setContentType("application/json");
|
||||
response.getContent().append(result);
|
||||
sendResult(request, response);
|
||||
}
|
||||
|
||||
public static void print(String msg) {
|
||||
System.out.println(System.currentTimeMillis() + " " + msg);
|
||||
}
|
||||
}
|
||||
@ -1,91 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.catalog.Database;
|
||||
import org.apache.doris.catalog.OlapTable;
|
||||
import org.apache.doris.catalog.Table;
|
||||
import org.apache.doris.catalog.Table.TableType;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.List;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
public class ShowDataAction extends RestBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(ShowDataAction.class);
|
||||
|
||||
public ShowDataAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction (ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/api/show_data", new ShowDataAction(controller));
|
||||
}
|
||||
|
||||
public long getDataSizeOfDatabase(Database db) {
|
||||
long totalSize = 0;
|
||||
long tableSize = 0;
|
||||
// sort by table name
|
||||
List<Table> tables = db.getTables();
|
||||
for (Table table : tables) {
|
||||
if (table.getType() != TableType.OLAP) {
|
||||
continue;
|
||||
}
|
||||
table.readLock();
|
||||
try {
|
||||
tableSize = ((OlapTable)table).getDataSize();
|
||||
} finally {
|
||||
table.readUnlock();
|
||||
}
|
||||
totalSize += tableSize;
|
||||
} // end for tables
|
||||
return totalSize;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(BaseRequest request, BaseResponse response) {
|
||||
String dbName = request.getSingleParameter("db");
|
||||
ConcurrentHashMap<String, Database> fullNameToDb = Catalog.getCurrentCatalog().getFullNameToDb();
|
||||
long totalSize = 0;
|
||||
if (dbName != null) {
|
||||
Database db = fullNameToDb.get("default_cluster:"+dbName);
|
||||
if (db == null) {
|
||||
response.getContent().append("database " + dbName + " not found.");
|
||||
sendResult(request, response, HttpResponseStatus.NOT_FOUND);
|
||||
return;
|
||||
}
|
||||
totalSize = getDataSizeOfDatabase(db);
|
||||
} else {
|
||||
for (Database db : fullNameToDb.values()) {
|
||||
LOG.info("database name: {}", db.getFullName());
|
||||
totalSize += getDataSizeOfDatabase(db);
|
||||
}
|
||||
}
|
||||
response.getContent().append(String.valueOf(totalSize));
|
||||
sendResult(request, response);
|
||||
}
|
||||
}
|
||||
@ -1,203 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.catalog.Database;
|
||||
import org.apache.doris.catalog.MaterializedIndex;
|
||||
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
|
||||
import org.apache.doris.catalog.OlapTable;
|
||||
import org.apache.doris.catalog.Partition;
|
||||
import org.apache.doris.catalog.Replica;
|
||||
import org.apache.doris.catalog.Replica.ReplicaState;
|
||||
import org.apache.doris.catalog.Table;
|
||||
import org.apache.doris.catalog.Table.TableType;
|
||||
import org.apache.doris.catalog.Tablet;
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.ha.HAProtocol;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.persist.Storage;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
public class ShowMetaInfoAction extends RestBaseAction {
|
||||
private enum Action {
|
||||
SHOW_DB_SIZE,
|
||||
SHOW_HA,
|
||||
INVALID;
|
||||
|
||||
public static Action getAction(String str) {
|
||||
try {
|
||||
return valueOf(str);
|
||||
} catch (Exception ex) {
|
||||
return INVALID;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static final Logger LOG = LogManager.getLogger(ShowMetaInfoAction.class);
|
||||
|
||||
public ShowMetaInfoAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/api/show_meta_info",
|
||||
new ShowMetaInfoAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(BaseRequest request, BaseResponse response) {
|
||||
String action = request.getSingleParameter("action");
|
||||
Gson gson = new Gson();
|
||||
response.setContentType("application/json");
|
||||
|
||||
switch (Action.getAction(action.toUpperCase())) {
|
||||
case SHOW_DB_SIZE:
|
||||
response.getContent().append(gson.toJson(getDataSize()));
|
||||
break;
|
||||
case SHOW_HA:
|
||||
response.getContent().append(gson.toJson(getHaInfo()));
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
sendResult(request, response);
|
||||
}
|
||||
|
||||
public Map<String, String> getHaInfo() {
|
||||
HashMap<String, String> feInfo = new HashMap<String, String>();
|
||||
feInfo.put("role", Catalog.getCurrentCatalog().getFeType().toString());
|
||||
if (Catalog.getCurrentCatalog().isMaster()) {
|
||||
feInfo.put("current_journal_id",
|
||||
String.valueOf(Catalog.getCurrentCatalog().getEditLog().getMaxJournalId()));
|
||||
} else {
|
||||
feInfo.put("current_journal_id",
|
||||
String.valueOf(Catalog.getCurrentCatalog().getReplayedJournalId()));
|
||||
}
|
||||
|
||||
HAProtocol haProtocol = Catalog.getCurrentCatalog().getHaProtocol();
|
||||
if (haProtocol != null) {
|
||||
|
||||
InetSocketAddress master = null;
|
||||
try {
|
||||
master = haProtocol.getLeader();
|
||||
} catch (Exception e) {
|
||||
// this may happen when majority of FOLLOWERS are down and no MASTER right now.
|
||||
LOG.warn("failed to get leader: {}", e.getMessage());
|
||||
}
|
||||
if (master != null) {
|
||||
feInfo.put("master", master.getHostString());
|
||||
} else {
|
||||
feInfo.put("master", "unknown");
|
||||
}
|
||||
|
||||
List<InetSocketAddress> electableNodes = haProtocol.getElectableNodes(false);
|
||||
ArrayList<String> electableNodeNames = new ArrayList<String>();
|
||||
if (!electableNodes.isEmpty()) {
|
||||
for (InetSocketAddress node : electableNodes) {
|
||||
electableNodeNames.add(node.getHostString());
|
||||
}
|
||||
feInfo.put("electable_nodes", StringUtils.join(electableNodeNames.toArray(), ","));
|
||||
}
|
||||
|
||||
List<InetSocketAddress> observerNodes = haProtocol.getObserverNodes();
|
||||
ArrayList<String> observerNodeNames = new ArrayList<String>();
|
||||
if (observerNodes != null) {
|
||||
for (InetSocketAddress node : observerNodes) {
|
||||
observerNodeNames.add(node.getHostString());
|
||||
}
|
||||
feInfo.put("observer_nodes", StringUtils.join(observerNodeNames.toArray(), ","));
|
||||
}
|
||||
}
|
||||
|
||||
feInfo.put("can_read", String.valueOf(Catalog.getCurrentCatalog().canRead()));
|
||||
feInfo.put("is_ready", String.valueOf(Catalog.getCurrentCatalog().isReady()));
|
||||
try {
|
||||
Storage storage = new Storage(Config.meta_dir + "/image");
|
||||
feInfo.put("last_checkpoint_version", String.valueOf(storage.getImageSeq()));
|
||||
long lastCheckpointTime = storage.getCurrentImageFile().lastModified();
|
||||
feInfo.put("last_checkpoint_time", String.valueOf(lastCheckpointTime));
|
||||
} catch (IOException e) {
|
||||
LOG.warn(e.getMessage());
|
||||
}
|
||||
return feInfo;
|
||||
}
|
||||
|
||||
public Map<String, Long> getDataSize() {
|
||||
Map<String, Long> result = new HashMap<String, Long>();
|
||||
List<String> dbNames = Catalog.getCurrentCatalog().getDbNames();
|
||||
|
||||
for (String dbName : dbNames) {
|
||||
Database db = Catalog.getCurrentCatalog().getDbNullable(dbName);
|
||||
if (db == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
long totalSize = 0;
|
||||
List<Table> tables = db.getTables();
|
||||
for (Table table : tables) {
|
||||
if (table.getType() != TableType.OLAP) {
|
||||
continue;
|
||||
}
|
||||
|
||||
OlapTable olapTable = (OlapTable) table;
|
||||
long tableSize = 0;
|
||||
for (Partition partition : olapTable.getAllPartitions()) {
|
||||
long partitionSize = 0;
|
||||
for (MaterializedIndex mIndex : partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
|
||||
long indexSize = 0;
|
||||
for (Tablet tablet : mIndex.getTablets()) {
|
||||
long maxReplicaSize = 0;
|
||||
for (Replica replica : tablet.getReplicas()) {
|
||||
if (replica.getState() == ReplicaState.NORMAL
|
||||
|| replica.getState() == ReplicaState.SCHEMA_CHANGE) {
|
||||
if (replica.getDataSize() > maxReplicaSize) {
|
||||
maxReplicaSize = replica.getDataSize();
|
||||
}
|
||||
}
|
||||
} // end for replicas
|
||||
indexSize += maxReplicaSize;
|
||||
} // end for tablets
|
||||
partitionSize += indexSize;
|
||||
} // end for tables
|
||||
tableSize += partitionSize;
|
||||
} // end for partitions
|
||||
totalSize += tableSize;
|
||||
} // end for tables
|
||||
result.put(dbName, totalSize);
|
||||
} // end for dbs
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@ -1,136 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.analysis.RedirectStatus;
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.common.AnalysisException;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.common.proc.ProcNodeInterface;
|
||||
import org.apache.doris.common.proc.ProcResult;
|
||||
import org.apache.doris.common.proc.ProcService;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.qe.MasterOpExecutor;
|
||||
import org.apache.doris.qe.OriginStatement;
|
||||
import org.apache.doris.qe.ShowResultSet;
|
||||
import org.apache.doris.system.SystemInfoService;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.gson.Gson;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
// Format:
|
||||
// http://username:password@192.168.1.1:8030/api/show_proc?path=/
|
||||
public class ShowProcAction extends RestBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(ShowProcAction.class);
|
||||
|
||||
public ShowProcAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/api/show_proc", new ShowProcAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
|
||||
// check authority
|
||||
checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
|
||||
|
||||
String path = request.getSingleParameter("path");
|
||||
String forward = request.getSingleParameter("forward");
|
||||
boolean isForward = false;
|
||||
if (!Strings.isNullOrEmpty(forward) && forward.equals("true")) {
|
||||
isForward = true;
|
||||
}
|
||||
|
||||
// forward to master if necessary
|
||||
if (!Catalog.getCurrentCatalog().isMaster() && isForward) {
|
||||
String showProcStmt = "SHOW PROC \"" + path + "\"";
|
||||
ConnectContext context = new ConnectContext(null);
|
||||
context.setCatalog(Catalog.getCurrentCatalog());
|
||||
context.setCluster(SystemInfoService.DEFAULT_CLUSTER);
|
||||
context.setQualifiedUser(ConnectContext.get().getQualifiedUser());
|
||||
context.setRemoteIP(ConnectContext.get().getRemoteIP());
|
||||
MasterOpExecutor masterOpExecutor = new MasterOpExecutor(new OriginStatement(showProcStmt, 0), context,
|
||||
RedirectStatus.FORWARD_NO_SYNC, true);
|
||||
LOG.debug("need to transfer to Master. stmt: {}", context.getStmtId());
|
||||
|
||||
try {
|
||||
masterOpExecutor.execute();
|
||||
} catch (Exception e) {
|
||||
response.appendContent("Failed to forward stmt: " + e.getMessage());
|
||||
sendResult(request, response);
|
||||
return;
|
||||
}
|
||||
|
||||
ShowResultSet resultSet = masterOpExecutor.getProxyResultSet();
|
||||
if (resultSet == null) {
|
||||
response.appendContent("Failed to get result set");
|
||||
sendResult(request, response);
|
||||
return;
|
||||
}
|
||||
|
||||
Gson gson = new Gson();
|
||||
response.setContentType("application/json");
|
||||
response.getContent().append(gson.toJson(resultSet.getResultRows()));
|
||||
|
||||
} else {
|
||||
ProcNodeInterface procNode = null;
|
||||
ProcService instance = ProcService.getInstance();
|
||||
try {
|
||||
if (Strings.isNullOrEmpty(path)) {
|
||||
procNode = instance.open("/");
|
||||
} else {
|
||||
procNode = instance.open(path);
|
||||
}
|
||||
} catch (AnalysisException e) {
|
||||
LOG.warn(e.getMessage());
|
||||
response.getContent().append("[]");
|
||||
}
|
||||
|
||||
if (procNode != null) {
|
||||
ProcResult result;
|
||||
try {
|
||||
result = procNode.fetchResult();
|
||||
List<List<String>> rows = result.getRows();
|
||||
|
||||
Gson gson = new Gson();
|
||||
response.setContentType("application/json");
|
||||
response.getContent().append(gson.toJson(rows));
|
||||
} catch (AnalysisException e) {
|
||||
LOG.warn(e.getMessage());
|
||||
response.getContent().append("[]");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sendResult(request, response);
|
||||
}
|
||||
}
|
||||
@ -1,69 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import com.google.gson.Gson;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.util.HashMap;
|
||||
|
||||
public class ShowRuntimeInfoAction extends RestBaseAction {
|
||||
private static final Logger LOG = LogManager.getLogger(ShowRuntimeInfoAction.class);
|
||||
|
||||
public ShowRuntimeInfoAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET, "/api/show_runtime_info",
|
||||
new ShowRuntimeInfoAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(BaseRequest request, BaseResponse response) {
|
||||
HashMap<String, String> feInfo = new HashMap<String, String>();
|
||||
|
||||
// Get memory info
|
||||
Runtime r = Runtime.getRuntime();
|
||||
feInfo.put("free_mem", String.valueOf(r.freeMemory()));
|
||||
feInfo.put("total_mem", String.valueOf(r.totalMemory()));
|
||||
feInfo.put("max_mem", String.valueOf(r.maxMemory()));
|
||||
|
||||
// Get thread count
|
||||
ThreadGroup parentThread;
|
||||
for (parentThread = Thread.currentThread().getThreadGroup();
|
||||
parentThread.getParent() != null;
|
||||
parentThread = parentThread.getParent()) {
|
||||
};
|
||||
feInfo.put("thread_cnt", String.valueOf(parentThread.activeCount()));
|
||||
|
||||
Gson gson = new Gson();
|
||||
response.setContentType("application/json");
|
||||
response.getContent().append(gson.toJson(feInfo));
|
||||
|
||||
sendResult(request, response);
|
||||
}
|
||||
}
|
||||
@ -1,98 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.catalog.Database;
|
||||
import org.apache.doris.catalog.MaterializedIndexMeta;
|
||||
import org.apache.doris.catalog.OlapTable;
|
||||
import org.apache.doris.catalog.Table;
|
||||
import org.apache.doris.catalog.Table.TableType;
|
||||
import org.apache.doris.cluster.ClusterNamespace;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.thrift.TStorageType;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import org.json.simple.JSONObject;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
|
||||
public class StorageTypeCheckAction extends RestBaseAction {
|
||||
public StorageTypeCheckAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
StorageTypeCheckAction action = new StorageTypeCheckAction(controller);
|
||||
controller.registerHandler(HttpMethod.GET, "/api/_check_storagetype", action);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
|
||||
checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
|
||||
|
||||
String dbName = request.getSingleParameter(DB_KEY);
|
||||
if (Strings.isNullOrEmpty(dbName)) {
|
||||
throw new DdlException("Parameter db is missing");
|
||||
}
|
||||
|
||||
String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), dbName);
|
||||
Database db = catalog.getDbOrDdlException(fullDbName);
|
||||
|
||||
JSONObject root = new JSONObject();
|
||||
List<Table> tableList = db.getTables();
|
||||
|
||||
for (Table tbl : tableList) {
|
||||
if (tbl.getType() != TableType.OLAP) {
|
||||
continue;
|
||||
}
|
||||
|
||||
OlapTable olapTbl = (OlapTable) tbl;
|
||||
olapTbl.readLock();
|
||||
try {
|
||||
JSONObject indexObj = new JSONObject();
|
||||
for (Map.Entry<Long, MaterializedIndexMeta> entry : olapTbl.getIndexIdToMeta().entrySet()) {
|
||||
MaterializedIndexMeta indexMeta = entry.getValue();
|
||||
if (indexMeta.getStorageType() == TStorageType.ROW) {
|
||||
indexObj.put(olapTbl.getIndexNameById(entry.getKey()), indexMeta.getStorageType().name());
|
||||
}
|
||||
}
|
||||
root.put(tbl.getName(), indexObj);
|
||||
} finally {
|
||||
olapTbl.readUnlock();
|
||||
}
|
||||
}
|
||||
|
||||
// to json response
|
||||
String result = root.toString();
|
||||
|
||||
// send result
|
||||
response.setContentType("application/json");
|
||||
response.getContent().append(result);
|
||||
sendResult(request, response);
|
||||
}
|
||||
}
|
||||
@ -1,306 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.analysis.InlineViewRef;
|
||||
import org.apache.doris.analysis.SelectStmt;
|
||||
import org.apache.doris.analysis.StatementBase;
|
||||
import org.apache.doris.analysis.TableName;
|
||||
import org.apache.doris.analysis.TableRef;
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.catalog.Database;
|
||||
import org.apache.doris.catalog.Table;
|
||||
import org.apache.doris.cluster.ClusterNamespace;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.common.DorisHttpException;
|
||||
import org.apache.doris.common.MetaNotFoundException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.planner.PlanFragment;
|
||||
import org.apache.doris.planner.Planner;
|
||||
import org.apache.doris.planner.ScanNode;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.qe.OriginStatement;
|
||||
import org.apache.doris.qe.StmtExecutor;
|
||||
import org.apache.doris.thrift.TDataSink;
|
||||
import org.apache.doris.thrift.TDataSinkType;
|
||||
import org.apache.doris.thrift.TMemoryScratchSink;
|
||||
import org.apache.doris.thrift.TNetworkAddress;
|
||||
import org.apache.doris.thrift.TPaloScanRange;
|
||||
import org.apache.doris.thrift.TPlanFragment;
|
||||
import org.apache.doris.thrift.TQueryOptions;
|
||||
import org.apache.doris.thrift.TQueryPlanInfo;
|
||||
import org.apache.doris.thrift.TScanRangeLocations;
|
||||
import org.apache.doris.thrift.TTabletVersionInfo;
|
||||
import org.apache.doris.thrift.TUniqueId;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.thrift.TException;
|
||||
import org.apache.thrift.TSerializer;
|
||||
import org.codehaus.jackson.map.ObjectMapper;
|
||||
import org.json.simple.JSONObject;
|
||||
import org.json.simple.JSONValue;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
|
||||
/**
|
||||
* This class responsible for parse the sql and generate the query plan fragment for a (only one) table{@see OlapTable}
|
||||
* the related tablet maybe pruned by query planer according the `where` predicate.
|
||||
*/
|
||||
public class TableQueryPlanAction extends RestBaseAction {
|
||||
|
||||
public static final Logger LOG = LogManager.getLogger(TableQueryPlanAction.class);
|
||||
|
||||
public TableQueryPlanAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.POST,
|
||||
"/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_query_plan",
|
||||
new TableQueryPlanAction(controller));
|
||||
controller.registerHandler(HttpMethod.GET,
|
||||
"/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_query_plan",
|
||||
new TableQueryPlanAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void executeWithoutPassword(BaseRequest request, BaseResponse response)
|
||||
throws DdlException {
|
||||
// just allocate 2 slot for top holder map
|
||||
Map<String, Object> resultMap = new HashMap<>(4);
|
||||
String dbName = request.getSingleParameter(DB_KEY);
|
||||
String tableName = request.getSingleParameter(TABLE_KEY);
|
||||
String postContent = request.getContent();
|
||||
try {
|
||||
// may be these common validate logic should be moved to one base class
|
||||
if (Strings.isNullOrEmpty(dbName)
|
||||
|| Strings.isNullOrEmpty(tableName)) {
|
||||
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "{database}/{table} must be selected");
|
||||
}
|
||||
if (Strings.isNullOrEmpty(postContent)) {
|
||||
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "POST body must contains [sql] root object");
|
||||
}
|
||||
JSONObject jsonObject = (JSONObject) JSONValue.parse(postContent);
|
||||
if (jsonObject == null) {
|
||||
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "malformed json [ " + postContent + " ]");
|
||||
}
|
||||
String sql = (String) jsonObject.get("sql");
|
||||
if (Strings.isNullOrEmpty(sql)) {
|
||||
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "POST body must contains [sql] root object");
|
||||
}
|
||||
LOG.info("receive SQL statement [{}] from external service [ user [{}]] for database [{}] table [{}]",
|
||||
sql, ConnectContext.get().getCurrentUserIdentity(), dbName, tableName);
|
||||
|
||||
String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), dbName);
|
||||
// check privilege for select, otherwise return HTTP 401
|
||||
checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tableName, PrivPredicate.SELECT);
|
||||
Table table;
|
||||
try {
|
||||
Database db = Catalog.getCurrentCatalog().getDbOrMetaException(fullDbName);
|
||||
table = db.getTableOrMetaException(tableName, Table.TableType.OLAP);
|
||||
} catch (MetaNotFoundException e) {
|
||||
throw new DorisHttpException(HttpResponseStatus.FORBIDDEN, e.getMessage());
|
||||
}
|
||||
|
||||
// may be should acquire writeLock
|
||||
table.readLock();
|
||||
try {
|
||||
// parse/analysis/plan the sql and acquire tablet distributions
|
||||
handleQuery(ConnectContext.get(), fullDbName, tableName, sql, resultMap);
|
||||
} finally {
|
||||
table.readUnlock();
|
||||
}
|
||||
} catch (DorisHttpException e) {
|
||||
// status code should conforms to HTTP semantic
|
||||
resultMap.put("status", e.getCode().code());
|
||||
resultMap.put("exception", e.getMessage());
|
||||
}
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
try {
|
||||
String result = mapper.writeValueAsString(resultMap);
|
||||
// send result with extra information
|
||||
response.setContentType("application/json");
|
||||
response.getContent().append(result);
|
||||
sendResult(request, response, HttpResponseStatus.valueOf(Integer.parseInt(String.valueOf(resultMap.get("status")))));
|
||||
} catch (Exception e) {
|
||||
// may be this never happen
|
||||
response.getContent().append(e.getMessage());
|
||||
sendResult(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* process the sql syntax and return the resolved pruned tablet
|
||||
*
|
||||
* @param context context for analyzer
|
||||
* @param sql the single table select statement
|
||||
* @param result the acquired results
|
||||
* @return
|
||||
* @throws DorisHttpException
|
||||
*/
|
||||
private void handleQuery(ConnectContext context, String requestDb, String requestTable, String sql,
|
||||
Map<String, Object> result) throws DorisHttpException {
|
||||
// use SE to resolve sql
|
||||
StmtExecutor stmtExecutor = new StmtExecutor(context, new OriginStatement(sql, 0), false);
|
||||
try {
|
||||
TQueryOptions tQueryOptions = context.getSessionVariable().toThrift();
|
||||
// Conduct Planner create SingleNodePlan#createPlanFragments
|
||||
tQueryOptions.num_nodes = 1;
|
||||
// analyze sql
|
||||
stmtExecutor.analyze(tQueryOptions);
|
||||
} catch (Exception e) {
|
||||
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, e.getMessage());
|
||||
}
|
||||
// the parsed logical statement
|
||||
StatementBase query = stmtExecutor.getParsedStmt();
|
||||
// only process select semantic
|
||||
if (!(query instanceof SelectStmt)) {
|
||||
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "Select statement needed, but found [" + sql + " ]");
|
||||
}
|
||||
SelectStmt stmt = (SelectStmt) query;
|
||||
// just only process sql like `select * from table where <predicate>`, only support executing scan semantic
|
||||
if (stmt.hasAggInfo() || stmt.hasAnalyticInfo()
|
||||
|| stmt.hasOrderByClause() || stmt.hasOffset() || stmt.hasLimit() || stmt.isExplain()) {
|
||||
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "only support single table filter-prune-scan, but found [ " + sql + "]");
|
||||
}
|
||||
// process only one table by one http query
|
||||
List<TableRef> fromTables = stmt.getTableRefs();
|
||||
if (fromTables.size() != 1) {
|
||||
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "Select statement must have only one table");
|
||||
}
|
||||
|
||||
TableRef fromTable = fromTables.get(0);
|
||||
if (fromTable instanceof InlineViewRef) {
|
||||
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "Select statement must not embed another statement");
|
||||
}
|
||||
// check consistent http requested resource with sql referenced
|
||||
// if consistent in this way, can avoid check privilege
|
||||
TableName tableAndDb = fromTables.get(0).getName();
|
||||
if (!(tableAndDb.getDb().equals(requestDb) && tableAndDb.getTbl().equals(requestTable))) {
|
||||
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "requested database and table must consistent with sql: request [ "
|
||||
+ requestDb + "." + requestTable + "]" + "and sql [" + tableAndDb.toString() + "]");
|
||||
}
|
||||
|
||||
// acquired Planner to get PlanNode and fragment templates
|
||||
Planner planner = stmtExecutor.planner();
|
||||
// acquire ScanNode to obtain pruned tablet
|
||||
// in this way, just retrieve only one scannode
|
||||
List<ScanNode> scanNodes = planner.getScanNodes();
|
||||
if (scanNodes.size() != 1) {
|
||||
throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Planner should plan just only one ScanNode but found [ " + scanNodes.size() + "]");
|
||||
}
|
||||
List<TScanRangeLocations> scanRangeLocations = scanNodes.get(0).getScanRangeLocations(0);
|
||||
// acquire the PlanFragment which the executable template
|
||||
List<PlanFragment> fragments = planner.getFragments();
|
||||
if (fragments.size() != 1) {
|
||||
throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Planner should plan just only one PlanFragment but found [ " + fragments.size() + "]");
|
||||
}
|
||||
|
||||
TQueryPlanInfo tQueryPlanInfo = new TQueryPlanInfo();
|
||||
|
||||
|
||||
// acquire TPlanFragment
|
||||
TPlanFragment tPlanFragment = fragments.get(0).toThrift();
|
||||
// set up TMemoryScratchSink
|
||||
TDataSink tDataSink = new TDataSink();
|
||||
tDataSink.type = TDataSinkType.MEMORY_SCRATCH_SINK;
|
||||
tDataSink.memory_scratch_sink = new TMemoryScratchSink();
|
||||
tPlanFragment.output_sink = tDataSink;
|
||||
|
||||
tQueryPlanInfo.plan_fragment = tPlanFragment;
|
||||
tQueryPlanInfo.desc_tbl = query.getAnalyzer().getDescTbl().toThrift();
|
||||
// set query_id
|
||||
UUID uuid = UUID.randomUUID();
|
||||
tQueryPlanInfo.query_id = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits());
|
||||
|
||||
Map<Long, TTabletVersionInfo> tablet_info = new HashMap<>();
|
||||
// acquire resolved tablet distribution
|
||||
Map<String, Node> tabletRoutings = assemblePrunedPartitions(scanRangeLocations);
|
||||
tabletRoutings.forEach((tabletId, node) -> {
|
||||
long tablet = Long.parseLong(tabletId);
|
||||
tablet_info.put(tablet, new TTabletVersionInfo(tablet, node.version, 0l /*versionHash*/, node.schemaHash));
|
||||
});
|
||||
tQueryPlanInfo.tablet_info = tablet_info;
|
||||
|
||||
// serialize TQueryPlanInfo and encode plan with Base64 to string in order to translate by json format
|
||||
TSerializer serializer = new TSerializer();
|
||||
String opaqued_query_plan;
|
||||
try {
|
||||
byte[] query_plan_stream = serializer.serialize(tQueryPlanInfo);
|
||||
opaqued_query_plan = Base64.getEncoder().encodeToString(query_plan_stream);
|
||||
} catch (TException e) {
|
||||
throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "TSerializer failed to serialize PlanFragment, reason [ " + e.getMessage() + " ]");
|
||||
}
|
||||
result.put("partitions", tabletRoutings);
|
||||
result.put("opaqued_query_plan", opaqued_query_plan);
|
||||
result.put("status", 200);
|
||||
}
|
||||
|
||||
/**
|
||||
* acquire all involved (already pruned) tablet routing
|
||||
*
|
||||
* @param scanRangeLocationsList
|
||||
* @return
|
||||
*/
|
||||
private Map<String, Node> assemblePrunedPartitions(List<TScanRangeLocations> scanRangeLocationsList) {
|
||||
Map<String, Node> result = new HashMap<>();
|
||||
for (TScanRangeLocations scanRangeLocations : scanRangeLocationsList) {
|
||||
// only process palo(doris) scan range
|
||||
TPaloScanRange scanRange = scanRangeLocations.scan_range.palo_scan_range;
|
||||
Node tabletRouting = new Node(Long.parseLong(scanRange.version), Integer.parseInt(scanRange.schema_hash));
|
||||
for (TNetworkAddress address : scanRange.hosts) {
|
||||
tabletRouting.addRouting(address.hostname + ":" + address.port);
|
||||
}
|
||||
result.put(String.valueOf(scanRange.tablet_id), tabletRouting);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// helper class for json transformation
|
||||
final class Node {
|
||||
// ["host1:port1", "host2:port2", "host3:port3"]
|
||||
public List<String> routings = new ArrayList<>();
|
||||
public long version;
|
||||
public int schemaHash;
|
||||
|
||||
public Node(long version, int schemaHash) {
|
||||
this.version = version;
|
||||
this.schemaHash = schemaHash;
|
||||
}
|
||||
|
||||
private void addRouting(String routing) {
|
||||
routings.add(routing);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,113 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.catalog.Database;
|
||||
import org.apache.doris.catalog.OlapTable;
|
||||
import org.apache.doris.catalog.Table;
|
||||
import org.apache.doris.cluster.ClusterNamespace;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.common.DorisHttpException;
|
||||
import org.apache.doris.common.MetaNotFoundException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import org.codehaus.jackson.map.ObjectMapper;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
|
||||
/**
|
||||
* This class is responsible for fetch the approximate row count of the specified table from cluster-meta data,
|
||||
* the approximate row maybe used for some computing system to decide use which compute-algorithm can be used
|
||||
* such as shuffle join or broadcast join.
|
||||
* <p>
|
||||
* This API is not intended to compute the exact row count of the specified table, if you need the exact row count,
|
||||
* please consider using the sql syntax `select count(*) from {table}`
|
||||
*/
|
||||
public class TableRowCountAction extends RestBaseAction {
|
||||
public TableRowCountAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
controller.registerHandler(HttpMethod.GET,
|
||||
"/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_count",
|
||||
new TableRowCountAction(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void executeWithoutPassword(BaseRequest request, BaseResponse response)
|
||||
throws DdlException {
|
||||
// just allocate 2 slot for top holder map
|
||||
Map<String, Object> resultMap = new HashMap<>(4);
|
||||
String dbName = request.getSingleParameter(DB_KEY);
|
||||
String tableName = request.getSingleParameter(TABLE_KEY);
|
||||
try {
|
||||
if (Strings.isNullOrEmpty(dbName)
|
||||
|| Strings.isNullOrEmpty(tableName)) {
|
||||
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "{database}/{table} must be selected");
|
||||
}
|
||||
String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), dbName);
|
||||
// check privilege for select, otherwise return HTTP 401
|
||||
checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tableName, PrivPredicate.SELECT);
|
||||
Table table;
|
||||
try {
|
||||
Database db = Catalog.getCurrentCatalog().getDbOrMetaException(fullDbName);
|
||||
table = db.getTableOrMetaException(tableName, Table.TableType.OLAP);
|
||||
} catch (MetaNotFoundException e) {
|
||||
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, e.getMessage());
|
||||
}
|
||||
|
||||
table.readLock();
|
||||
try {
|
||||
OlapTable olapTable = (OlapTable) table;
|
||||
resultMap.put("status", 200);
|
||||
resultMap.put("size", olapTable.proximateRowCount());
|
||||
} finally {
|
||||
table.readUnlock();
|
||||
}
|
||||
} catch (DorisHttpException e) {
|
||||
// status code should conforms to HTTP semantic
|
||||
resultMap.put("status", e.getCode().code());
|
||||
resultMap.put("exception", e.getMessage());
|
||||
}
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
try {
|
||||
String result = mapper.writeValueAsString(resultMap);
|
||||
// send result with extra information
|
||||
response.setContentType("application/json");
|
||||
response.getContent().append(result);
|
||||
sendResult(request, response, HttpResponseStatus.valueOf(Integer.parseInt(String.valueOf(resultMap.get("status")))));
|
||||
} catch (Exception e) {
|
||||
// may be this never happen
|
||||
response.getContent().append(e.getMessage());
|
||||
sendResult(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,135 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.catalog.Column;
|
||||
import org.apache.doris.catalog.Database;
|
||||
import org.apache.doris.catalog.PrimitiveType;
|
||||
import org.apache.doris.catalog.ScalarType;
|
||||
import org.apache.doris.catalog.Table;
|
||||
import org.apache.doris.catalog.Type;
|
||||
import org.apache.doris.cluster.ClusterNamespace;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.common.DorisHttpException;
|
||||
import org.apache.doris.common.MetaNotFoundException;
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.BaseRequest;
|
||||
import org.apache.doris.http.BaseResponse;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
import org.codehaus.jackson.map.ObjectMapper;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
|
||||
/**
|
||||
* Get table schema for specified cluster.database.table with privilege checking
|
||||
*/
|
||||
public class TableSchemaAction extends RestBaseAction {
|
||||
|
||||
public TableSchemaAction(ActionController controller) {
|
||||
super(controller);
|
||||
}
|
||||
|
||||
public static void registerAction(ActionController controller) throws IllegalArgException {
|
||||
// the extra `/api` path is so disgusting
|
||||
controller.registerHandler(HttpMethod.GET,
|
||||
"/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_schema", new TableSchemaAction
|
||||
(controller));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
|
||||
// just allocate 2 slot for top holder map
|
||||
Map<String, Object> resultMap = new HashMap<>(2);
|
||||
String dbName = request.getSingleParameter(DB_KEY);
|
||||
String tableName = request.getSingleParameter(TABLE_KEY);
|
||||
try {
|
||||
if (Strings.isNullOrEmpty(dbName)
|
||||
|| Strings.isNullOrEmpty(tableName)) {
|
||||
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "No database or table selected.");
|
||||
}
|
||||
String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), dbName);
|
||||
// check privilege for select, otherwise return 401 HTTP status
|
||||
checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tableName, PrivPredicate.SELECT);
|
||||
Table table;
|
||||
try {
|
||||
Database db = Catalog.getCurrentCatalog().getDbOrMetaException(fullDbName);
|
||||
table = db.getTableOrMetaException(tableName, Table.TableType.OLAP);
|
||||
} catch (MetaNotFoundException e) {
|
||||
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, e.getMessage());
|
||||
}
|
||||
|
||||
table.readLock();
|
||||
try {
|
||||
List<Column> columns = table.getBaseSchema();
|
||||
List<Map<String, String>> propList = new ArrayList(columns.size());
|
||||
for (Column column : columns) {
|
||||
Map<String, String> baseInfo = new HashMap<>(2);
|
||||
Type colType = column.getOriginType();
|
||||
PrimitiveType primitiveType = colType.getPrimitiveType();
|
||||
if (primitiveType == PrimitiveType.DECIMALV2) {
|
||||
ScalarType scalarType = (ScalarType) colType;
|
||||
baseInfo.put("precision", scalarType.getPrecision() + "");
|
||||
baseInfo.put("scale", scalarType.getScalarScale() + "");
|
||||
}
|
||||
baseInfo.put("type", primitiveType.toString());
|
||||
baseInfo.put("comment", column.getComment());
|
||||
baseInfo.put("name", column.getDisplayName());
|
||||
Optional aggregationType = Optional.ofNullable(column.getAggregationType());
|
||||
baseInfo.put("aggregation_type", aggregationType.isPresent() ? column.getAggregationType().toSql() : "");
|
||||
propList.add(baseInfo);
|
||||
}
|
||||
resultMap.put("status", 200);
|
||||
resultMap.put("properties", propList);
|
||||
} catch (Exception e) {
|
||||
// Transform the general Exception to custom DorisHttpException
|
||||
throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, e.getMessage() == null ? "Null Pointer Exception" : e.getMessage());
|
||||
} finally {
|
||||
table.readUnlock();
|
||||
}
|
||||
} catch (DorisHttpException e) {
|
||||
// status code should conforms to HTTP semantic
|
||||
resultMap.put("status", e.getCode().code());
|
||||
resultMap.put("exception", e.getMessage());
|
||||
}
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
try {
|
||||
String result = mapper.writeValueAsString(resultMap);
|
||||
// send result with extra information
|
||||
response.setContentType("application/json");
|
||||
response.getContent().append(result);
|
||||
sendResult(request, response, HttpResponseStatus.valueOf(Integer.parseInt(String.valueOf(resultMap.get("status")))));
|
||||
} catch (Exception e) {
|
||||
// may be this never happen
|
||||
response.getContent().append(e.getMessage());
|
||||
sendResult(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -14,21 +14,20 @@
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http;
|
||||
package org.apache.doris.httpv2;
|
||||
|
||||
public class IllegalArgException extends Exception {
|
||||
private static final long serialVersionUID = 3344697787301861667L;
|
||||
private static final long serialVersionUID = 3344697787301861667L;
|
||||
|
||||
public IllegalArgException() {
|
||||
super("");
|
||||
}
|
||||
public IllegalArgException() {
|
||||
super("");
|
||||
}
|
||||
|
||||
public IllegalArgException(String msg) {
|
||||
super(msg);
|
||||
}
|
||||
public IllegalArgException(String msg) {
|
||||
super(msg);
|
||||
}
|
||||
|
||||
public IllegalArgException(String msg, Throwable cause) {
|
||||
super(msg, cause);
|
||||
}
|
||||
}
|
||||
public IllegalArgException(String msg, Throwable cause) {
|
||||
super(msg, cause);
|
||||
}
|
||||
}
|
||||
@ -15,7 +15,7 @@
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
package org.apache.doris.httpv2.entity;
|
||||
|
||||
// Status of RESTful action
|
||||
public enum ActionStatus {
|
||||
@ -14,14 +14,14 @@
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
package org.apache.doris.httpv2.entity;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
|
||||
// Base restful result
|
||||
public class RestBaseResult {
|
||||
|
||||
private static final RestBaseResult OK = new RestBaseResult();
|
||||
public ActionStatus status;
|
||||
public String msg;
|
||||
@ -15,7 +15,7 @@
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.http.rest;
|
||||
package org.apache.doris.httpv2.entity;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
@ -23,6 +23,7 @@ import com.google.common.collect.Maps;
|
||||
import com.google.gson.Gson;
|
||||
|
||||
public class RestResult extends RestBaseResult {
|
||||
|
||||
private Map<String, Object> resultMap;
|
||||
|
||||
public RestResult() {
|
||||
@ -38,7 +39,7 @@ public class RestResult extends RestBaseResult {
|
||||
Gson gson = new Gson();
|
||||
addResultEntry("status", status);
|
||||
if (status != ActionStatus.OK) {
|
||||
addResultEntry("msg", msg);
|
||||
addResultEntry("msg", msg);
|
||||
}
|
||||
return gson.toJson(resultMap);
|
||||
}
|
||||
@ -20,7 +20,7 @@ package org.apache.doris.httpv2.rest;
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.common.MetaNotFoundException;
|
||||
import org.apache.doris.http.rest.RestBaseResult;
|
||||
import org.apache.doris.httpv2.entity.RestBaseResult;
|
||||
import org.apache.doris.load.Load;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
|
||||
@ -20,7 +20,7 @@ package org.apache.doris.httpv2.rest;
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.cluster.ClusterNamespace;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.http.rest.RestBaseResult;
|
||||
import org.apache.doris.httpv2.entity.RestBaseResult;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.service.ExecuteEnv;
|
||||
|
||||
@ -19,7 +19,7 @@ package org.apache.doris.httpv2.rest;
|
||||
|
||||
import org.apache.doris.analysis.LoadStmt;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.http.rest.RestBaseResult;
|
||||
import org.apache.doris.httpv2.entity.RestBaseResult;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.service.ExecuteEnv;
|
||||
|
||||
@ -20,8 +20,8 @@ package org.apache.doris.journal.bdbje;
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.common.FeMetaVersion;
|
||||
import org.apache.doris.common.ThreadPoolManager;
|
||||
import org.apache.doris.http.HttpServer;
|
||||
import org.apache.doris.http.IllegalArgException;
|
||||
import org.apache.doris.httpv2.HttpServer;
|
||||
import org.apache.doris.httpv2.IllegalArgException;
|
||||
import org.apache.doris.journal.JournalEntity;
|
||||
import org.apache.doris.meta.MetaContext;
|
||||
import org.apache.doris.qe.QeService;
|
||||
@ -86,20 +86,10 @@ public class BDBDebugger {
|
||||
// Only start MySQL and HttpServer
|
||||
private void startService(String dorisHomeDir) throws IllegalArgException, IOException {
|
||||
// HTTP server
|
||||
if (!Config.enable_http_server_v2) {
|
||||
HttpServer httpServer = new HttpServer(
|
||||
Config.http_port,
|
||||
Config.http_max_line_length,
|
||||
Config.http_max_header_size,
|
||||
Config.http_max_chunk_size
|
||||
);
|
||||
httpServer.setup();
|
||||
httpServer.start();
|
||||
} else {
|
||||
org.apache.doris.httpv2.HttpServer httpServer2 = new org.apache.doris.httpv2.HttpServer();
|
||||
httpServer2.setPort(Config.http_port);
|
||||
httpServer2.start(dorisHomeDir);
|
||||
}
|
||||
|
||||
HttpServer httpServer = new HttpServer();
|
||||
httpServer.setPort(Config.http_port);
|
||||
httpServer.start(dorisHomeDir);
|
||||
|
||||
// MySQl server
|
||||
QeService qeService = new QeService(Config.query_port, Config.mysql_service_nio_enabled, ExecuteEnv.getInstance().getScheduler());
|
||||
|
||||
@ -19,11 +19,6 @@ package org.apache.doris.common.path;
|
||||
|
||||
import static com.google.common.collect.Maps.newHashMap;
|
||||
|
||||
import org.apache.doris.http.ActionController;
|
||||
import org.apache.doris.http.rest.MultiStart;
|
||||
import org.apache.doris.http.rest.TableQueryPlanAction;
|
||||
import org.apache.doris.http.rest.TableRowCountAction;
|
||||
import org.apache.doris.http.rest.TableSchemaAction;
|
||||
import org.apache.doris.service.ExecuteEnv;
|
||||
|
||||
import com.google.common.collect.Maps;
|
||||
@ -165,18 +160,4 @@ public class PathTrieTest {
|
||||
Assert.assertEquals(params.get("test"), "*");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInsert(@Injectable ActionController controller,
|
||||
@Injectable ExecuteEnv execEnv) {
|
||||
PathTrie pathTrie = new PathTrie();
|
||||
pathTrie.insert("/api/{db}/_multi_start", new MultiStart(controller, execEnv));
|
||||
pathTrie.insert("/api/{db}/{table}/_count", new TableRowCountAction(controller));
|
||||
pathTrie.insert("/api/{db}/{table}/_schema", new TableSchemaAction(controller));
|
||||
pathTrie.insert("/api/{db}/{table}/_query_plan", new TableQueryPlanAction(controller));
|
||||
Map<String, String> params = Maps.newHashMap();
|
||||
pathTrie.retrieve("/api/test/_multi_start", params);
|
||||
Assert.assertEquals("test", params.get("db"));
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@ -38,9 +38,12 @@ import org.apache.doris.catalog.Tablet;
|
||||
import org.apache.doris.catalog.TabletInvertedIndex;
|
||||
import org.apache.doris.catalog.TabletMeta;
|
||||
import org.apache.doris.common.AnalysisException;
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.common.ExceptionChecker.ThrowingRunnable;
|
||||
import org.apache.doris.common.jmockit.Deencapsulation;
|
||||
import org.apache.doris.httpv2.HttpServer;
|
||||
import org.apache.doris.httpv2.IllegalArgException;
|
||||
import org.apache.doris.load.Load;
|
||||
import org.apache.doris.mysql.privilege.PaloAuth;
|
||||
import org.apache.doris.persist.EditLog;
|
||||
@ -110,6 +113,8 @@ abstract public class DorisHttpTestCase {
|
||||
|
||||
protected String rootAuth = Credentials.basic("root", "");
|
||||
|
||||
public static final String DORIS_HOME_DIR = System.getenv("DORIS_HOME");
|
||||
|
||||
@Mocked
|
||||
private static EditLog editLog;
|
||||
|
||||
@ -291,13 +296,13 @@ abstract public class DorisHttpTestCase {
|
||||
}
|
||||
}
|
||||
|
||||
httpServer = new HttpServer(HTTP_PORT);
|
||||
httpServer.setup();
|
||||
httpServer.start();
|
||||
// must ensure the http server started before any unit test
|
||||
while (!httpServer.isStarted()) {
|
||||
Thread.sleep(500);
|
||||
}
|
||||
httpServer = new HttpServer();
|
||||
httpServer.setPort(Config.http_port);
|
||||
httpServer.setMaxHttpPostSize(Config.jetty_server_max_http_post_size);
|
||||
httpServer.setAcceptors(Config.jetty_server_acceptors);
|
||||
httpServer.setSelectors(Config.jetty_server_selectors);
|
||||
httpServer.setWorkers(Config.jetty_server_workers);
|
||||
httpServer.start(DORIS_HOME_DIR);
|
||||
}
|
||||
|
||||
|
||||
@ -337,10 +342,6 @@ abstract public class DorisHttpTestCase {
|
||||
public void tearDown() {
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void closeHttpServer() {
|
||||
httpServer.shutDown();
|
||||
}
|
||||
|
||||
public void doSetUp() {
|
||||
|
||||
|
||||
@ -17,9 +17,12 @@
|
||||
|
||||
package org.apache.doris.http;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import org.apache.doris.analysis.UserIdentity;
|
||||
import org.apache.doris.http.HttpAuthManager.SessionValue;
|
||||
import org.apache.doris.httpv2.HttpAuthManager.SessionValue;
|
||||
|
||||
import org.apache.doris.httpv2.HttpAuthManager;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
@ -31,15 +34,20 @@ public class HttpAuthManagerTest {
|
||||
HttpAuthManager authMgr = HttpAuthManager.getInstance();
|
||||
String sessionId = "test_session_id";
|
||||
String username = "test-user";
|
||||
SessionValue sessionValue = new SessionValue();
|
||||
HttpAuthManager.SessionValue sessionValue = new HttpAuthManager.SessionValue();
|
||||
sessionValue.currentUser = UserIdentity.createAnalyzedUserIdentWithIp(username, "%");
|
||||
authMgr.addSessionValue(sessionId, sessionValue);
|
||||
Assert.assertEquals(1, authMgr.getAuthSessions().size());
|
||||
System.out.println("username in test: " + authMgr.getSessionValue(sessionId).currentUser);
|
||||
Assert.assertEquals(username, authMgr.getSessionValue(sessionId).currentUser.getQualifiedUser());
|
||||
List<String> sessionIds = new ArrayList<>();
|
||||
sessionIds.add (sessionId);
|
||||
System.out.println("username in test: " + authMgr.getSessionValue(sessionIds).currentUser);
|
||||
Assert.assertEquals(username, authMgr.getSessionValue(sessionIds).currentUser.getQualifiedUser());
|
||||
|
||||
|
||||
String noExistSession = "no-exist-session-id";
|
||||
Assert.assertNull(authMgr.getSessionValue(noExistSession));
|
||||
sessionIds.clear();
|
||||
sessionIds.add (noExistSession);
|
||||
Assert.assertNull(authMgr.getSessionValue(sessionIds));
|
||||
Assert.assertEquals(1, authMgr.getAuthSessions().size());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user