From e52f212ab89c2ac1da7ab801c7b676398ea6d1c1 Mon Sep 17 00:00:00 2001 From: "jiafeng.zhang" Date: Wed, 6 Apr 2022 09:19:36 +0800 Subject: [PATCH 1/3] remove http v1 http v2 has been actually tested in production, and it is completely replaceable to have http code. In order to simplify code maintenance, remove the previous http part of the code --- .../main/java/org/apache/doris/PaloFe.java | 29 +- .../org/apache/doris/catalog/Catalog.java | 2 +- .../java/org/apache/doris/common/Config.java | 16 - .../apache/doris/http/ActionController.java | 88 ---- .../org/apache/doris/http/BaseAction.java | 375 ------------------ .../org/apache/doris/http/BaseRequest.java | 158 -------- .../org/apache/doris/http/BaseResponse.java | 99 ----- .../apache/doris/http/HttpAuthManager.java | 64 --- .../org/apache/doris/http/HttpServer.java | 279 ------------- .../apache/doris/http/HttpServerHandler.java | 117 ------ .../java/org/apache/doris/http/IAction.java | 25 -- .../doris/http/UnauthorizedException.java | 26 -- .../java/org/apache/doris/http/WebUtils.java | 230 ----------- .../doris/http/action/BackendAction.java | 101 ----- .../apache/doris/http/action/HaAction.java | 174 -------- .../apache/doris/http/action/HelpAction.java | 223 ----------- .../apache/doris/http/action/IndexAction.java | 275 ------------- .../apache/doris/http/action/LogAction.java | 168 -------- .../doris/http/action/NotFoundAction.java | 52 --- .../apache/doris/http/action/QueryAction.java | 127 ------ .../doris/http/action/QueryProfileAction.java | 68 ---- .../doris/http/action/SessionAction.java | 85 ---- .../http/action/StaticResourceAction.java | 242 ----------- .../doris/http/action/SystemAction.java | 184 --------- .../doris/http/action/VariableAction.java | 84 ---- .../doris/http/action/WebBaseAction.java | 374 ----------------- .../common/DorisHttpPostObjectAggregator.java | 61 --- .../doris/http/meta/ColocateMetaService.java | 229 ----------- .../http/meta/InvalidClientException.java | 26 -- .../doris/http/meta/MetaBaseAction.java | 108 ----- .../apache/doris/http/meta/MetaService.java | 356 ----------------- .../http/rest/BootstrapFinishAction.java | 162 -------- .../doris/http/rest/CancelStreamLoad.java | 83 ---- .../http/rest/CheckDecommissionAction.java | 93 ----- .../doris/http/rest/ConnectionAction.java | 62 --- .../doris/http/rest/GetDdlStmtAction.java | 108 ----- .../doris/http/rest/GetLoadInfoAction.java | 88 ---- .../doris/http/rest/GetLogFileAction.java | 131 ------ .../doris/http/rest/GetSmallFileAction.java | 92 ----- .../doris/http/rest/GetStreamLoadState.java | 86 ---- .../apache/doris/http/rest/HealthAction.java | 47 --- .../apache/doris/http/rest/LoadAction.java | 141 ------- .../http/rest/MetaReplayerCheckAction.java | 70 ---- .../apache/doris/http/rest/MetricsAction.java | 65 --- .../doris/http/rest/MigrationAction.java | 164 -------- .../apache/doris/http/rest/MultiAbort.java | 72 ---- .../apache/doris/http/rest/MultiCommit.java | 77 ---- .../org/apache/doris/http/rest/MultiDesc.java | 85 ---- .../org/apache/doris/http/rest/MultiList.java | 82 ---- .../apache/doris/http/rest/MultiStart.java | 92 ----- .../apache/doris/http/rest/MultiUnload.java | 76 ---- .../apache/doris/http/rest/ProfileAction.java | 60 --- .../doris/http/rest/QueryDetailAction.java | 56 --- .../doris/http/rest/RestBaseAction.java | 127 ------ .../doris/http/rest/RowCountAction.java | 115 ------ .../doris/http/rest/SetConfigAction.java | 136 ------- .../doris/http/rest/ShowDataAction.java | 91 ----- .../doris/http/rest/ShowMetaInfoAction.java | 203 ---------- .../doris/http/rest/ShowProcAction.java | 136 ------- .../http/rest/ShowRuntimeInfoAction.java | 69 ---- .../http/rest/StorageTypeCheckAction.java | 98 ----- .../doris/http/rest/TableQueryPlanAction.java | 306 -------------- .../doris/http/rest/TableRowCountAction.java | 113 ------ .../doris/http/rest/TableSchemaAction.java | 135 ------- .../{http => httpv2}/IllegalArgException.java | 25 +- .../rest => httpv2/entity}/ActionStatus.java | 6 +- .../entity}/RestBaseResult.java | 40 +- .../rest => httpv2/entity}/RestResult.java | 33 +- .../doris/httpv2/rest/GetLoadInfoAction.java | 2 +- .../apache/doris/httpv2/rest/LoadAction.java | 2 +- .../apache/doris/httpv2/rest/MultiAction.java | 2 +- .../doris/journal/bdbje/BDBDebugger.java | 22 +- .../doris/common/path/PathTrieTest.java | 19 - .../apache/doris/http/DorisHttpTestCase.java | 23 +- .../doris/http/HttpAuthManagerTest.java | 18 +- 75 files changed, 96 insertions(+), 8062 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/ActionController.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/BaseAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/BaseRequest.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/BaseResponse.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/HttpAuthManager.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/HttpServer.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/HttpServerHandler.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/IAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/UnauthorizedException.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/WebUtils.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/BackendAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/HaAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/HelpAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/IndexAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/LogAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/NotFoundAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/QueryAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/QueryProfileAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/SessionAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/StaticResourceAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/SystemAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/VariableAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/WebBaseAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/common/DorisHttpPostObjectAggregator.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/meta/ColocateMetaService.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/meta/InvalidClientException.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaBaseAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaService.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/BootstrapFinishAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/CancelStreamLoad.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/CheckDecommissionAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/ConnectionAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/GetDdlStmtAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLoadInfoAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLogFileAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/GetSmallFileAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/GetStreamLoadState.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/HealthAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/LoadAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/MetaReplayerCheckAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/MetricsAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/MigrationAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiAbort.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiCommit.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiDesc.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiList.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiStart.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiUnload.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/ProfileAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/QueryDetailAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/RowCountAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowDataAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowMetaInfoAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowProcAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowRuntimeInfoAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/StorageTypeCheckAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/TableRowCountAction.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/TableSchemaAction.java rename fe/fe-core/src/main/java/org/apache/doris/{http => httpv2}/IllegalArgException.java (72%) rename fe/fe-core/src/main/java/org/apache/doris/{http/rest => httpv2/entity}/ActionStatus.java (93%) rename fe/fe-core/src/main/java/org/apache/doris/{http/rest => httpv2/entity}/RestBaseResult.java (62%) rename fe/fe-core/src/main/java/org/apache/doris/{http/rest => httpv2/entity}/RestResult.java (65%) diff --git a/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java b/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java index c954055c5a08b5..c8ee60a4e1bd34 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java +++ b/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java @@ -26,7 +26,7 @@ import org.apache.doris.common.Version; import org.apache.doris.common.util.JdkUtils; import org.apache.doris.common.util.NetUtils; -import org.apache.doris.http.HttpServer; +import org.apache.doris.httpv2.HttpServer; import org.apache.doris.journal.bdbje.BDBDebugger; import org.apache.doris.journal.bdbje.BDBTool; import org.apache.doris.journal.bdbje.BDBToolOptions; @@ -134,25 +134,14 @@ public static void start(String dorisHomeDir, String pidDir, String[] args) { feServer.start(); - if (!Config.enable_http_server_v2) { - HttpServer httpServer = new HttpServer( - Config.http_port, - Config.http_max_line_length, - Config.http_max_header_size, - Config.http_max_chunk_size - ); - httpServer.setup(); - httpServer.start(); - } else { - org.apache.doris.httpv2.HttpServer httpServer2 = new org.apache.doris.httpv2.HttpServer(); - httpServer2.setPort(Config.http_port); - httpServer2.setMaxHttpPostSize(Config.jetty_server_max_http_post_size); - httpServer2.setAcceptors(Config.jetty_server_acceptors); - httpServer2.setSelectors(Config.jetty_server_selectors); - httpServer2.setWorkers(Config.jetty_server_workers); - httpServer2.start(dorisHomeDir); - } - + HttpServer httpServer = new HttpServer(); + httpServer.setPort(Config.http_port); + httpServer.setMaxHttpPostSize(Config.jetty_server_max_http_post_size); + httpServer.setAcceptors(Config.jetty_server_acceptors); + httpServer.setSelectors(Config.jetty_server_selectors); + httpServer.setWorkers(Config.jetty_server_workers); + httpServer.start(dorisHomeDir); + qeService.start(); ThreadPoolManager.registerAllThreadPoolMetric(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index 57a066680fe0e3..71ebf3c5bfef84 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -156,7 +156,7 @@ import org.apache.doris.ha.FrontendNodeType; import org.apache.doris.ha.HAProtocol; import org.apache.doris.ha.MasterInfo; -import org.apache.doris.http.meta.MetaBaseAction; +import org.apache.doris.httpv2.meta.MetaBaseAction; import org.apache.doris.journal.JournalCursor; import org.apache.doris.journal.JournalEntity; import org.apache.doris.journal.bdbje.Timestamp; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java index 040e8ca092b765..355477d9fcf963 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java @@ -327,22 +327,6 @@ public class Config extends ConfigBase { */ @ConfField public static int http_port = 8030; - /* - * Netty http param - */ - @ConfField public static int http_max_line_length = HttpServer.DEFAULT_MAX_LINE_LENGTH; - - @ConfField public static int http_max_header_size = HttpServer.DEFAULT_MAX_HEADER_SIZE; - - @ConfField public static int http_max_chunk_size = HttpServer.DEFAULT_MAX_CHUNK_SIZE; - - /** - * The backlog_num for netty http server - * When you enlarge this backlog_num, you should enlarge the value in - * the linux /proc/sys/net/core/somaxconn file at the same time - */ - @ConfField public static int http_backlog_num = 1024; - /** * Jetty container default configuration * Jetty's thread architecture model is very simple, divided into three thread pools: diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/ActionController.java b/fe/fe-core/src/main/java/org/apache/doris/http/ActionController.java deleted file mode 100644 index 8264f9169f7f6f..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/ActionController.java +++ /dev/null @@ -1,88 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - -import org.apache.doris.common.path.PathTrie; -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; - -public class ActionController { - private final PathTrie getHandlers = new PathTrie<>(WebUtils.REST_DECODER); - private final PathTrie postHandlers = new PathTrie<>(WebUtils.REST_DECODER); - private final PathTrie putHandlers = new PathTrie<>(WebUtils.REST_DECODER); - private final PathTrie deleteHandlers = new PathTrie<>(WebUtils.REST_DECODER); - private final PathTrie headHandlers = new PathTrie<>(WebUtils.REST_DECODER); - private final PathTrie optionsHandlers = new PathTrie<>(WebUtils.REST_DECODER); - - // Registers a rest handler to be execute when the provided method and path match the request. - public void registerHandler(HttpMethod method, String path, IAction handler) - throws IllegalArgException { - if (method.equals(HttpMethod.GET)) { - getHandlers.insert(path, handler); - } else if (method.equals(HttpMethod.POST)) { - postHandlers.insert(path, handler); - } else if (method.equals(HttpMethod.HEAD)) { - headHandlers.insert(path, handler); - } else if (method.equals(HttpMethod.DELETE)) { - deleteHandlers.insert(path, handler); - } else if (method.equals(HttpMethod.OPTIONS)) { - optionsHandlers.insert(path, handler); - } else if (method.equals(HttpMethod.PUT)) { - putHandlers.insert(path, handler); - } else { - throw new IllegalArgException( - "Can't handle [" + method + "] for path [" + path + "]"); - } - } - - public IAction getHandler(BaseRequest request) { - String path = getPath(request.getRequest().uri()); - HttpMethod method = request.getRequest().method(); - if (method.equals(HttpMethod.GET)) { - return getHandlers.retrieve(path, request.getParams()); - } else if (method.equals(HttpMethod.POST)) { - return postHandlers.retrieve(path, request.getParams()); - } else if (method.equals(HttpMethod.PUT)) { - return putHandlers.retrieve(path, request.getParams()); - } else if (method.equals(HttpMethod.DELETE)) { - return deleteHandlers.retrieve(path, request.getParams()); - } else if (method.equals(HttpMethod.HEAD)) { - return headHandlers.retrieve(path, request.getParams()); - } else if (method.equals(HttpMethod.OPTIONS)) { - return optionsHandlers.retrieve(path, request.getParams()); - } else { - return null; - } - } - - // e.g. - // in: /www/system?path=//jobs - // out: /www/system - private String getPath(String uri) { - if (Strings.isNullOrEmpty(uri)) { - return ""; - } - int pathEndIndex = uri.indexOf('?'); - if (pathEndIndex < 0) { - return uri; - } else { - return uri.substring(0, pathEndIndex); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/BaseAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/BaseAction.java deleted file mode 100644 index b3df5e3e338718..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/BaseAction.java +++ /dev/null @@ -1,375 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - -import org.apache.doris.analysis.UserIdentity; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.system.SystemInfoService; - -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.io.UnsupportedEncodingException; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelProgressiveFuture; -import io.netty.channel.ChannelProgressiveFutureListener; -import io.netty.channel.DefaultFileRegion; -import io.netty.handler.codec.base64.Base64; -import io.netty.handler.codec.http.DefaultFullHttpResponse; -import io.netty.handler.codec.http.DefaultHttpResponse; -import io.netty.handler.codec.http.FullHttpResponse; -import io.netty.handler.codec.http.HttpChunkedInput; -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpHeaderValues; -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponse; -import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.codec.http.HttpUtil; -import io.netty.handler.codec.http.HttpVersion; -import io.netty.handler.codec.http.LastHttpContent; -import io.netty.handler.codec.http.cookie.Cookie; -import io.netty.handler.codec.http.cookie.ServerCookieEncoder; -import io.netty.handler.ssl.SslHandler; -import io.netty.handler.stream.ChunkedFile; -import io.netty.handler.stream.ChunkedInput; -import io.netty.handler.stream.ChunkedStream; -import io.netty.util.CharsetUtil; - -public abstract class BaseAction implements IAction { - private static final Logger LOG = LogManager.getLogger(BaseAction.class); - - protected ActionController controller; - protected Catalog catalog; - - public BaseAction(ActionController controller) { - this.controller = controller; - // TODO(zc): remove this instance - this.catalog = Catalog.getCurrentCatalog(); - } - - @Override - public void handleRequest(BaseRequest request) throws Exception { - BaseResponse response = new BaseResponse(); - LOG.debug("receive http request. url={}", request.getRequest().uri()); - try { - execute(request, response); - } catch (Exception e) { - LOG.warn("fail to process url: {}", request.getRequest().uri(), e); - if (e instanceof UnauthorizedException) { - response.updateHeader(HttpHeaderNames.WWW_AUTHENTICATE.toString(), "Basic realm=\"\""); - writeResponse(request, response, HttpResponseStatus.UNAUTHORIZED); - } else { - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); - } - } - } - - public abstract void execute(BaseRequest request, BaseResponse response) throws DdlException; - - protected void writeResponse(BaseRequest request, BaseResponse response, HttpResponseStatus status) { - // if (HttpHeaders.is100ContinueExpected(request.getRequest())) { - // ctx.write(new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, - // HttpResponseStatus.CONTINUE)); - // } - - FullHttpResponse responseObj = null; - try { - responseObj = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, status, - Unpooled.wrappedBuffer(response.getContent().toString().getBytes("UTF-8"))); - } catch (UnsupportedEncodingException e) { - LOG.warn("get exception.", e); - responseObj = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, status, - Unpooled.wrappedBuffer(response.getContent().toString().getBytes())); - } - Preconditions.checkNotNull(responseObj); - HttpMethod method = request.getRequest().method(); - - checkDefaultContentTypeHeader(response, responseObj); - if (!method.equals(HttpMethod.HEAD)) { - response.updateHeader(HttpHeaderNames.CONTENT_LENGTH.toString(), - String.valueOf(responseObj.content().readableBytes())); - } - writeCustomHeaders(response, responseObj); - writeCookies(response, responseObj); - - boolean keepAlive = HttpUtil.isKeepAlive(request.getRequest()); - if (!keepAlive) { - request.getContext().write(responseObj).addListener(ChannelFutureListener.CLOSE); - } else { - responseObj.headers().set(HttpHeaderNames.CONNECTION.toString(), HttpHeaderValues.KEEP_ALIVE.toString()); - request.getContext().write(responseObj); - } - } - - // Object only support File or byte[] - protected void writeObjectResponse(BaseRequest request, BaseResponse response, HttpResponseStatus status, - Object obj, String fileName, boolean isOctStream) { - Preconditions.checkState((obj instanceof File) || (obj instanceof byte[])); - - HttpResponse responseObj = new DefaultHttpResponse(HttpVersion.HTTP_1_1, status); - - if (HttpUtil.isKeepAlive(request.getRequest())) { - response.updateHeader(HttpHeaderNames.CONNECTION.toString(), HttpHeaderValues.KEEP_ALIVE.toString()); - } - - if (isOctStream) { - response.updateHeader(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.APPLICATION_OCTET_STREAM.toString()); - response.updateHeader(HttpHeaderNames.CONTENT_DISPOSITION.toString(), - HttpHeaderValues.ATTACHMENT.toString() + "; " + HttpHeaderValues.FILENAME.toString() + "=" + fileName); - } - - ChannelFuture sendFileFuture; - ChannelFuture lastContentFuture; - - try { - Object writable = null; - long contentLen = 0; - boolean sslEnable = request.getContext().pipeline().get(SslHandler.class) != null; - if (obj instanceof File) { - RandomAccessFile rafFile = new RandomAccessFile((File) obj, "r"); - contentLen = rafFile.length(); - if (!sslEnable) { - // use zero-copy file transfer. - writable = new DefaultFileRegion(rafFile.getChannel(), 0, contentLen); - } else { - // cannot use zero-copy file transfer. - writable = new ChunkedFile(rafFile, 0, contentLen, 8192); - } - } else if (obj instanceof byte[]) { - contentLen = ((byte[]) obj).length; - if (!sslEnable) { - writable = Unpooled.wrappedBuffer((byte[]) obj); - } else { - writable = new ChunkedStream(new ByteArrayInputStream((byte[]) obj)); - } - } - - response.updateHeader(HttpHeaderNames.CONTENT_LENGTH.toString(), String.valueOf(contentLen)); - writeCookies(response, responseObj); - writeCustomHeaders(response, responseObj); - - // Write headers - request.getContext().write(responseObj); - - // Write object - if (!sslEnable) { - sendFileFuture = request.getContext().write(writable, request.getContext().newProgressivePromise()); - // Write the end marker. - lastContentFuture = request.getContext().writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT); - } else { - sendFileFuture = request.getContext().writeAndFlush( - new HttpChunkedInput((ChunkedInput) writable), - request.getContext().newProgressivePromise()); - // HttpChunkedInput will write the end marker (LastHttpContent) for us. - lastContentFuture = sendFileFuture; - } - } catch (FileNotFoundException ignore) { - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); - return; - } catch (IOException e1) { - writeResponse(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR); - return; - } - - sendFileFuture.addListener(new ChannelProgressiveFutureListener() { - @Override - public void operationProgressed(ChannelProgressiveFuture future, long progress, long total) { - if (total < 0) { // total unknown - LOG.debug("{} Transfer progress: {}", future.channel(), progress); - } else { - LOG.debug("{} Transfer progress: {} / {}", future.channel(), progress, total); - } - } - - @Override - public void operationComplete(ChannelProgressiveFuture future) { - LOG.debug("{} Transfer complete.", future.channel()); - if (!future.isSuccess()) { - Throwable cause = future.cause(); - LOG.error("something wrong. ", cause); - } - } - }); - - // Decide whether to close the connection or not. - boolean keepAlive = HttpUtil.isKeepAlive(request.getRequest()); - if (!keepAlive) { - // Close the connection when the whole content is written out. - lastContentFuture.addListener(ChannelFutureListener.CLOSE); - } - } - - // Set 'CONTENT_TYPE' header if it hasn't been set. - protected void checkDefaultContentTypeHeader(BaseResponse response, Object responseOj) { - if (!Strings.isNullOrEmpty(response.getContentType())) { - response.updateHeader(HttpHeaderNames.CONTENT_TYPE.toString(), response.getContentType()); - } else { - response.updateHeader(HttpHeaderNames.CONTENT_TYPE.toString(), "text/html"); - } - } - - protected void writeCustomHeaders(BaseResponse response, HttpResponse responseObj) { - for (Map.Entry> entry : response.getCustomHeaders().entrySet()) { - responseObj.headers().add(entry.getKey(), entry.getValue()); - } - } - - protected void writeCookies(BaseResponse response, HttpResponse responseObj) { - for (Cookie cookie : response.getCookies()) { - responseObj.headers().add(HttpHeaderNames.SET_COOKIE.toString(), ServerCookieEncoder.LAX.encode(cookie)); - } - } - - public static class ActionAuthorizationInfo { - public String fullUserName; - public String remoteIp; - public String password; - public String cluster; - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("user: ").append(fullUserName).append(", remote ip: ").append(remoteIp); - sb.append(", password: ").append(password).append(", cluster: ").append(cluster); - return sb.toString(); - } - } - - protected void checkGlobalAuth(UserIdentity currentUser, PrivPredicate predicate) throws UnauthorizedException { - if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(currentUser, predicate)) { - throw new UnauthorizedException("Access denied; you need (at least one of) the " - + predicate.getPrivs().toString() + " privilege(s) for this operation"); - } - } - - protected void checkDbAuth(UserIdentity currentUser, String db, PrivPredicate predicate) - throws UnauthorizedException { - if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(currentUser, db, predicate)) { - throw new UnauthorizedException("Access denied; you need (at least one of) the " - + predicate.getPrivs().toString() + " privilege(s) for this operation"); - } - } - - protected void checkTblAuth(UserIdentity currentUser, String db, String tbl, PrivPredicate predicate) - throws UnauthorizedException { - if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(currentUser, db, tbl, predicate)) { - throw new UnauthorizedException("Access denied; you need (at least one of) the " - + predicate.getPrivs().toString() + " privilege(s) for this operation"); - } - } - - // return currentUserIdentity from Doris auth - protected UserIdentity checkPassword(ActionAuthorizationInfo authInfo) - throws UnauthorizedException { - List currentUser = Lists.newArrayList(); - if (!Catalog.getCurrentCatalog().getAuth().checkPlainPassword(authInfo.fullUserName, - authInfo.remoteIp, authInfo.password, currentUser)) { - throw new UnauthorizedException("Access denied for " - + authInfo.fullUserName + "@" + authInfo.remoteIp); - } - Preconditions.checkState(currentUser.size() == 1); - return currentUser.get(0); - } - - public ActionAuthorizationInfo getAuthorizationInfo(BaseRequest request) - throws UnauthorizedException { - ActionAuthorizationInfo authInfo = new ActionAuthorizationInfo(); - if (!parseAuthInfo(request, authInfo)) { - LOG.info("parse auth info failed, Authorization header {}, url {}", - request.getAuthorizationHeader(), request.getRequest().uri()); - throw new UnauthorizedException("Need auth information."); - } - LOG.debug("get auth info: {}", authInfo); - return authInfo; - } - - private boolean parseAuthInfo(BaseRequest request, ActionAuthorizationInfo authInfo) { - String encodedAuthString = request.getAuthorizationHeader(); - if (Strings.isNullOrEmpty(encodedAuthString)) { - return false; - } - String[] parts = encodedAuthString.split(" "); - if (parts.length != 2) { - return false; - } - encodedAuthString = parts[1]; - ByteBuf buf = null; - ByteBuf decodeBuf = null; - try { - buf = Unpooled.copiedBuffer(ByteBuffer.wrap(encodedAuthString.getBytes())); - - // The authString is a string connecting user-name and password with - // a colon(':') - decodeBuf = Base64.decode(buf); - String authString = decodeBuf.toString(CharsetUtil.UTF_8); - // Note that password may contain colon, so can not simply use a - // colon to split. - int index = authString.indexOf(":"); - authInfo.fullUserName = authString.substring(0, index); - final String[] elements = authInfo.fullUserName.split("@"); - if (elements != null && elements.length < 2) { - authInfo.fullUserName = ClusterNamespace.getFullName(SystemInfoService.DEFAULT_CLUSTER, - authInfo.fullUserName); - authInfo.cluster = SystemInfoService.DEFAULT_CLUSTER; - } else if (elements != null && elements.length == 2) { - authInfo.fullUserName = ClusterNamespace.getFullName(elements[1], elements[0]); - authInfo.cluster = elements[1]; - } - authInfo.password = authString.substring(index + 1); - authInfo.remoteIp = request.getHostString(); - } finally { - // release the buf and decode buf after using Unpooled.copiedBuffer - // or it will get memory leak - if (buf != null) { - buf.release(); - } - - if (decodeBuf != null) { - decodeBuf.release(); - } - } - return true; - } - - protected int checkIntParam(String strParam) { - return Integer.parseInt(strParam); - } - - protected long checkLongParam(String strParam) { - return Long.parseLong(strParam); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/BaseRequest.java b/fe/fe-core/src/main/java/org/apache/doris/http/BaseRequest.java deleted file mode 100644 index 81633a5cdc587c..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/BaseRequest.java +++ /dev/null @@ -1,158 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - -import org.apache.doris.common.DdlException; - -import com.google.common.base.Strings; -import com.google.common.collect.Maps; - -import java.net.InetSocketAddress; -import java.nio.charset.Charset; -import java.util.List; -import java.util.Map; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.FullHttpRequest; -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.QueryStringDecoder; -import io.netty.handler.codec.http.cookie.ClientCookieDecoder; -import io.netty.handler.codec.http.cookie.Cookie; - -public class BaseRequest { - protected ChannelHandlerContext context; - protected HttpRequest request; - protected Map params = Maps.newHashMap(); - - private boolean isAuthorized = false; - private QueryStringDecoder decoder; - - public BaseRequest(ChannelHandlerContext ctx, HttpRequest request) { - this.context = ctx; - this.request = request; - } - - public ChannelHandlerContext getContext() { - return context; - } - - public void setContext(ChannelHandlerContext context) { - this.context = context; - } - - public HttpRequest getRequest() { - return request; - } - - public void setRequest(HttpRequest request) { - this.request = request; - } - - public Map getParams() { - return params; - } - - public void setParams(Map params) { - this.params = params; - } - - public boolean isAuthorized() { - return isAuthorized; - } - - public void setAuthorized(boolean isAuthorized) { - this.isAuthorized = isAuthorized; - } - - public Cookie getCookieByName(String cookieName) { - String cookieString = request.headers().get(HttpHeaderNames.COOKIE.toString()); - if (!Strings.isNullOrEmpty(cookieString)) { - Cookie cookie = ClientCookieDecoder.STRICT.decode(cookieString); - if (cookie.name().equalsIgnoreCase(cookieName)) { - return cookie; - } - } - return null; - } - - public String getCookieValue(String cookieName) { - Cookie cookie = getCookieByName(cookieName); - if (cookie != null) { - return cookie.value(); - } - return null; - } - - // get a single parameter. - // return null if key is not exist; return the first value if key is an array - public String getSingleParameter(String key) { - String uri = request.uri(); - if (decoder == null) { - decoder = new QueryStringDecoder(uri); - } - - List values = decoder.parameters().get(key); - if (values != null && values.size() > 0) { - return values.get(0); - } - - return params.get(key); - } - - public String getContent() throws DdlException { - if (request instanceof FullHttpRequest) { - FullHttpRequest fullHttpRequest = (FullHttpRequest) request; - return fullHttpRequest.content().toString(Charset.forName("UTF-8")); - } else { - throw new DdlException("Invalid request"); - } - } - - // get an array parameter. - // eg. ?a=1&a=2 - public List getArrayParameter(String key) { - String uri = request.uri(); - if (decoder == null) { - decoder = new QueryStringDecoder(uri); - } - - return decoder.parameters().get(key); - } - - public Map> getAllParameters() { - String uri = request.uri(); - if (decoder == null) { - decoder = new QueryStringDecoder(uri); - } - - return decoder.parameters(); - } - - public String getAuthorizationHeader() { - String authString = request.headers().get("Authorization"); - return authString; - } - - public String getHostString() { - // get client host - InetSocketAddress clientSocket = (InetSocketAddress) context.channel().remoteAddress(); - String clientIp = clientSocket.getHostString(); - return clientIp; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/BaseResponse.java b/fe/fe-core/src/main/java/org/apache/doris/http/BaseResponse.java deleted file mode 100644 index 297c8c7f8c9474..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/BaseResponse.java +++ /dev/null @@ -1,99 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; - -import java.util.List; -import java.util.Map; -import java.util.Set; - -import io.netty.handler.codec.http.cookie.Cookie; - -public class BaseResponse { - private String contentType; - protected StringBuilder content = new StringBuilder(); - - protected Map> customHeaders = Maps.newHashMap(); - private Set cookies = Sets.newHashSet(); - - public String getContentType() { - return contentType; - } - - public void setContentType(String contentType) { - this.contentType = contentType; - } - - public StringBuilder getContent() { - return content; - } - - public Set getCookies() { - return cookies; - } - - public Map> getCustomHeaders() { - return customHeaders; - } - - // update old key-value mapping of 'name' if Exist, or add new mapping if not exists. - // It will only change the mapping of 'name', other header will not be changed. - public void updateHeader(String name, String value) { - if (customHeaders == null) { - customHeaders = Maps.newHashMap(); - } - customHeaders.remove(name); - addHeader(name, value); - } - - // Add a custom header. - private void addHeader(String name, String value) { - if (customHeaders == null) { - customHeaders = Maps.newHashMap(); - } - List header = customHeaders.get(name); - if (header == null) { - header = Lists.newArrayList(); - customHeaders.put(name, header); - } - header.add(value); - } - - public void appendContent(String buffer) { - if (content == null) { - content = new StringBuilder(); - } - content.append(buffer); - } - - public void addCookie(Cookie cookie) { - cookies.add(cookie); - } - - public void updateCookieAge(BaseRequest request, String cookieName, long age) { - Cookie cookie = request.getCookieByName(cookieName); - if (cookie != null) { - cookies.remove(cookie); - cookie.setMaxAge(age); - cookies.add(cookie); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/HttpAuthManager.java b/fe/fe-core/src/main/java/org/apache/doris/http/HttpAuthManager.java deleted file mode 100644 index d0cd55ef80cff0..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/HttpAuthManager.java +++ /dev/null @@ -1,64 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - -import org.apache.doris.analysis.UserIdentity; - -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; - -import java.util.concurrent.TimeUnit; - -// We simulate a simplified session here: only store user-name of clients who already logged in, -// and we only have a default admin user for now. -public final class HttpAuthManager { - private static long SESSION_EXPIRE_TIME = 2; // hour - private static long SESSION_MAX_SIZE = 100; // avoid to store too many - - private static HttpAuthManager instance = new HttpAuthManager(); - - public static class SessionValue { - public UserIdentity currentUser; - } - - // session_id => session value - private Cache authSessions = CacheBuilder.newBuilder() - .maximumSize(SESSION_MAX_SIZE) - .expireAfterAccess(SESSION_EXPIRE_TIME, TimeUnit.HOURS) - .build(); - - private HttpAuthManager() { - // do nothing - } - - public static HttpAuthManager getInstance() { - return instance; - } - - public SessionValue getSessionValue(String sessionId) { - return authSessions.getIfPresent(sessionId); - } - - public void addSessionValue(String key, SessionValue value) { - authSessions.put(key, value); - } - - public Cache getAuthSessions() { - return authSessions; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/HttpServer.java b/fe/fe-core/src/main/java/org/apache/doris/http/HttpServer.java deleted file mode 100644 index 1b1df9cf2601c1..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/HttpServer.java +++ /dev/null @@ -1,279 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - -import org.apache.doris.common.Config; -import org.apache.doris.http.action.BackendAction; -import org.apache.doris.http.action.HaAction; -import org.apache.doris.http.action.HelpAction; -import org.apache.doris.http.action.IndexAction; -import org.apache.doris.http.action.LogAction; -import org.apache.doris.http.action.QueryAction; -import org.apache.doris.http.action.QueryProfileAction; -import org.apache.doris.http.action.SessionAction; -import org.apache.doris.http.action.StaticResourceAction; -import org.apache.doris.http.action.SystemAction; -import org.apache.doris.http.action.VariableAction; -import org.apache.doris.http.common.DorisHttpPostObjectAggregator; -import org.apache.doris.http.meta.ColocateMetaService; -import org.apache.doris.http.meta.MetaService.CheckAction; -import org.apache.doris.http.meta.MetaService.DumpAction; -import org.apache.doris.http.meta.MetaService.ImageAction; -import org.apache.doris.http.meta.MetaService.InfoAction; -import org.apache.doris.http.meta.MetaService.JournalIdAction; -import org.apache.doris.http.meta.MetaService.PutAction; -import org.apache.doris.http.meta.MetaService.RoleAction; -import org.apache.doris.http.meta.MetaService.VersionAction; -import org.apache.doris.http.rest.BootstrapFinishAction; -import org.apache.doris.http.rest.CancelStreamLoad; -import org.apache.doris.http.rest.CheckDecommissionAction; -import org.apache.doris.http.rest.ConnectionAction; -import org.apache.doris.http.rest.GetDdlStmtAction; -import org.apache.doris.http.rest.GetLoadInfoAction; -import org.apache.doris.http.rest.GetLogFileAction; -import org.apache.doris.http.rest.GetSmallFileAction; -import org.apache.doris.http.rest.GetStreamLoadState; -import org.apache.doris.http.rest.HealthAction; -import org.apache.doris.http.rest.LoadAction; -import org.apache.doris.http.rest.MetaReplayerCheckAction; -import org.apache.doris.http.rest.MetricsAction; -import org.apache.doris.http.rest.MigrationAction; -import org.apache.doris.http.rest.MultiAbort; -import org.apache.doris.http.rest.MultiCommit; -import org.apache.doris.http.rest.MultiDesc; -import org.apache.doris.http.rest.MultiList; -import org.apache.doris.http.rest.MultiStart; -import org.apache.doris.http.rest.MultiUnload; -import org.apache.doris.http.rest.ProfileAction; -import org.apache.doris.http.rest.QueryDetailAction; -import org.apache.doris.http.rest.RowCountAction; -import org.apache.doris.http.rest.SetConfigAction; -import org.apache.doris.http.rest.ShowDataAction; -import org.apache.doris.http.rest.ShowMetaInfoAction; -import org.apache.doris.http.rest.ShowProcAction; -import org.apache.doris.http.rest.ShowRuntimeInfoAction; -import org.apache.doris.http.rest.StorageTypeCheckAction; -import org.apache.doris.http.rest.TableQueryPlanAction; -import org.apache.doris.http.rest.TableRowCountAction; -import org.apache.doris.http.rest.TableSchemaAction; -import org.apache.doris.master.MetaHelper; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.File; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.Channel; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.handler.codec.http.HttpServerCodec; -import io.netty.handler.stream.ChunkedWriteHandler; - -public class HttpServer { - private static final Logger LOG = LogManager.getLogger(HttpServer.class); - - /** - * The default netty param, witch is the same as `HttpServerCodec`. - */ - public static final int DEFAULT_MAX_LINE_LENGTH = 4096; - public static final int DEFAULT_MAX_HEADER_SIZE = 8192; - public static final int DEFAULT_MAX_CHUNK_SIZE = 8192; - - private final int port; - private final int maxInitialLineLength; - private final int maxHeaderSize; - private final int maxChunkSize; - private ActionController controller; - - private Thread serverThread; - - private AtomicBoolean isStarted = new AtomicBoolean(false); - - public HttpServer(int port) { - this(port, DEFAULT_MAX_LINE_LENGTH, DEFAULT_MAX_HEADER_SIZE, DEFAULT_MAX_CHUNK_SIZE); - } - - public HttpServer(int port, int maxInitialLineLength, int maxHeaderSize, int maxChunkSize) { - this.port = port; - this.maxInitialLineLength = maxInitialLineLength; - this.maxHeaderSize = maxHeaderSize; - this.maxChunkSize = maxChunkSize; - controller = new ActionController(); - } - - public void setup() throws IllegalArgException { - registerActions(); - } - - private void registerActions() throws IllegalArgException { - // add rest action - LoadAction.registerAction(controller); - GetLoadInfoAction.registerAction(controller); - SetConfigAction.registerAction(controller); - GetDdlStmtAction.registerAction(controller); - MigrationAction.registerAction(controller); - StorageTypeCheckAction.registerAction(controller); - CancelStreamLoad.registerAction(controller); - GetStreamLoadState.registerAction(controller); - - // add web action - IndexAction.registerAction(controller); - SystemAction.registerAction(controller); - BackendAction.registerAction(controller); - LogAction.registerAction(controller); - QueryAction.registerAction(controller); - QueryProfileAction.registerAction(controller); - SessionAction.registerAction(controller); - VariableAction.registerAction(controller); - HelpAction.registerAction(controller); - StaticResourceAction.registerAction(controller); - HaAction.registerAction(controller); - - // Add multi action - MultiStart.registerAction(controller); - MultiDesc.registerAction(controller); - MultiCommit.registerAction(controller); - MultiUnload.registerAction(controller); - MultiAbort.registerAction(controller); - MultiList.registerAction(controller); - - // rest action - HealthAction.registerAction(controller); - MetricsAction.registerAction(controller); - ShowMetaInfoAction.registerAction(controller); - ShowProcAction.registerAction(controller); - ShowRuntimeInfoAction.registerAction(controller); - GetLogFileAction.registerAction(controller); - GetSmallFileAction.registerAction(controller); - RowCountAction.registerAction(controller); - CheckDecommissionAction.registerAction(controller); - MetaReplayerCheckAction.registerAction(controller); - ColocateMetaService.BucketSeqAction.registerAction(controller); - ColocateMetaService.ColocateMetaAction.registerAction(controller); - ColocateMetaService.MarkGroupStableAction.registerAction(controller); - ProfileAction.registerAction(controller); - QueryDetailAction.registerAction(controller); - ConnectionAction.registerAction(controller); - ShowDataAction.registerAction(controller); - - // meta service action - File imageDir = MetaHelper.getMasterImageDir(); - ImageAction.registerAction(controller, imageDir); - InfoAction.registerAction(controller, imageDir); - VersionAction.registerAction(controller, imageDir); - PutAction.registerAction(controller, imageDir); - JournalIdAction.registerAction(controller, imageDir); - CheckAction.registerAction(controller, imageDir); - DumpAction.registerAction(controller, imageDir); - RoleAction.registerAction(controller, imageDir); - - // external usage - TableRowCountAction.registerAction(controller); - TableSchemaAction.registerAction(controller); - TableQueryPlanAction.registerAction(controller); - - BootstrapFinishAction.registerAction(controller); - } - - public void start() { - serverThread = new Thread(new HttpServerThread(), "FE Http Server"); - serverThread.start(); - } - - protected class PaloHttpServerInitializer extends ChannelInitializer { - @Override - protected void initChannel(SocketChannel ch) throws Exception { - ch.pipeline().addLast(new HttpServerCodec(maxInitialLineLength, maxHeaderSize, maxChunkSize)); - ch.pipeline().addLast(new DorisHttpPostObjectAggregator(100 * 65536)); - ch.pipeline().addLast(new ChunkedWriteHandler()); - ch.pipeline().addLast(new HttpServerHandler(controller)); - } - } - - ServerBootstrap serverBootstrap; - - private class HttpServerThread implements Runnable { - @Override - public void run() { - // Configure the server. - EventLoopGroup bossGroup = new NioEventLoopGroup(); - EventLoopGroup workerGroup = new NioEventLoopGroup(); - try { - serverBootstrap = new ServerBootstrap(); - serverBootstrap.option(ChannelOption.SO_BACKLOG, Config.http_backlog_num); - // reused address and port to avoid bind already exception - serverBootstrap.option(ChannelOption.SO_REUSEADDR, true); - serverBootstrap.childOption(ChannelOption.SO_REUSEADDR, true); - serverBootstrap.group(bossGroup, workerGroup) - .channel(NioServerSocketChannel.class) - .childHandler(new PaloHttpServerInitializer()); - Channel ch = serverBootstrap.bind(port).sync().channel(); - - isStarted.set(true); - LOG.info("HttpServer started with port {}", port); - // block until server is closed - ch.closeFuture().sync(); - } catch (Exception e) { - LOG.error("Fail to start FE query http server[port: " + port + "] ", e); - System.exit(-1); - } finally { - bossGroup.shutdownGracefully(); - workerGroup.shutdownGracefully(); - } - } - } - - // used for test, release bound port - public void shutDown() { - if (serverBootstrap != null) { - Future future = serverBootstrap.config().group().shutdownGracefully(0, 1, TimeUnit.SECONDS).syncUninterruptibly(); - try { - future.get(); - isStarted.set(false); - LOG.info("HttpServer was closed completely"); - } catch (Throwable e) { - LOG.warn("Exception happened when close HttpServer", e); - } - serverBootstrap = null; - } - } - - public boolean isStarted() { - return isStarted.get(); - } - - public static void main(String[] args) throws Exception { - HttpServer httpServer = new HttpServer(8080); - httpServer.setup(); - System.out.println("before start http server."); - httpServer.start(); - System.out.println("after start http server."); - - while (true) { - Thread.sleep(2000); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/HttpServerHandler.java b/fe/fe-core/src/main/java/org/apache/doris/http/HttpServerHandler.java deleted file mode 100644 index eecbe60a6e49cb..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/HttpServerHandler.java +++ /dev/null @@ -1,117 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - -import org.apache.doris.http.action.IndexAction; -import org.apache.doris.http.action.NotFoundAction; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.net.URISyntaxException; - -import io.netty.buffer.Unpooled; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.handler.codec.http.DefaultFullHttpResponse; -import io.netty.handler.codec.http.FullHttpRequest; -import io.netty.handler.codec.http.FullHttpResponse; -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.codec.http.HttpVersion; -import io.netty.util.ReferenceCountUtil; - -public class HttpServerHandler extends ChannelInboundHandlerAdapter { - private static final Logger LOG = LogManager.getLogger(HttpServerHandler.class); - - private ActionController controller = null; - protected FullHttpRequest fullRequest = null; - protected HttpRequest request = null; - private BaseAction action = null; - - public HttpServerHandler(ActionController controller) { - super(); - this.controller = controller; - } - - @Override - public void channelReadComplete(ChannelHandlerContext ctx) { - ctx.flush(); - } - - @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - if (msg instanceof HttpRequest) { - this.request = (HttpRequest) msg; - LOG.debug("request: url:[{}]", request.uri()); - if (!isRequestValid(ctx, request)) { - writeResponse(ctx, HttpResponseStatus.BAD_REQUEST, "this is a bad request."); - return; - } - BaseRequest req = new BaseRequest(ctx, request); - - action = getAction(req); - if (action != null) { - LOG.debug("action: {} ", action.getClass().getName()); - action.handleRequest(req); - } - } else { - ReferenceCountUtil.release(msg); - } - } - - private boolean isRequestValid(ChannelHandlerContext ctx, HttpRequest request) throws URISyntaxException { - return true; - } - - private void writeResponse(ChannelHandlerContext context , HttpResponseStatus status, String content) { - FullHttpResponse responseObj = new DefaultFullHttpResponse( - HttpVersion.HTTP_1_1, - status, - Unpooled.wrappedBuffer(content.getBytes())); - responseObj.headers().set(HttpHeaderNames.CONTENT_TYPE.toString(), "text/html"); - responseObj.headers().set(HttpHeaderNames.CONTENT_LENGTH.toString(), responseObj.content().readableBytes()); - context.writeAndFlush(responseObj).addListener(ChannelFutureListener.CLOSE); - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - LOG.warn("Handle http from " + ctx.channel().remoteAddress() + " caught exception", cause); - ctx.close(); - } - - private BaseAction getAction(BaseRequest request) { - String uri = request.getRequest().uri(); - // ignore this request, which is a default request from client's browser. - if (uri.endsWith("/favicon.ico")) { - return NotFoundAction.getNotFoundAction(); - } else if (uri.equals("/")) { - return new IndexAction(controller); - } - - // Map params = Maps.newHashMap(); - BaseAction action = (BaseAction) controller.getHandler(request); - if (action == null) { - action = NotFoundAction.getNotFoundAction(); - } - - return action; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/IAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/IAction.java deleted file mode 100644 index a89d0ee3d035fa..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/IAction.java +++ /dev/null @@ -1,25 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - - -public interface IAction { - - public void handleRequest(BaseRequest request) throws Exception; - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/UnauthorizedException.java b/fe/fe-core/src/main/java/org/apache/doris/http/UnauthorizedException.java deleted file mode 100644 index 4802498ce09b98..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/UnauthorizedException.java +++ /dev/null @@ -1,26 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - -import org.apache.doris.common.DdlException; - -public class UnauthorizedException extends DdlException { - public UnauthorizedException(String msg) { - super(msg); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/WebUtils.java b/fe/fe-core/src/main/java/org/apache/doris/http/WebUtils.java deleted file mode 100644 index c9efe86ac0195e..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/WebUtils.java +++ /dev/null @@ -1,230 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - -import org.apache.doris.common.path.PathTrie; - -import com.google.common.base.Charsets; - -import java.nio.charset.Charset; -import java.util.Map; - -public class WebUtils { - public static final String HTTP_CORS_ALLOW_ORIGIN_SETTING = "http.cors.allow-origin"; - - public static final PathTrie.Decoder REST_DECODER = new PathTrie.Decoder() { - @Override - public String decode(String value) { - return WebUtils.decodeComponent(value); - } - }; - - public static boolean isBrowser(String userAgent) { - if (userAgent == null) { - return false; - } - // chrome, safari, firefox, ie - if (userAgent.startsWith("Mozilla")) { - return true; - } - return false; - } - - public static void decodeQueryString(String s, int fromIndex, Map params) { - if (fromIndex < 0) { - return; - } - if (fromIndex >= s.length()) { - return; - } - - String name = null; - int pos = fromIndex; // Beginning of the unprocessed region - int i; // End of the unprocessed region - char c = 0; // Current character - for (i = fromIndex; i < s.length(); i++) { - c = s.charAt(i); - if (c == '=' && name == null) { - if (pos != i) { - name = decodeComponent(s.substring(pos, i)); - } - pos = i + 1; - } else if (c == '&') { - if (name == null && pos != i) { - // We haven't seen an `=' so far but moved forward. - // Must be a param of the form '&a&' so add it with - // an empty value. - addParam(params, decodeComponent(s.substring(pos, i)), ""); - } else if (name != null) { - addParam(params, name, decodeComponent(s.substring(pos, i))); - name = null; - } - pos = i + 1; - } - } - - if (pos != i) { // Are there characters we haven't dealt with? - if (name == null) { // Yes and we haven't seen any `='. - addParam(params, decodeComponent(s.substring(pos, i)), ""); - } else { // Yes and this must be the last value. - addParam(params, name, decodeComponent(s.substring(pos, i))); - } - } else if (name != null) { // Have we seen a name without value? - addParam(params, name, ""); - } - } - - private static void addParam(Map params, String name, String value) { - params.put(name, value); - } - - /** - * Decodes a bit of an URL encoded by a browser. - *

- * This is equivalent to calling {@link #decodeComponent(String, Charset)} - * with the UTF-8 charset (recommended to comply with RFC 3986, Section 2). - * - * @param s The string to decode (can be empty). - * @return The decoded string, or {@code s} if there's nothing to decode. - * If the string to decode is {@code null}, returns an empty string. - * @throws IllegalArgumentException if the string contains a malformed - * escape sequence. - */ - public static String decodeComponent(final String s) { - return decodeComponent(s, Charsets.UTF_8); - } - - /** - * Decodes a bit of an URL encoded by a browser. - *

- * The string is expected to be encoded as per RFC 3986, Section 2. - * This is the encoding used by JavaScript functions {@code encodeURI} - * and {@code encodeURIComponent}, but not {@code escape}. For example - * in this encoding, é (in Unicode {@code U+00E9} or in UTF-8 - * {@code 0xC3 0xA9}) is encoded as {@code %C3%A9} or {@code %c3%a9}. - *

- * This is essentially equivalent to calling - * {@link java.net.URLDecoder URLDecoder}.{@link - * java.net.URLDecoder#decode(String, String)} - * except that it's over 2x faster and generates less garbage for the GC. - * Actually this function doesn't allocate any memory if there's nothing - * to decode, the argument itself is returned. - * - * @param s The string to decode (can be empty). - * @param charset The charset to use to decode the string (should really - * be {@link Charsets#UTF_8}. - * @return The decoded string, or {@code s} if there's nothing to decode. - * If the string to decode is {@code null}, returns an empty string. - * @throws IllegalArgumentException if the string contains a malformed - * escape sequence. - */ - @SuppressWarnings("fallthrough") - public static String decodeComponent(final String s, final Charset charset) { - if (s == null) { - return ""; - } - final int size = s.length(); - boolean modified = false; - for (int i = 0; i < size; i++) { - final char c = s.charAt(i); - switch (c) { - case '%': - i++; // We can skip at least one char, e.g. `%%'. - // Fall through. - case '+': - modified = true; - break; - } - } - if (!modified) { - return s; - } - final byte[] buf = new byte[size]; - int pos = 0; // position in `buf'. - for (int i = 0; i < size; i++) { - char c = s.charAt(i); - switch (c) { - case '+': - buf[pos++] = ' '; // "+" -> " " - break; - case '%': - if (i == size - 1) { - throw new IllegalArgumentException("unterminated escape" - + " sequence at end of string: " + s); - } - c = s.charAt(++i); - if (c == '%') { - buf[pos++] = '%'; // "%%" -> "%" - break; - } else if (i == size - 1) { - throw new IllegalArgumentException("partial escape" - + " sequence at end of string: " + s); - } - c = decodeHexNibble(c); - final char c2 = decodeHexNibble(s.charAt(++i)); - if (c == Character.MAX_VALUE || c2 == Character.MAX_VALUE) { - throw new IllegalArgumentException( - "invalid escape sequence `%" + s.charAt(i - 1) - + s.charAt(i) + "' at index " + (i - 2) - + " of: " + s); - } - c = (char) (c * 16 + c2); - // Fall through. - default: - buf[pos++] = (byte) c; - break; - } - } - return new String(buf, 0, pos, charset); - } - - /** - * Helper to decode half of a hexadecimal number from a string. - * - * @param c The ASCII character of the hexadecimal number to decode. - * Must be in the range {@code [0-9a-fA-F]}. - * @return The hexadecimal value represented in the ASCII character - * given, or {@link Character#MAX_VALUE} if the character is invalid. - */ - private static char decodeHexNibble(final char c) { - if ('0' <= c && c <= '9') { - return (char) (c - '0'); - } else if ('a' <= c && c <= 'f') { - return (char) (c - 'a' + 10); - } else if ('A' <= c && c <= 'F') { - return (char) (c - 'A' + 10); - } else { - return Character.MAX_VALUE; - } - } - - /** - * Determine if CORS setting is a regex - */ -// public static Pattern getCorsSettingRegex(Settings settings) { -// String corsSetting = settings.get(HTTP_CORS_ALLOW_ORIGIN_SETTING, "*"); -// int len = corsSetting.length(); -// boolean isRegex = len > 2 && corsSetting.startsWith("/") && corsSetting.endsWith("/"); -// -// if (isRegex) { -// return Pattern.compile(corsSetting.substring(1, corsSetting.length()-1)); -// } -// -// return null; -// } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/BackendAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/BackendAction.java deleted file mode 100644 index 73b44131189e49..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/BackendAction.java +++ /dev/null @@ -1,101 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.util.ListComparator; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.system.Backend; -import com.google.common.collect.ImmutableMap; - -import io.netty.handler.codec.http.HttpMethod; - -public class BackendAction extends WebBaseAction { - private static final Logger LOG = LogManager.getLogger(BackendAction.class); - - public BackendAction(ActionController controller) { - super(controller); - // TODO Auto-generated constructor stub - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/backend", new BackendAction(controller)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - - appendKnownBackendsInfo(response.getContent()); - - getPageFooter(response.getContent()); - writeResponse(request, response); - } - - private void appendKnownBackendsInfo(StringBuilder buffer) { - ImmutableMap backendMap = Catalog.getCurrentSystemInfo().getIdToBackend(); - - List> backendInfos = new ArrayList>(); - for (Backend backend : backendMap.values()) { - List backendInfo = new ArrayList(); - InetAddress address = null; - try { - address = InetAddress.getByName(backend.getHost()); - } catch (UnknownHostException e) { - LOG.warn("unknown host: " + backend.getHost(), e); - continue; - } - backendInfo.add(address.getHostName()); - backendInfo.add(backend.getId()); - backendInfo.add("heart_port: " + backend.getHeartbeatPort() - + ", be_port: " + backend.getBePort() - + ", http_port: " + backend.getHttpPort()); - - backendInfos.add(backendInfo); - } - - // sort by id - ListComparator> comparator = new ListComparator>(1); - Collections.sort(backendInfos, comparator); - - // set result - buffer.append("

Known Backends(" + backendMap.size() + ")

"); - - buffer.append("
");
-        for (List info : backendInfos) {
-            buffer.append(info.get(0));
-            buffer.append(" [id: " + info.get(1));
-            buffer.append(", " + info.get(2) + "]");
-            buffer.append(System.getProperty("line.separator"));
-        }
-        buffer.append("
"); - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/HaAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/HaAction.java deleted file mode 100644 index aa759b1da52d34..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/HaAction.java +++ /dev/null @@ -1,174 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.Config; -import org.apache.doris.ha.HAProtocol; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.persist.Storage; -import org.apache.doris.system.Frontend; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Date; -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; - -public class HaAction extends WebBaseAction { - - public HaAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/ha", new HaAction(controller)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - - appendRoleInfo(response.getContent()); - appendJournalInfo(response.getContent()); - appendCanReadInfo(response.getContent()); - appendNodesInfo(response.getContent()); - appendImageInfo(response.getContent()); - appendDbNames(response.getContent()); - appendFe(response.getContent()); - appendRemovedFe(response.getContent()); - - getPageFooter(response.getContent()); - writeResponse(request, response); - } - - private void appendRoleInfo(StringBuilder buffer) { - buffer.append("

Frontend Role

"); - buffer.append("
");
-        buffer.append("

" + Catalog.getCurrentCatalog().getFeType() + "

"); - buffer.append("
"); - } - - private void appendJournalInfo(StringBuilder buffer) { - buffer.append("

Current Journal Id

"); - buffer.append("
");
-        if (Catalog.getCurrentCatalog().isMaster()) {
-            buffer.append("

" + Catalog.getCurrentCatalog().getEditLog().getMaxJournalId() + "

"); - } else { - buffer.append("

" + Catalog.getCurrentCatalog().getReplayedJournalId() + "

"); - } - buffer.append("
"); - } - - private void appendNodesInfo(StringBuilder buffer) { - HAProtocol haProtocol = Catalog.getCurrentCatalog().getHaProtocol(); - if (haProtocol == null) { - return; - } - List electableNodes = haProtocol.getElectableNodes(true); - if (electableNodes.isEmpty()) { - return; - } - buffer.append("

Electable nodes

"); - buffer.append("
");
-        for (InetSocketAddress node : electableNodes) {
-            buffer.append("

" + node.getAddress() + "

"); - - } - buffer.append("
"); - - List observerNodes = haProtocol.getObserverNodes(); - if (observerNodes == null) { - return; - } - buffer.append("

Observer nodes

"); - buffer.append("
");
-        for (InetSocketAddress node : observerNodes) {
-            buffer.append("

" + node.getHostString() + "

"); - } - buffer.append("
"); - } - - private void appendCanReadInfo(StringBuilder buffer) { - buffer.append("

Can Read

"); - buffer.append("
");
-        buffer.append("

" + Catalog.getCurrentCatalog().canRead() + "

"); - - buffer.append("
"); - } - - private void appendImageInfo(StringBuilder buffer) { - try { - Storage storage = new Storage(Config.meta_dir + "/image"); - buffer.append("

Checkpoint Info

"); - buffer.append("
");
-            buffer.append("

last checkpoint version:" + storage.getImageSeq() + "

"); - long lastCheckpointTime = storage.getCurrentImageFile().lastModified(); - Date date = new Date(lastCheckpointTime); - buffer.append("

last checkpoint time: " + date + "

"); - buffer.append("
"); - } catch (IOException e) { - e.printStackTrace(); - } - } - - private void appendDbNames(StringBuilder buffer) { - List names = Catalog.getCurrentCatalog().getEditLog().getDatabaseNames(); - if (names == null) { - return; - } - - String msg = ""; - for (long name : names) { - msg += name + " "; - } - buffer.append("

Database names

"); - buffer.append("
");
-        buffer.append("

" + msg + "

"); - buffer.append("
"); - } - - private void appendFe(StringBuilder buffer) { - List fes = Catalog.getCurrentCatalog().getFrontends(null /* all */); - if (fes == null) { - return; - } - - buffer.append("

Allowed Frontends

"); - buffer.append("
");
-        for (Frontend fe : fes) {
-            buffer.append("

" + fe.toString() + "

"); - } - buffer.append("
"); - } - - private void appendRemovedFe(StringBuilder buffer) { - List feNames = Catalog.getCurrentCatalog().getRemovedFrontendNames(); - buffer.append("

Removed Frontends

"); - buffer.append("
");
-        for (String feName : feNames) {
-            buffer.append("

" + feName + "

"); - } - buffer.append("
"); - } - -} \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/HelpAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/HelpAction.java deleted file mode 100644 index 786426250b9811..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/HelpAction.java +++ /dev/null @@ -1,223 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.qe.HelpModule; -import org.apache.doris.qe.HelpTopic; - -import com.google.common.base.Strings; - -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; - -public class HelpAction extends WebBaseAction { - private static final String DIV_BACKGROUND_COLOR = "#FCFCFC"; - - String queryString = null; - - public HelpAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/help", new HelpAction(controller)); - } - - @Override - public boolean needAdmin() { - return false; - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - appendHelpStyle(response.getContent()); - - queryString = request.getSingleParameter("query"); - if (Strings.isNullOrEmpty(queryString)) { - // ATTN: according to Mysql protocol, the default query should be "contents" - // when you want to get server side help. - queryString = "contents"; - } else { - queryString = queryString.trim(); - } - appendHelpInfo(response.getContent()); - - getPageFooter(response.getContent()); - writeResponse(request, response); - } - - private void appendHelpInfo(StringBuilder buffer) { - buffer.append("

Help Info

"); - buffer.append("

This page lists the help info, " - + "like 'help contents' in Mysql client.

"); - - appendSearchButton(buffer); - appendExactMatchTopic(buffer); - appendFuzzyMatchTopic(buffer); - appendCategories(buffer); - } - - private void appendSearchButton(StringBuilder buffer) { - buffer.append("
" - + "
" - + "
" - + " " - + " " - + " " - + " " - + "
" - + "
" - + "Back To Home" - + "
"); - } - - private void appendExactMatchTopic(StringBuilder buffer) { - buffer.append("

Exact Matching Topic

"); - buffer.append("
"); - HelpModule module = HelpModule.getInstance(); - HelpTopic topic = module.getTopic(queryString); - if (topic == null) { - buffer.append("
No Exact Matching Topic.
"); - } else { - appendOneTopicInfo(buffer, topic); - } - buffer.append("
"); - } - - private void appendFuzzyMatchTopic(StringBuilder buffer) { - buffer.append("

Fuzzy Matching Topic(By Keyword)

"); - buffer.append("
"); - HelpModule module = HelpModule.getInstance(); - List topics = module.listTopicByKeyword(queryString); - if (topics.isEmpty()) { - buffer.append("
No Fuzzy Matching Topic.
"); - } else if (topics.size() == 1) { - buffer.append("

" - + "Find only one topic, show you the detail info below.

"); - appendOneTopicInfo(buffer, module.getTopic(topics.get(0))); - } else { - buffer.append("

Find " + topics.size() + " topics:

"); - appendNameList(buffer, topics, "Topics"); - } - buffer.append("
"); - } - - private void appendCategories(StringBuilder buffer) { - buffer.append("

Category Info

"); - buffer.append("
"); - HelpModule module = HelpModule.getInstance(); - List categories = module.listCategoryByName(queryString); - if (categories.isEmpty()) { - buffer.append("
No Matching Category.
"); - } else if (categories.size() == 1) { - buffer.append("

" - + "Find only one category, so show you the detail info below.

"); - List topics = module.listTopicByCategory(categories.get(0)); - if (topics.size() > 0) { - buffer.append("

Find " - + topics.size() - + " sub topics.

"); - appendNameList(buffer, topics, "Sub Topics"); - } - - List subCategories = module.listCategoryByCategory(categories.get(0)); - if (subCategories.size() > 0) { - buffer.append("

Find " - + subCategories.size() - + " sub categories.

"); - appendNameList(buffer, subCategories, "Sub Categories"); - } - } else { - buffer.append("

Find " + categories.size() + " category:

"); - appendNameList(buffer, categories, "Categories"); - } - buffer.append("
"); - } - - // The browser will combine continuous whitespace to one, we use
 tag to solve this issue.
-    private void appendOneTopicInfo(StringBuilder buffer, HelpTopic topic) {
-        buffer.append("
"); - buffer.append("

'" + escapeHtmlInPreTag(topic.getName()) + "'

"); - - buffer.append("Description"); - buffer.append("
" 
-                + escapeHtmlInPreTag(topic.getDescription())
-                + "
"); - - buffer.append("Example"); - buffer.append("
" 
-                + escapeHtmlInPreTag(topic.getExample()) 
-                + "
"); - - buffer.append("Keyword"); - buffer.append("
" 
-                + escapeHtmlInPreTag(topic.getKeywords().toString()) 
-                + "
"); - - buffer.append("Url"); - buffer.append("
" 
-                + escapeHtmlInPreTag(topic.getUrl()) 
-                + "
"); - buffer.append("
"); - } - - private void appendNameList(StringBuilder buffer, List names, String tableHeadName) { - buffer.append("
"); - buffer.append(""); - buffer.append(""); - - final String href = "?query="; - for (String name : names) { - buffer.append(""); - } - buffer.append("
" + tableHeadName + "
" + name + "
"); - buffer.append("
"); - } - - private void appendHelpStyle(StringBuilder buffer) { - buffer.append(""); - } -} - diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/IndexAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/IndexAction.java deleted file mode 100644 index af17b29db5f787..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/IndexAction.java +++ /dev/null @@ -1,275 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.common.Version; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; -import oshi.SystemInfo; -import oshi.hardware.CentralProcessor; -import oshi.hardware.GlobalMemory; -import oshi.hardware.HWDiskStore; -import oshi.hardware.HWPartition; -import oshi.hardware.HardwareAbstractionLayer; -import oshi.hardware.NetworkIF; -import oshi.hardware.VirtualMemory; -import oshi.software.os.FileSystem; -import oshi.software.os.NetworkParams; -import oshi.software.os.OSFileStore; -import oshi.software.os.OSProcess; -import oshi.software.os.OperatingSystem; -import oshi.util.FormatUtil; -import oshi.util.Util; - -public class IndexAction extends WebBaseAction { - - public IndexAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/index", new IndexAction(controller)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - appendVersionInfo(response.getContent()); - appendHardwareInfo(response.getContent()); - getPageFooter(response.getContent()); - writeResponse(request, response); - } - - private void appendVersionInfo(StringBuilder buffer) { - buffer.append("

Version

"); - buffer.append("
version info
"); - buffer.append("Version: " + Version.DORIS_BUILD_VERSION + "
"); - buffer.append("Git: " + Version.DORIS_BUILD_HASH + "
"); - buffer.append("Build Info: " + Version.DORIS_BUILD_INFO + "
"); - buffer.append("Build Time: " + Version.DORIS_BUILD_TIME + "
"); - buffer.append("
"); - } - - private void appendHardwareInfo(StringBuilder buffer) { - List hardwareInfo = new ArrayList<>(); - SystemInfo si = new SystemInfo(); - OperatingSystem os = si.getOperatingSystem(); - HardwareAbstractionLayer hal = si.getHardware(); - CentralProcessor processor = hal.getProcessor(); - GlobalMemory memory = hal.getMemory(); - buffer.append("

Hardware Info

"); - buffer.append("
");
-        buffer.append(String.join("
", getOperatingSystem(os)) + "
"); - buffer.append(String.join("
", getProcessor(processor)) + "
"); - buffer.append(String.join("
", getMemory(memory)) + "
"); - buffer.append(String.join("
", getProcesses(os, memory)) + "
"); - buffer.append(String.join("
", getDisks(hal.getDiskStores())) + "
"); - buffer.append(String.join("
", getFileSystem(os.getFileSystem())) + "
"); - buffer.append(String.join("
", getNetworkInterfaces(hal.getNetworkIFs())) + "
"); - buffer.append(String.join("
", getNetworkParameters(os.getNetworkParams())) + "
"); - buffer.append("
"); - } - private List getOperatingSystem(OperatingSystem os) { - List osInfo = new ArrayList<>(); - osInfo.add(String.valueOf(os)); - osInfo.add("Booted: " + Instant.ofEpochSecond(os.getSystemBootTime())); - osInfo.add("Uptime: " + FormatUtil.formatElapsedSecs(os.getSystemUptime())); - osInfo.add("Running with" + (os.isElevated() ? "" : "out") + " elevated permissions."); - return osInfo; - } - private List getProcessor(CentralProcessor processor) { - List processorInfo = new ArrayList<>(); - processorInfo.add(String.valueOf(processor)); - processorInfo.add(" " + processor.getPhysicalPackageCount() + " physical CPU package(s)"); - processorInfo.add(" " + processor.getPhysicalProcessorCount() + " physical CPU core(s)"); - processorInfo.add(" " + processor.getLogicalProcessorCount() + " logical CPU(s)"); - - processorInfo.add("Identifier: " + processor.getIdentifier()); - processorInfo.add("ProcessorID: " + processor.getProcessorID()); - processorInfo.add("Context Switches/Interrupts: " + processor.getContextSwitches() - + " / " + processor.getInterrupts() + "\n"); - - long[] prevTicks = processor.getSystemCpuLoadTicks(); - long[][] prevProcTicks = processor.getProcessorCpuLoadTicks(); - processorInfo.add("CPU, IOWait, and IRQ ticks @ 0 sec:" + Arrays.toString(prevTicks)); - // Wait a second... - Util.sleep(1000); - long[] ticks = processor.getSystemCpuLoadTicks(); - processorInfo.add("CPU, IOWait, and IRQ ticks @ 1 sec:" + Arrays.toString(ticks)); - long user = ticks[CentralProcessor.TickType.USER.getIndex()] - prevTicks[CentralProcessor.TickType.USER.getIndex()]; - long nice = ticks[CentralProcessor.TickType.NICE.getIndex()] - prevTicks[CentralProcessor.TickType.NICE.getIndex()]; - long sys = ticks[CentralProcessor.TickType.SYSTEM.getIndex()] - prevTicks[CentralProcessor.TickType.SYSTEM.getIndex()]; - long idle = ticks[CentralProcessor.TickType.IDLE.getIndex()] - prevTicks[CentralProcessor.TickType.IDLE.getIndex()]; - long iowait = ticks[CentralProcessor.TickType.IOWAIT.getIndex()] - prevTicks[CentralProcessor.TickType.IOWAIT.getIndex()]; - long irq = ticks[CentralProcessor.TickType.IRQ.getIndex()] - prevTicks[CentralProcessor.TickType.IRQ.getIndex()]; - long softirq = ticks[CentralProcessor.TickType.SOFTIRQ.getIndex()] - prevTicks[CentralProcessor.TickType.SOFTIRQ.getIndex()]; - long steal = ticks[CentralProcessor.TickType.STEAL.getIndex()] - prevTicks[CentralProcessor.TickType.STEAL.getIndex()]; - long totalCpu = user + nice + sys + idle + iowait + irq + softirq + steal; - - processorInfo.add(String.format( - "User: %.1f%% Nice: %.1f%% System: %.1f%% Idle: %.1f%% IOwait: %.1f%% IRQ: %.1f%% SoftIRQ: %.1f%% Steal: %.1f%%", - 100d * user / totalCpu, 100d * nice / totalCpu, 100d * sys / totalCpu, 100d * idle / totalCpu, - 100d * iowait / totalCpu, 100d * irq / totalCpu, 100d * softirq / totalCpu, 100d * steal / totalCpu)); - processorInfo.add(String.format("CPU load: %.1f%%", - processor.getSystemCpuLoadBetweenTicks(prevTicks) * 100)); - double[] loadAverage = processor.getSystemLoadAverage(3); - processorInfo.add("CPU load averages:" + (loadAverage[0] < 0 ? " N/A" : String.format(" %.2f", loadAverage[0])) - + (loadAverage[1] < 0 ? " N/A" : String.format(" %.2f", loadAverage[1])) - + (loadAverage[2] < 0 ? " N/A" : String.format(" %.2f", loadAverage[2]))); - // per core CPU - StringBuilder procCpu = new StringBuilder("CPU load per processor:"); - double[] load = processor.getProcessorCpuLoadBetweenTicks(prevProcTicks); - for (double avg : load) { - procCpu.append(String.format(" %.1f%%", avg * 100)); - } - processorInfo.add(procCpu.toString()); - long freq = processor.getVendorFreq(); - if (freq > 0) { - processorInfo.add("Vendor Frequency: " + FormatUtil.formatHertz(freq)); - } - freq = processor.getMaxFreq(); - if (freq > 0) { - processorInfo.add("Max Frequency: " + FormatUtil.formatHertz(freq)); - } - long[] freqs = processor.getCurrentFreq(); - if (freqs[0] > 0) { - StringBuilder sb = new StringBuilder("Current Frequencies: "); - for (int i = 0; i < freqs.length; i++) { - if (i > 0) { - sb.append(", "); - } - sb.append(FormatUtil.formatHertz(freqs[i])); - } - processorInfo.add(sb.toString()); - } - return processorInfo; - } - - private List getMemory(GlobalMemory memory) { - List memoryInfo = new ArrayList<>(); - memoryInfo.add("Memory: " + FormatUtil.formatBytes(memory.getAvailable()) + "/" - + FormatUtil.formatBytes(memory.getTotal())); - VirtualMemory vm = memory.getVirtualMemory(); - memoryInfo.add("Swap used: " + FormatUtil.formatBytes(vm.getSwapUsed()) + "/" - + FormatUtil.formatBytes(vm.getSwapTotal())); - return memoryInfo; - } - - private List getProcesses(OperatingSystem os, GlobalMemory memory) { - List processInfo = new ArrayList<>(); - processInfo.add("Processes: " + os.getProcessCount() + ", Threads: " + os.getThreadCount()); - // Sort by highest CPU - List procs = Arrays.asList(os.getProcesses(5, OperatingSystem.ProcessSort.CPU)); - - processInfo.add(" PID %CPU %MEM VSZ RSS Name"); - for (int i = 0; i < procs.size() && i < 5; i++) { - OSProcess p = procs.get(i); - processInfo.add(String.format(" %5d %5.1f %4.1f %9s %9s %s", p.getProcessID(), - 100d * (p.getKernelTime() + p.getUserTime()) / p.getUpTime(), - 100d * p.getResidentSetSize() / memory.getTotal(), FormatUtil.formatBytes(p.getVirtualSize()), - FormatUtil.formatBytes(p.getResidentSetSize()), p.getName())); - } - return processInfo; - } - private List getDisks(HWDiskStore[] diskStores) { - List diskInfo = new ArrayList<>(); - diskInfo.add("Disks:"); - for (HWDiskStore disk : diskStores) { - boolean readwrite = disk.getReads() > 0 || disk.getWrites() > 0; - diskInfo.add(String.format(" %s: (model: %s - S/N: %s) size: %s, reads: %s (%s), writes: %s (%s), xfer: %s ms", - disk.getName(), disk.getModel(), disk.getSerial(), - disk.getSize() > 0 ? FormatUtil.formatBytesDecimal(disk.getSize()) : "?", - readwrite ? disk.getReads() : "?", readwrite ? FormatUtil.formatBytes(disk.getReadBytes()) : "?", - readwrite ? disk.getWrites() : "?", readwrite ? FormatUtil.formatBytes(disk.getWriteBytes()) : "?", - readwrite ? disk.getTransferTime() : "?")); - HWPartition[] partitions = disk.getPartitions(); - for (HWPartition part : partitions) { - diskInfo.add(String.format(" |-- %s: %s (%s) Maj:Min=%d:%d, size: %s%s", part.getIdentification(), - part.getName(), part.getType(), part.getMajor(), part.getMinor(), - FormatUtil.formatBytesDecimal(part.getSize()), - part.getMountPoint().isEmpty() ? "" : " @ " + part.getMountPoint())); - } - } - return diskInfo; - } - - private List getFileSystem(FileSystem fileSystem) { - List fsInfo = new ArrayList<>(); - fsInfo.add("File System:"); - - fsInfo.add(String.format(" File Descriptors: %d/%d", fileSystem.getOpenFileDescriptors(), - fileSystem.getMaxFileDescriptors())); - - OSFileStore[] fsArray = fileSystem.getFileStores(); - for (OSFileStore fs : fsArray) { - long usable = fs.getUsableSpace(); - long total = fs.getTotalSpace(); - fsInfo.add(String.format(" %s (%s) [%s] %s of %s free (%.1f%%), %s of %s files free (%.1f%%) is %s " + - (fs.getLogicalVolume() != null && fs.getLogicalVolume().length() > 0 ? "[%s]" : "%s") + - " and is mounted at %s", - fs.getName(), fs.getDescription().isEmpty() ? "file system" : fs.getDescription(), fs.getType(), - FormatUtil.formatBytes(usable), FormatUtil.formatBytes(fs.getTotalSpace()), 100d * usable / total, - FormatUtil.formatValue(fs.getFreeInodes(), ""), FormatUtil.formatValue(fs.getTotalInodes(), ""), - 100d * fs.getFreeInodes() / fs.getTotalInodes(), fs.getVolume(), fs.getLogicalVolume(), - fs.getMount())); - } - return fsInfo; - } - - private List getNetworkInterfaces(NetworkIF[] networkIFs) { - List getNetwork = new ArrayList<>(); - getNetwork.add("Network interfaces:"); - for (NetworkIF net : networkIFs) { - getNetwork.add(String.format(" Name: %s (%s)", net.getName(), net.getDisplayName())); - getNetwork.add(String.format(" MAC Address: %s", net.getMacaddr())); - getNetwork.add(String.format(" MTU: %s, Speed: %s", net.getMTU(), FormatUtil.formatValue(net.getSpeed(), "bps"))); - getNetwork.add(String.format(" IPv4: %s", Arrays.toString(net.getIPv4addr()))); - getNetwork.add(String.format(" IPv6: %s", Arrays.toString(net.getIPv6addr()))); - boolean hasData = net.getBytesRecv() > 0 || net.getBytesSent() > 0 || net.getPacketsRecv() > 0 - || net.getPacketsSent() > 0; - getNetwork.add(String.format(" Traffic: received %s/%s%s; transmitted %s/%s%s", - hasData ? net.getPacketsRecv() + " packets" : "?", - hasData ? FormatUtil.formatBytes(net.getBytesRecv()) : "?", - hasData ? " (" + net.getInErrors() + " err)" : "", - hasData ? net.getPacketsSent() + " packets" : "?", - hasData ? FormatUtil.formatBytes(net.getBytesSent()) : "?", - hasData ? " (" + net.getOutErrors() + " err)" : "")); - } - return getNetwork; - } - - private List getNetworkParameters(NetworkParams networkParams) { - List networkParameterInfo = new ArrayList<>(); - networkParameterInfo.add("Network parameters:"); - networkParameterInfo.add(String.format(" Host name: %s", networkParams.getHostName())); - networkParameterInfo.add(String.format(" Domain name: %s", networkParams.getDomainName())); - networkParameterInfo.add(String.format(" DNS servers: %s", Arrays.toString(networkParams.getDnsServers()))); - networkParameterInfo.add(String.format(" IPv4 Gateway: %s", networkParams.getIpv4DefaultGateway())); - networkParameterInfo.add(String.format(" IPv6 Gateway: %s", networkParams.getIpv6DefaultGateway())); - return networkParameterInfo; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/LogAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/LogAction.java deleted file mode 100644 index 6185debfc7b108..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/LogAction.java +++ /dev/null @@ -1,168 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.common.Config; -import org.apache.doris.common.Log4jConfig; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import io.netty.handler.codec.http.HttpMethod; - -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.util.List; - -public class LogAction extends WebBaseAction { - private static final Logger LOG = LogManager.getLogger(LogAction.class); - private static long WEB_LOG_BYTES = 1024 * 1024; // 1MB - - private String addVerboseName; - private String delVerboseName; - - public LogAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/log", new LogAction(controller)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - - // get parameters - addVerboseName = request.getSingleParameter("add_verbose"); - delVerboseName = request.getSingleParameter("del_verbose"); - LOG.info("add verbose name: {}, del verbose name: {}", addVerboseName, delVerboseName); - - appendLogConf(response.getContent()); - appendLogInfo(response.getContent()); - - getPageFooter(response.getContent()); - writeResponse(request, response); - } - - private void appendLogConf(StringBuilder buffer) { - buffer.append("

Log Configuration

"); - try { - Log4jConfig.Tuple configs = Log4jConfig.updateLogging(null, null, null); - if (!Strings.isNullOrEmpty(addVerboseName)) { - addVerboseName = addVerboseName.trim(); - List verboseNames = Lists.newArrayList(configs.y); - if (!verboseNames.contains(addVerboseName)) { - verboseNames.add(addVerboseName); - configs = Log4jConfig.updateLogging(null, verboseNames.toArray(new String[verboseNames.size()]), - null); - } - } - if (!Strings.isNullOrEmpty(delVerboseName)) { - delVerboseName = delVerboseName.trim(); - List verboseNames = Lists.newArrayList(configs.y); - if (verboseNames.contains(delVerboseName)) { - verboseNames.remove(delVerboseName); - configs = Log4jConfig.updateLogging(null, verboseNames.toArray(new String[verboseNames.size()]), - null); - } - } - - buffer.append("Level: " + configs.x + "
"); - buffer.append("Verbose Names: " + StringUtils.join(configs.y, ",") + "
"); - buffer.append("Audit Names: " + StringUtils.join(configs.z, ",") + "
"); - appendUpdateVerboseButton(buffer, "add_verbose"); - appendUpdateVerboseButton(buffer, "del_verbose"); - } catch (IOException e) { - LOG.error(e); - e.printStackTrace(); - } - } - - private void appendUpdateVerboseButton(StringBuilder buffer, String type) { - String placeHolder = ""; - String buttonName = ""; - if (type.equals("add_verbose")) { - placeHolder = "new verbose name"; - buttonName = "Add"; - } else if (type.equals("del_verbose")) { - placeHolder = "del verbose name"; - buttonName = "Delete"; - } else { - return; - } - - buffer.append("
" - + "
" - + "
" - + " " - + " " - + " " - + " \n" - + "
\n" - + "
" - + "
"); - } - - private void appendLogInfo(StringBuilder buffer) { - buffer.append("

Log Contents

"); - - final String logPath = Config.sys_log_dir + "/fe.warn.log"; - buffer.append("Log path is: " + logPath + "
"); - - RandomAccessFile raf = null; - try { - raf = new RandomAccessFile(logPath, "r"); - long fileSize = raf.length(); - long startPos = fileSize < WEB_LOG_BYTES ? 0L : fileSize - WEB_LOG_BYTES; - long webContentLength = fileSize < WEB_LOG_BYTES ? fileSize : WEB_LOG_BYTES; - raf.seek(startPos); - buffer.append("

Showing last " + webContentLength + " bytes of log

"); - buffer.append("
");
-            String fileBuffer = null;
-            while ((fileBuffer = raf.readLine()) != null) {
-                buffer.append(fileBuffer).append("\n");
-            }
-            buffer.append("
"); - } catch (FileNotFoundException e) { - buffer.append("

Couldn't open log file: " - + logPath + "

"); - } catch (IOException e) { - buffer.append("

Failed to read log file: " - + logPath + "

"); - } finally { - try { - if (raf != null) { - raf.close(); - } - } catch (IOException e) { - LOG.warn("fail to close log file: " + logPath, e); - } - } - } -} - diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/NotFoundAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/NotFoundAction.java deleted file mode 100644 index 4cb6d28dd34a44..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/NotFoundAction.java +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -public class NotFoundAction extends WebBaseAction { - - public NotFoundAction(ActionController controller) { - super(controller); - } - - @Override - public boolean needAdmin() { - return false; - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - - response.appendContent("this is 404 page."); - - getPageFooter(response.getContent()); - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/404", new NotFoundAction(controller)); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/QueryAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/QueryAction.java deleted file mode 100644 index e3e4f6af1ac79a..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/QueryAction.java +++ /dev/null @@ -1,127 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.common.util.ProfileManager; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.List; - -public class QueryAction extends WebBaseAction { - private static final Logger LOG = LogManager.getLogger(QueryAction.class); - - public QueryAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/query", new QueryAction(controller)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - - addFinishedQueryInfo(response.getContent()); - - getPageFooter(response.getContent()); - - writeResponse(request, response); - } - - // Note: we do not show 'Query ID' column in web page - private void addFinishedQueryInfo(StringBuilder buffer) { - buffer.append("

Finished Queries

"); - buffer.append("

You need to set session variable by executing 'set is_report_success=true' before executing the SQL, to view the profile

"); - buffer.append("

This table lists the latest 100 queries

"); - - List> finishedQueries = ProfileManager.getInstance().getAllQueries(); - List columnHeaders = ProfileManager.PROFILE_HEADERS; - int queryIdIndex = 0; // the first column is 'Query ID' by default - for (int i = 0; i < columnHeaders.size(); ++i) { - if (columnHeaders.get(i).equals(ProfileManager.QUERY_ID)) { - queryIdIndex = i; - } - } - appendFinishedQueryTableHeader(buffer, columnHeaders, queryIdIndex); - appendFinishedQueryTableBody(buffer, finishedQueries, columnHeaders, queryIdIndex); - appendTableFooter(buffer); - } - - private void appendFinishedQueryTableHeader( - StringBuilder buffer, - final List columnHeaders, - int queryIdIndex) { - buffer.append(""); - for (int i = 0; i < columnHeaders.size(); ++i) { - if (i == queryIdIndex) { - continue; - } - buffer.append(""); - } - - buffer.append(""); - buffer.append(""); - } - - private void appendFinishedQueryTableBody( - StringBuilder buffer, - List> bodies, - List columnHeaders, - int queryIdIndex) { - for ( List row : bodies) { - buffer.append(""); - String queryId = row.get(queryIdIndex); - - for (int i = 0; i < row.size(); ++i) { - if (i == queryIdIndex) { - continue; - } - buffer.append(""); - } - - // add 'Profile' column - if (Strings.isNullOrEmpty(queryId)) { - LOG.warn("query id is null or empty, maybe we forget to push it " - + "into array when generate profile info."); - buffer.append(""); - } else { - buffer.append(""); - } - - buffer.append(""); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/QueryProfileAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/QueryProfileAction.java deleted file mode 100644 index 69ac168587e09d..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/QueryProfileAction.java +++ /dev/null @@ -1,68 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.common.util.ProfileManager; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -public class QueryProfileAction extends WebBaseAction { - - public QueryProfileAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/query_profile", new QueryProfileAction(controller)); - } - - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - - String queryId = request.getSingleParameter("query_id"); - if (Strings.isNullOrEmpty(queryId)) { - response.appendContent(""); - response.appendContent("

Must specify a query_id[]

"); - } - - String queryProfileStr = ProfileManager.getInstance().getProfile(queryId); - if (queryProfileStr != null) { - appendQueryProfile(response.getContent(), queryProfileStr); - getPageFooter(response.getContent()); - writeResponse(request, response); - } else { - appendQueryProfile(response.getContent(), "query id " + queryId + " not found."); - getPageFooter(response.getContent()); - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); - } - } - - private void appendQueryProfile(StringBuilder buffer, String queryProfileStr) { - buffer.append("
");
-        buffer.append(queryProfileStr);
-        buffer.append("
"); - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/SessionAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/SessionAction.java deleted file mode 100644 index 435cacfa481870..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/SessionAction.java +++ /dev/null @@ -1,85 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.service.ExecuteEnv; - -import com.google.common.collect.Lists; - -import java.util.ArrayList; -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; - -public class SessionAction extends WebBaseAction { - // we make - private static final ArrayList SESSION_TABLE_HEADER = Lists.newArrayList(); - - static { - SESSION_TABLE_HEADER.add("Id"); - SESSION_TABLE_HEADER.add("User"); - SESSION_TABLE_HEADER.add("Host"); - SESSION_TABLE_HEADER.add("Cluster"); - SESSION_TABLE_HEADER.add("Db"); - SESSION_TABLE_HEADER.add("Command"); - SESSION_TABLE_HEADER.add("Time"); - SESSION_TABLE_HEADER.add("State"); - SESSION_TABLE_HEADER.add("Info"); - } - - public SessionAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/session", new SessionAction(controller)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - appendSessionInfo(response.getContent()); - getPageFooter(response.getContent()); - writeResponse(request, response); - } - - private void appendSessionInfo(StringBuilder buffer) { - buffer.append("

Session Info

"); - - List threadInfos = ExecuteEnv.getInstance().getScheduler().listConnection("root"); - List> rowSet = Lists.newArrayList(); - long nowMs = System.currentTimeMillis(); - for (ConnectContext.ThreadInfo info : threadInfos) { - rowSet.add(info.toRow(nowMs)); - } - - buffer.append("

This page lists the session info, there are " - + rowSet.size() - + " active sessions.

"); - - appendTableHeader(buffer, SESSION_TABLE_HEADER); - appendTableBody(buffer, rowSet); - appendTableFooter(buffer); - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/StaticResourceAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/StaticResourceAction.java deleted file mode 100644 index cbd540d29a4827..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/StaticResourceAction.java +++ /dev/null @@ -1,242 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.PaloFe; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import com.google.common.base.Strings; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.File; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; -import java.util.GregorianCalendar; -import java.util.Locale; -import java.util.TimeZone; -import java.util.regex.Pattern; - -import javax.activation.MimetypesFileTypeMap; - -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -/** - * A simple handler that serves incoming HTTP requests to send their respective - * HTTP responses. It also implements {@code 'If-Modified-Since'} header to - * take advantage of browser cache, as described in - * RFC 2616. - * - *

How Browser Caching Works

- * - * Web browser caching works with HTTP headers as illustrated by the following - * sample: - *
    - *
  1. Request #1 returns the content of {@code /file1.txt}.
  2. - *
  3. Contents of {@code /file1.txt} is cached by the browser.
  4. - *
  5. Request #2 for {@code /file1.txt} does return the contents of the - * file again. Rather, a 304 Not Modified is returned. This tells the - * browser to use the contents stored in its cache.
  6. - *
  7. The server knows the file has not been modified because the - * {@code If-Modified-Since} date is the same as the file's last - * modified date.
  8. - *
- * - *
- * Request #1 Headers
- * ===================
- * GET /file1.txt HTTP/1.1
- *
- * Response #1 Headers
- * ===================
- * HTTP/1.1 200 OK
- * Date:               Tue, 01 Mar 2011 22:44:26 GMT
- * Last-Modified:      Wed, 30 Jun 2010 21:36:48 GMT
- * Expires:            Tue, 01 Mar 2012 22:44:26 GMT
- * Cache-Control:      private, max-age=31536000
- *
- * Request #2 Headers
- * ===================
- * GET /file1.txt HTTP/1.1
- * If-Modified-Since:  Wed, 30 Jun 2010 21:36:48 GMT
- *
- * Response #2 Headers
- * ===================
- * HTTP/1.1 304 Not Modified
- * Date:               Tue, 01 Mar 2011 22:44:28 GMT
- *
- * 
- * - * We use parameter named 'res' to specify the static resource path, it relative to the - * root path of http server. - */ -public class StaticResourceAction extends WebBaseAction { - private static final Logger LOG = LogManager.getLogger(StaticResourceAction.class); - - public static final String HTTP_DATE_FORMAT = "EEE, dd MMM yyyy HH:mm:ss zzz"; - public static final String HTTP_DATE_GMT_TIMEZONE = "GMT"; - public static final int HTTP_CACHE_SECONDS = 2592000; - - public MimetypesFileTypeMap mimeTypesMap; - public String rootDir; - - public StaticResourceAction(ActionController controller, String rootDir) { - super(controller); - mimeTypesMap = new MimetypesFileTypeMap(); - mimeTypesMap.addMimeTypes("text/html html htm"); - // According to RFC 4329(http://tools.ietf.org/html/rfc4329#section-7.2), the MIME type of - // javascript script is 'application/javascript' - mimeTypesMap.addMimeTypes("application/javascript js"); - // mimeTypesMap.addMimeTypes("text/javascript js"); - mimeTypesMap.addMimeTypes("text/css css"); - - this.rootDir = rootDir; - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - String httpDir = PaloFe.DORIS_HOME_DIR + "/webroot"; - StaticResourceAction action = new StaticResourceAction(controller, httpDir + "/static"); - controller.registerHandler(HttpMethod.GET, "/static/js", action); - controller.registerHandler(HttpMethod.GET, "/static/css", action); - controller.registerHandler(HttpMethod.GET, "/static", action); - controller.registerHandler(HttpMethod.GET, "/static/resource", action); - controller.registerHandler(HttpMethod.GET, "/static/images", action); - controller.registerHandler(HttpMethod.GET, "/static/Bootstrap-3.3.7/fonts/", action); - - StaticResourceAction action2 = new StaticResourceAction(controller, "webroot"); - controller.registerHandler(HttpMethod.GET, "/static_test", action2); - } - - @Override - public boolean needAdmin() { - return false; - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - String resourcePath = request.getSingleParameter("res"); - if (Strings.isNullOrEmpty(resourcePath)) { - LOG.error("Wrong request without 'res' parameter. url: {}", - request.getRequest().uri()); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - resourcePath = sanitizePath(resourcePath); - if (resourcePath == null) { - LOG.error("Close this request because of risk factor in 'res' parameter.url: {}", - request.getRequest().uri()); - writeResponse(request, response, HttpResponseStatus.FORBIDDEN); - return; - } - - String resourceAbsolutePath = rootDir + File.separator + resourcePath; - File resFile = new File(resourceAbsolutePath); - LOG.debug("resAbsolutePath: {}", resourceAbsolutePath); - - if (!resFile.exists() || resFile.isHidden() || resFile.isDirectory()) { - LOG.error("Request with wrong path. url: {}", request.getRequest().uri()); - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); - return; - } - if (!resFile.isFile()) { - LOG.error("Wrong request: not normal file. url: {}", request.getRequest().uri()); - writeResponse(request, response, HttpResponseStatus.FORBIDDEN); - return; - } - - // Cache validation - String ifModifiedSince = request.getRequest() - .headers().get(HttpHeaderNames.IF_MODIFIED_SINCE.toString()); - if (!Strings.isNullOrEmpty(ifModifiedSince)) { - SimpleDateFormat dateFormatter = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US); - Date ifModifiedSinceDate; - try { - ifModifiedSinceDate = dateFormatter.parse(ifModifiedSince); - // Only compare up to the second because the datetime format we send to the client - // does not have milliseconds - long ifModifiedSinceDateSeconds = ifModifiedSinceDate.getTime() / 1000; - long fileLastModifiedSeconds = resFile.lastModified() / 1000; - if (ifModifiedSinceDateSeconds == fileLastModifiedSeconds) { - writeResponse(request, response, HttpResponseStatus.NOT_MODIFIED); - return; - } - } catch (ParseException e) { - LOG.error("Fail to analyse IF_MODIFIED_SINCE header: ", e); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - } - - response.updateHeader(HttpHeaderNames.CONTENT_TYPE.toString(), getContentType(resourceAbsolutePath)); - setDateAndCacheHeaders(response, resFile); - - writeObjectResponse(request, response, HttpResponseStatus.OK, resFile, resFile.getName(), false); - } - - - - // Gets the content type header for the HTTP Response - private String getContentType(String filename) { - return mimeTypesMap.getContentType(filename); - } - - private void setDateAndCacheHeaders(BaseResponse response, File fileToCache) { - SimpleDateFormat dateFormatter = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US); - dateFormatter.setTimeZone(TimeZone.getTimeZone(HTTP_DATE_GMT_TIMEZONE)); - - // Date header - Calendar time = new GregorianCalendar(); - response.updateHeader(HttpHeaderNames.DATE.toString(), dateFormatter.format(time.getTime())); - - // Add cache headers - time.add(Calendar.SECOND, HTTP_CACHE_SECONDS); - response.updateHeader(HttpHeaderNames.EXPIRES.toString(), dateFormatter.format(time.getTime())); - response.updateHeader(HttpHeaderNames.CACHE_CONTROL.toString(), "private, max-age=" + HTTP_CACHE_SECONDS); - response.updateHeader(HttpHeaderNames.LAST_MODIFIED.toString(), - dateFormatter.format(new Date(fileToCache.lastModified()))); - } - - private static final Pattern INSECURE_URI = Pattern.compile(".*[<>&\"].*"); - - private String sanitizePath(String path) { - if (Strings.isNullOrEmpty(path)) { - return null; - } - // Convert file separators - String newPath = path.replace('/', File.separatorChar); - - // Simplistic dumb security check. - if (newPath.contains(File.separator + '.') - || newPath.contains('.' + File.separator) - || newPath.charAt(0) == '.' - || newPath.charAt(newPath.length() - 1) == '.' - || INSECURE_URI.matcher(newPath).matches()) { - return null; - } - - return newPath; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/SystemAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/SystemAction.java deleted file mode 100644 index 3c941d9c7f3cdc..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/SystemAction.java +++ /dev/null @@ -1,184 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import io.netty.handler.codec.http.HttpMethod; - -import org.apache.doris.analysis.RedirectStatus; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; -import org.apache.doris.common.proc.ProcDirInterface; -import org.apache.doris.common.proc.ProcNodeInterface; -import org.apache.doris.common.proc.ProcResult; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.MasterOpExecutor; -import org.apache.doris.qe.OriginStatement; -import org.apache.doris.qe.ShowResultSet; - -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; - -import org.apache.commons.validator.routines.UrlValidator; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.List; -import java.util.stream.Collectors; - -public class SystemAction extends WebBaseAction { - private static final Logger LOG = LogManager.getLogger(SystemAction.class); - - public SystemAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/system", new SystemAction(controller)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - - String currentPath = request.getSingleParameter("path"); - if (Strings.isNullOrEmpty(currentPath)) { - currentPath = "/"; - } - appendSystemInfo(response.getContent(), currentPath, currentPath); - - getPageFooter(response.getContent()); - writeResponse(request, response); - } - - private void appendSystemInfo(StringBuilder buffer, String procPath, String path) { - buffer.append("

System Info

"); - buffer.append("

This page lists the system info, like /proc in Linux.

"); - buffer.append("

Current path: " + path + "" - + "Parent Dir


"); - - ProcNodeInterface procNode = getProcNode(procPath); - if (procNode == null) { - buffer.append("

No such proc path[" + path + "]

"); - return; - } - boolean isDir = (procNode instanceof ProcDirInterface); - - List columnNames = null; - List> rows = null; - if (!Catalog.getCurrentCatalog().isMaster() && !Config.enable_bdbje_debug_mode) { - // forward to master - String showProcStmt = "SHOW PROC \"" + procPath + "\""; - MasterOpExecutor masterOpExecutor = new MasterOpExecutor(new OriginStatement(showProcStmt, 0), - ConnectContext.get(), RedirectStatus.FORWARD_NO_SYNC, true); - try { - masterOpExecutor.execute(); - } catch (Exception e) { - LOG.warn("Fail to forward. ", e); - buffer.append("

Failed to forward request to master

"); - return; - } - - ShowResultSet resultSet = masterOpExecutor.getProxyResultSet(); - if (resultSet == null) { - buffer.append("

Failed to get result from master

"); - return; - } - - columnNames = resultSet.getMetaData().getColumns().stream().map(c -> c.getName()).collect( - Collectors.toList()); - rows = resultSet.getResultRows(); - } else { - ProcResult result; - try { - result = procNode.fetchResult(); - } catch (AnalysisException e) { - buffer.append("

The result is null, " - + "maybe haven't be implemented completely[" + e.getMessage() + "], please check.

"); - buffer.append("

" - + "INFO: ProcNode type is [" + procNode.getClass().getName() - + "]

"); - return; - } - - columnNames = result.getColumnNames(); - rows = result.getRows(); - } - - Preconditions.checkNotNull(columnNames); - Preconditions.checkNotNull(rows); - - appendTableHeader(buffer, columnNames); - appendSystemTableBody(buffer, rows, isDir, path); - appendTableFooter(buffer); - } - - private void appendSystemTableBody(StringBuilder buffer, List> rows, boolean isDir, String path) { - UrlValidator validator = new UrlValidator(); - for ( List strList : rows) { - buffer.append("
"); - int columnIndex = 1; - for (String str : strList) { - buffer.append(""); - ++columnIndex; - } - buffer.append(""); - } - } - - // some example: - // '/' => '/' - // '///aaa' => '///' - // '/aaa/bbb///' => '/aaa' - // '/aaa/bbb/ccc' => '/aaa/bbb' - // ATTN: the root path's parent is itself. - private String getParentPath(String path) { - int lastSlashIndex = path.length() - 1; - while (lastSlashIndex > 0) { - int tempIndex = path.lastIndexOf('/', lastSlashIndex); - if (tempIndex > 0) { - if (tempIndex == lastSlashIndex) { - lastSlashIndex = tempIndex - 1; - continue; - } else if (tempIndex < lastSlashIndex) { // '//aaa/bbb' - lastSlashIndex = tempIndex; - return path.substring(0, lastSlashIndex); - } - } - } - return "/"; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/VariableAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/VariableAction.java deleted file mode 100644 index 0df3428f6f8e8b..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/VariableAction.java +++ /dev/null @@ -1,84 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.analysis.SetType; -import org.apache.doris.common.Config; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.qe.VariableMgr; - -import com.google.common.collect.Lists; - -import io.netty.handler.codec.http.HttpMethod; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; - -public class VariableAction extends WebBaseAction { - - public VariableAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/variable", new VariableAction(controller)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - - appendConfigureInfo(response.getContent()); - appendVariableInfo(response.getContent()); - - getPageFooter(response.getContent()); - writeResponse(request, response); - } - - public void appendConfigureInfo(StringBuilder buffer) { - buffer.append("

Configure Info

"); - buffer.append("
");
-        HashMap confmap;
-        try {
-            confmap = Config.dump();
-            List keyList = Lists.newArrayList(confmap.keySet());
-            Collections.sort(keyList);
-            for (String key : keyList) {
-                buffer.append(key + "=" + confmap.get(key) + "\n");
-            }
-        } catch (Exception e) {
-            buffer.append("read conf exception" + e.toString());
-        }
-        buffer.append("
"); - } - - private void appendVariableInfo(StringBuilder buffer) { - buffer.append("

Variable Info

"); - buffer.append("
");
-        List> variableInfo = VariableMgr.dump(SetType.GLOBAL, null, null);
-        for (List list : variableInfo) {
-            buffer.append(list.get(0) + "=" + list.get(1) + "\n");
-        }
-        buffer.append("
"); - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/WebBaseAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/WebBaseAction.java deleted file mode 100644 index e710d68cfeaf6b..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/WebBaseAction.java +++ /dev/null @@ -1,374 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.analysis.CompoundPredicate.Operator; -import org.apache.doris.analysis.UserIdentity; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; -import org.apache.doris.common.proc.ProcNodeInterface; -import org.apache.doris.common.proc.ProcService; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseAction; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.HttpAuthManager; -import org.apache.doris.http.HttpAuthManager.SessionValue; -import org.apache.doris.http.UnauthorizedException; -import org.apache.doris.http.rest.RestBaseResult; -import org.apache.doris.mysql.privilege.PaloPrivilege; -import org.apache.doris.mysql.privilege.PrivBitSet; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.system.SystemInfoService; - -import com.google.common.base.Strings; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.List; -import java.util.UUID; - -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.codec.http.cookie.DefaultCookie; - -public class WebBaseAction extends BaseAction { - private static final Logger LOG = LogManager.getLogger(WebBaseAction.class); - - protected static final String PALO_SESSION_ID = "PALO_SESSION_ID"; - private static final long PALO_SESSION_EXPIRED_TIME = 3600 * 24; // one day - - protected static final String PAGE_HEADER = "" - + "" - + "" - + " Apache Doris(Incubating)" - + " " - + " " - + " " - + " " - + " " - - + " " - + " " - + " " - - + " " - - + " " - + "" - + ""; - protected static final String PAGE_FOOTER = ""; - protected static final String NAVIGATION_BAR_PREFIX = - " " - + "
"; - - public WebBaseAction(ActionController controller) { - super(controller); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - if (!checkAuthWithCookie(request, response)) { - return; - } - - HttpMethod method = request.getRequest().method(); - if (method.equals(HttpMethod.GET)) { - executeGet(request, response); - } else if (method.equals(HttpMethod.POST)) { - executePost(request, response); - } else { - response.appendContent(new RestBaseResult("HTTP method is not allowed: " + method.name()).toJson()); - writeResponse(request, response, HttpResponseStatus.METHOD_NOT_ALLOWED); - } - } - - // Sub Action class should override this method - public void executeGet(BaseRequest request, BaseResponse response) { - response.appendContent(new RestBaseResult("Not implemented").toJson()); - writeResponse(request, response, HttpResponseStatus.NOT_IMPLEMENTED); - } - - // Sub Action class should override this method - public void executePost(BaseRequest request, BaseResponse response) { - response.appendContent(new RestBaseResult("Not implemented").toJson()); - writeResponse(request, response, HttpResponseStatus.NOT_IMPLEMENTED); - } - - // We first check cookie, if not admin, we check http's authority header - private boolean checkAuthWithCookie(BaseRequest request, BaseResponse response) { - if (!needPassword()) { - return true; - } - - if (checkCookie(request, response)) { - return true; - } - - // cookie is invalid. - ActionAuthorizationInfo authInfo; - try { - authInfo = getAuthorizationInfo(request); - UserIdentity currentUser = checkPassword(authInfo); - if (needAdmin()) { - checkGlobalAuth(currentUser, PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, - PaloPrivilege.NODE_PRIV), Operator.OR)); - } - request.setAuthorized(true); - SessionValue value = new SessionValue(); - value.currentUser = currentUser; - addSession(request, response, value); - - ConnectContext ctx = new ConnectContext(null); - ctx.setQualifiedUser(authInfo.fullUserName); - ctx.setRemoteIP(authInfo.remoteIp); - ctx.setCurrentUserIdentity(currentUser); - ctx.setCatalog(Catalog.getCurrentCatalog()); - ctx.setCluster(SystemInfoService.DEFAULT_CLUSTER); - ctx.setThreadLocalInfo(); - - return true; - } catch (UnauthorizedException e) { - response.appendContent("Authentication Failed.
" + e.getMessage()); - writeAuthResponse(request, response); - return false; - } - } - - private boolean checkCookie(BaseRequest request, BaseResponse response) { - String sessionId = request.getCookieValue(PALO_SESSION_ID); - HttpAuthManager authMgr = HttpAuthManager.getInstance(); - if (!Strings.isNullOrEmpty(sessionId)) { - SessionValue sessionValue = authMgr.getSessionValue(sessionId); - if (sessionValue == null) { - return false; - } - if (Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(sessionValue.currentUser, - PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, - PaloPrivilege.NODE_PRIV), - Operator.OR))) { - response.updateCookieAge(request, PALO_SESSION_ID, PALO_SESSION_EXPIRED_TIME); - request.setAuthorized(true); - - ConnectContext ctx = new ConnectContext(null); - ctx.setQualifiedUser(sessionValue.currentUser.getQualifiedUser()); - ctx.setRemoteIP(request.getHostString()); - ctx.setCurrentUserIdentity(sessionValue.currentUser); - ctx.setCatalog(Catalog.getCurrentCatalog()); - ctx.setCluster(SystemInfoService.DEFAULT_CLUSTER); - ctx.setThreadLocalInfo(); - return true; - } - } - return false; - } - - // return true if this Action need to check password. - // Currently, all sub actions need to check password except for MetaBaseAction. - // if needPassword() is false, then needAdmin() should also return false - public boolean needPassword() { - return true; - } - - // return true if this Action need Admin privilege. - public boolean needAdmin() { - return true; - } - - protected void writeAuthResponse(BaseRequest request, BaseResponse response) { - response.updateHeader(HttpHeaderNames.WWW_AUTHENTICATE.toString(), "Basic realm=\"\""); - writeResponse(request, response, HttpResponseStatus.UNAUTHORIZED); - } - - protected void writeResponse(BaseRequest request, BaseResponse response) { - writeResponse(request, response, HttpResponseStatus.OK); - } - - protected void addSession(BaseRequest request, BaseResponse response, SessionValue value) { - String key = UUID.randomUUID().toString(); - DefaultCookie cookie = new DefaultCookie(PALO_SESSION_ID, key); - cookie.setMaxAge(PALO_SESSION_EXPIRED_TIME); - response.addCookie(cookie); - HttpAuthManager.getInstance().addSessionValue(key, value); - } - - protected void getPageHeader(BaseRequest request, StringBuilder sb) { - String newPageHeaderString = PAGE_HEADER; - newPageHeaderString = newPageHeaderString.replaceAll("Apache Doris", - "" + Config.cluster_name + ""); - - sb.append(newPageHeaderString); - sb.append(NAVIGATION_BAR_PREFIX); - - if (request.isAuthorized()) { - sb.append("
  • ") - .append("system") - .append("
  • "); - sb.append("
  • ") - .append("backends") - .append("
  • "); - sb.append("
  • ") - .append("logs") - .append("
  • "); - sb.append("
  • ") - .append("queries") - .append("
  • "); - sb.append("
  • ") - .append("sessions") - .append("
  • "); - sb.append("
  • ") - .append("variables") - .append("
  • "); - sb.append("
  • ") - .append("ha") - .append("
  • "); - } - sb.append("
  • ") - .append("help") - .append("
  • "); - - sb.append(NAVIGATION_BAR_SUFFIX); - } - - protected void getPageFooter(StringBuilder sb) { - sb.append(PAGE_FOOTER); - } - - // Note: DO NOT omit '
    ', because it is useful in 'datatable' plugin. - protected void appendTableHeader(StringBuilder buffer, List columnHeaders) { - buffer.append("
    " + columnHeaders.get(i) + "Profile
    "); - buffer.append(row.get(i)); - buffer.append("Empty Query ID"); - buffer.append("Profile"); - buffer.append("
    "); - if (isDir && columnIndex == 1) { - String escapeStr = str.replace("%", "%25"); - buffer.append(""); - buffer.append(str); - buffer.append(""); - } else if (validator.isValid(str)) { - buffer.append(""); - buffer.append("URL"); - buffer.append(""); - } else { - buffer.append(str.replaceAll("\\n", "
    ")); - } - buffer.append("
    "); - buffer.append(" "); - for (String str : columnHeaders) { - buffer.append(""); - } - buffer.append(" "); - } - - protected void appendTableBody(StringBuilder buffer, List> bodies) { - buffer.append(""); - for ( List row : bodies) { - buffer.append(""); - for (String column : row) { - buffer.append(""); - } - buffer.append(""); - } - buffer.append(""); - } - - protected void appendTableFooter(StringBuilder buffer) { - buffer.append("
    " + str + "
    "); - buffer.append(column); - buffer.append("
    "); - } - - protected ProcNodeInterface getProcNode(String path) { - ProcService instance = ProcService.getInstance(); - ProcNodeInterface node; - try { - if (Strings.isNullOrEmpty(path)) { - node = instance.open("/"); - } else { - node = instance.open(path); - } - } catch (AnalysisException e) { - LOG.warn(e.getMessage()); - return null; - } - return node; - } - - // Because org.apache.commons.lang.StringEscapeUtils.escapeHtml() not only escape tags in html, - // but also escape Chinese character code, which may cause Chinese garbled in browser, so we - // define our own simplified escape method here. - // ATTN: we should make sure file-encoding of help files is utf-8 - protected String escapeHtmlInPreTag(String oriStr) { - if (oriStr == null) { - return ""; - } - - StringBuilder buff = new StringBuilder(); - char[] chars = oriStr.toCharArray(); - for (int i = 0; i < chars.length; ++i) { - switch (chars[i]) { - case '<': - buff.append("<"); - break; - case '>': - buff.append("<"); - break; - case '"': - buff.append("""); - break; - case '&': - buff.append("&"); - break; - default: - buff.append(chars[i]); - break; - } - } - - return buff.toString(); - } - - private static final NotFoundAction NOT_FOUND_ACTION = new NotFoundAction(null); - - public static NotFoundAction getNotFoundAction() { - return NOT_FOUND_ACTION; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/common/DorisHttpPostObjectAggregator.java b/fe/fe-core/src/main/java/org/apache/doris/http/common/DorisHttpPostObjectAggregator.java deleted file mode 100644 index d904c421a3da76..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/common/DorisHttpPostObjectAggregator.java +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.common; - -import io.netty.channel.ChannelPipeline; -import io.netty.handler.codec.http.HttpContent; -import io.netty.handler.codec.http.HttpMessage; -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpObject; -import io.netty.handler.codec.http.HttpObjectAggregator; -import io.netty.handler.codec.http.HttpRequest; - -/* - * only handle post request, avoid conflicting with {@link LoadAction} - * don't handle 100-continue header - */ -public class DorisHttpPostObjectAggregator extends HttpObjectAggregator { - private boolean startAggregated = false; - - public DorisHttpPostObjectAggregator(int maxContentLength) { - super(maxContentLength, false); - } - - @Override - protected boolean isStartMessage(HttpObject msg) throws Exception { - if (msg instanceof HttpMessage) { - HttpRequest request = (HttpRequest) msg; - if (request.method().equals(HttpMethod.POST)) { - startAggregated = true; - return true; - } - } - return false; - } - - @Override - protected boolean isContentMessage(HttpObject msg) throws Exception { - return msg instanceof HttpContent && startAggregated; - } - - // Doris FE needn't handle 100-continue header - @Override - protected Object newContinueResponse(HttpMessage start, int maxContentLength, ChannelPipeline pipeline) { - return null; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/meta/ColocateMetaService.java b/fe/fe-core/src/main/java/org/apache/doris/http/meta/ColocateMetaService.java deleted file mode 100644 index 42b713d13cfa67..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/meta/ColocateMetaService.java +++ /dev/null @@ -1,229 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.meta; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.ColocateGroupSchema; -import org.apache.doris.catalog.ColocateTableIndex; -import org.apache.doris.catalog.ColocateTableIndex.GroupId; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.http.rest.RestBaseAction; -import org.apache.doris.http.rest.RestBaseResult; -import org.apache.doris.http.rest.RestResult; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.lang.reflect.Type; -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -/* - * the colocate meta define in {@link ColocateTableIndex} - * The actions in ColocateMetaService is for modifying or showing colocate group info manually. - * - * ColocateMetaAction: - * get all information in ColocateTableIndex, as a json string - * eg: - * GET /api/colocate - * return: - * {"colocate_meta":{"groupName2Id":{...},"group2Tables":{}, ...},"status":"OK"} - * - * eg: - * POST /api/colocate/group_stable?db_id=123&group_id=456 (mark group[123.456] as unstable) - * DELETE /api/colocate/group_stable?db_id=123&group_id=456 (mark group[123.456] as stable) - * - * BucketSeqAction: - * change the backends per bucket sequence of a group - * eg: - * POST /api/colocate/bucketseq?db_id=123&group_id=456 - */ -public class ColocateMetaService { - private static final Logger LOG = LogManager.getLogger(ColocateMetaService.class); - private static final String GROUP_ID = "group_id"; - private static final String TABLE_ID = "table_id"; - private static final String DB_ID = "db_id"; - - private static ColocateTableIndex colocateIndex = Catalog.getCurrentColocateIndex(); - - private static GroupId checkAndGetGroupId(BaseRequest request) throws DdlException { - long grpId = Long.valueOf(request.getSingleParameter(GROUP_ID).trim()); - long dbId = Long.valueOf(request.getSingleParameter(DB_ID).trim()); - GroupId groupId = new GroupId(dbId, grpId); - - if (!colocateIndex.isGroupExist(groupId)) { - throw new DdlException("the group " + groupId + "isn't exist"); - } - return groupId; - } - - public static class ColocateMetaBaseAction extends RestBaseAction { - ColocateMetaBaseAction(ActionController controller) { - super(controller); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - if (redirectToMaster(request, response)) { - return; - } - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); - executeInMasterWithAdmin(request, response); - } - - // implement in derived classes - protected void executeInMasterWithAdmin(BaseRequest request, BaseResponse response) - throws DdlException { - throw new DdlException("Not implemented"); - } - } - - // get all colocate meta - public static class ColocateMetaAction extends ColocateMetaBaseAction { - ColocateMetaAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - ColocateMetaAction action = new ColocateMetaAction(controller); - controller.registerHandler(HttpMethod.GET, "/api/colocate", action); - } - - @Override - public void executeInMasterWithAdmin(BaseRequest request, BaseResponse response) - throws DdlException { - response.setContentType("application/json"); - RestResult result = new RestResult(); - result.addResultEntry("colocate_meta", Catalog.getCurrentColocateIndex()); - sendResult(request, response, result); - } - } - - // mark a colocate group as stable or unstable - public static class MarkGroupStableAction extends ColocateMetaBaseAction { - MarkGroupStableAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - MarkGroupStableAction action = new MarkGroupStableAction(controller); - controller.registerHandler(HttpMethod.POST, "/api/colocate/group_stable", action); - controller.registerHandler(HttpMethod.DELETE, "/api/colocate/group_stable", action); - } - - @Override - public void executeInMasterWithAdmin(BaseRequest request, BaseResponse response) - throws DdlException { - GroupId groupId = checkAndGetGroupId(request); - - HttpMethod method = request.getRequest().method(); - if (method.equals(HttpMethod.POST)) { - colocateIndex.markGroupUnstable(groupId, "mark unstable via http api", true); - } else if (method.equals(HttpMethod.DELETE)) { - colocateIndex.markGroupStable(groupId, true); - } else { - response.appendContent(new RestBaseResult("HTTP method is not allowed.").toJson()); - writeResponse(request, response, HttpResponseStatus.METHOD_NOT_ALLOWED); - } - - sendResult(request, response); - } - } - - // update a backendsPerBucketSeq meta for a colocate group - public static class BucketSeqAction extends ColocateMetaBaseAction { - private static final Logger LOG = LogManager.getLogger(BucketSeqAction.class); - - BucketSeqAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - BucketSeqAction action = new BucketSeqAction(controller); - controller.registerHandler(HttpMethod.POST, "/api/colocate/bucketseq", action); - } - - @Override - public void executeInMasterWithAdmin(BaseRequest request, BaseResponse response) - throws DdlException { - final String clusterName = ConnectContext.get().getClusterName(); - if (Strings.isNullOrEmpty(clusterName)) { - throw new DdlException("No cluster selected."); - } - GroupId groupId = checkAndGetGroupId(request); - - String meta = request.getContent(); - Type type = new TypeToken>>() {}.getType(); - List> backendsPerBucketSeq = new Gson().fromJson(meta, type); - LOG.info("get buckets sequence: {}", backendsPerBucketSeq); - - ColocateGroupSchema groupSchema = Catalog.getCurrentColocateIndex().getGroupSchema(groupId); - if (backendsPerBucketSeq.size() != groupSchema.getBucketsNum()) { - throw new DdlException("Invalid bucket num. expected: " + groupSchema.getBucketsNum() + ", actual: " - + backendsPerBucketSeq.size()); - } - - List clusterBackendIds = Catalog.getCurrentSystemInfo().getClusterBackendIds(clusterName, true); - //check the Backend id - for (List backendIds : backendsPerBucketSeq) { - if (backendIds.size() != groupSchema.getReplicaAlloc().getTotalReplicaNum()) { - throw new DdlException("Invalid backend num per bucket. expected: " - + groupSchema.getReplicaAlloc().getTotalReplicaNum() + ", actual: " + backendIds.size()); - } - for (Long beId : backendIds) { - if (!clusterBackendIds.contains(beId)) { - throw new DdlException("The backend " + beId + " does not exist or not available"); - } - } - } - - int bucketsNum = colocateIndex.getBackendsPerBucketSeq(groupId).size(); - Preconditions.checkState(backendsPerBucketSeq.size() == bucketsNum, - backendsPerBucketSeq.size() + " vs. " + bucketsNum); - updateBackendPerBucketSeq(groupId, backendsPerBucketSeq); - LOG.info("the group {} backendsPerBucketSeq meta has been changed to {}", groupId, backendsPerBucketSeq); - - sendResult(request, response); - } - - private void updateBackendPerBucketSeq(GroupId groupId, List> backendsPerBucketSeq) - throws DdlException { - throw new DdlException("Currently not support"); - /* - colocateIndex.addBackendsPerBucketSeq(groupId, backendsPerBucketSeq); - ColocatePersistInfo info2 = ColocatePersistInfo.createForBackendsPerBucketSeq(groupId, backendsPerBucketSeq); - Catalog.getCurrentCatalog().getEditLog().logColocateBackendsPerBucketSeq(info2); - */ - } - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/meta/InvalidClientException.java b/fe/fe-core/src/main/java/org/apache/doris/http/meta/InvalidClientException.java deleted file mode 100644 index 6c5d9dbcf00826..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/meta/InvalidClientException.java +++ /dev/null @@ -1,26 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.meta; - -import org.apache.doris.common.DdlException; - -public class InvalidClientException extends DdlException { - public InvalidClientException(String msg) { - super(msg); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaBaseAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaBaseAction.java deleted file mode 100644 index b8d8843420e147..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaBaseAction.java +++ /dev/null @@ -1,108 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.meta; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.action.WebBaseAction; -import org.apache.doris.master.MetaHelper; -import org.apache.doris.system.Frontend; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.File; - -import io.netty.handler.codec.http.HttpResponseStatus; - -public class MetaBaseAction extends WebBaseAction { - private static final Logger LOG = LogManager.getLogger(MetaBaseAction.class); - private static String CONTENT_DISPOSITION = "Content-disposition"; - - public static final String CLUSTER_ID = "cluster_id"; - public static final String TOKEN = "token"; - - protected File imageDir; - - public MetaBaseAction(ActionController controller, File imageDir) { - super(controller); - this.imageDir = imageDir; - } - - @Override - public boolean needAdmin() { - return false; - } - - @Override - public boolean needPassword() { - return false; - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - if (needCheckClientIsFe()) { - try { - checkFromValidFe(request, response); - } catch (InvalidClientException e) { - response.appendContent("invalid client host."); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - } - super.execute(request, response); - } - - protected boolean needCheckClientIsFe() { - return true; - } - - protected void writeFileResponse(BaseRequest request, BaseResponse response, File file) { - if (file == null || !file.exists()) { - response.appendContent("File does not exist."); - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); - return; - } - - // add custom header - response.updateHeader(CONTENT_DISPOSITION, "attachment; filename=" + file.getName()); - response.updateHeader(MetaHelper.X_IMAGE_SIZE, String.valueOf(file.length())); - - writeObjectResponse(request, response, HttpResponseStatus.OK, file, file.getName(), true); - return; - } - - private boolean isFromValidFe(BaseRequest request) { - String clientHost = request.getHostString(); - Frontend fe = Catalog.getCurrentCatalog().getFeByHost(clientHost); - if (fe == null) { - LOG.warn("request is not from valid FE. client: {}", clientHost); - return false; - } - return true; - } - - private void checkFromValidFe(BaseRequest request, BaseResponse response) - throws InvalidClientException { - if (!isFromValidFe(request)) { - throw new InvalidClientException("invalid client host"); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaService.java b/fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaService.java deleted file mode 100644 index a7079000351a9d..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaService.java +++ /dev/null @@ -1,356 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.meta; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.Config; -import org.apache.doris.ha.FrontendNodeType; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.master.MetaHelper; -import org.apache.doris.persist.MetaCleaner; -import org.apache.doris.persist.Storage; -import org.apache.doris.persist.StorageInfo; -import org.apache.doris.system.Frontend; - -import com.google.common.base.Strings; -import com.google.gson.Gson; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.OutputStream; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -public class MetaService { - private static final int TIMEOUT_SECOND = 10; - - public static class ImageAction extends MetaBaseAction { - private static final String VERSION = "version"; - - public ImageAction(ActionController controller, File imageDir) { - super(controller, imageDir); - } - - public static void registerAction(ActionController controller, File imageDir) - throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/image", new ImageAction(controller, imageDir)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - String versionStr = request.getSingleParameter(VERSION); - if (Strings.isNullOrEmpty(versionStr)) { - response.appendContent("Miss version parameter"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - - long version = checkLongParam(versionStr); - if (version < 0) { - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - - File imageFile = Storage.getImageFile(imageDir, version); - if (!imageFile.exists()) { - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); - return; - } - - writeFileResponse(request, response, imageFile); - } - } - - public static class InfoAction extends MetaBaseAction { - private static final Logger LOG = LogManager.getLogger(InfoAction.class); - - public InfoAction(ActionController controller, File imageDir) { - super(controller, imageDir); - } - - public static void registerAction (ActionController controller, File imageDir) - throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/info", new InfoAction(controller, imageDir)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - try { - Storage currentStorageInfo = new Storage(imageDir.getAbsolutePath()); - StorageInfo storageInfo = new StorageInfo(currentStorageInfo.getClusterID(), - currentStorageInfo.getImageSeq(), currentStorageInfo.getEditsSeq()); - - response.setContentType("application/json"); - Gson gson = new Gson(); - response.appendContent(gson.toJson(storageInfo)); - writeResponse(request, response); - return; - } catch (IOException e) { - LOG.warn("IO error.", e); - response.appendContent("failed to get master info."); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - } - } - - public static class VersionAction extends MetaBaseAction { - public VersionAction(ActionController controller, File imageDir) { - super(controller, imageDir); - } - - public static void registerAction (ActionController controller, File imageDir) - throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/version", new VersionAction(controller, imageDir)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - File versionFile = new File(imageDir, Storage.VERSION_FILE); - writeFileResponse(request, response, versionFile); - } - } - - public static class PutAction extends MetaBaseAction { - private static final Logger LOG = LogManager.getLogger(PutAction.class); - - private static final String VERSION = "version"; - private static final String PORT = "port"; - - public PutAction(ActionController controller, File imageDir) { - super(controller, imageDir); - } - - public static void registerAction (ActionController controller, File imageDir) - throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/put", new PutAction(controller, imageDir)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - String machine = request.getHostString(); - String portStr = request.getSingleParameter(PORT); - // check port to avoid SSRF(Server-Side Request Forgery) - if (Strings.isNullOrEmpty(portStr)) { - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - { - int port = Integer.parseInt(portStr); - if (port < 0 || port > 65535) { - LOG.warn("port is invalid. port={}", port); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - } - - String versionStr = request.getSingleParameter(VERSION); - if (Strings.isNullOrEmpty(versionStr)) { - response.appendContent("Miss version parameter"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - - long version = checkLongParam(versionStr); - - // for master node, reject image put - if (Catalog.getCurrentCatalog().isMaster()) { - response.appendContent("this node is master, reject image put"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - LOG.error("this node is master, but receive image put from host {}, reject it", machine); - return; - } - - // do not accept image whose version is bigger than max journalId - // if accepted, newly added log will not be replayed when restart - long maxJournalId = Catalog.getCurrentCatalog().getMaxJournalId(); - if (version > maxJournalId) { - response.appendContent("image version [" + version + "] is bigger than local max journal id [" - + maxJournalId + "], reject image put"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - LOG.error("receive image whose version [{}] is bigger than local max journal id [{}], reject it", - version, maxJournalId); - return; - } - - String url = "http://" + machine + ":" + portStr - + "/image?version=" + versionStr; - String filename = Storage.IMAGE + "." + versionStr; - - File dir = new File(Catalog.getCurrentCatalog().getImageDir()); - try { - OutputStream out = MetaHelper.getOutputStream(filename, dir); - MetaHelper.getRemoteFile(url, TIMEOUT_SECOND * 1000, out); - MetaHelper.complete(filename, dir); - writeResponse(request, response); - } catch (FileNotFoundException e) { - LOG.warn("file not found. file: {}", filename, e); - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); - return; - } catch (IOException e) { - LOG.warn("failed to get remote file. url: {}", url, e); - writeResponse(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR); - return; - } - - // Delete old image files - MetaCleaner cleaner = new MetaCleaner(Config.meta_dir + "/image"); - try { - cleaner.clean(); - } catch (IOException e) { - LOG.error("Follower/Observer delete old image file fail.", e); - } - } - } - - public static class JournalIdAction extends MetaBaseAction { - public JournalIdAction(ActionController controller, File imageDir) { - super(controller, imageDir); - } - - public static void registerAction (ActionController controller, File imageDir) - throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/journal_id", new JournalIdAction(controller, imageDir)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - long id = Catalog.getCurrentCatalog().getReplayedJournalId(); - response.updateHeader("id", Long.toString(id)); - writeResponse(request, response); - } - } - - public static class RoleAction extends MetaBaseAction { - private static final String HOST = "host"; - private static final String PORT = "port"; - - public RoleAction(ActionController controller, File imageDir) { - super(controller, imageDir); - } - - public static void registerAction (ActionController controller, File imageDir) - throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/role", new RoleAction(controller, imageDir)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - String host = request.getSingleParameter(HOST); - String portString = request.getSingleParameter(PORT); - - if (!Strings.isNullOrEmpty(host) && !Strings.isNullOrEmpty(portString)) { - int port = Integer.parseInt(portString); - Frontend fe = Catalog.getCurrentCatalog().checkFeExist(host, port); - if (fe == null) { - response.updateHeader("role", FrontendNodeType.UNKNOWN.name()); - } else { - response.updateHeader("role", fe.getRole().name()); - response.updateHeader("name", fe.getNodeName()); - } - writeResponse(request, response); - } else { - response.appendContent("Miss parameter"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - } - } - - /* - * This action is used to get the electable_nodes config and the cluster id of - * the fe with the given ip and port. When one frontend start, it should check - * the local electable_nodes config and local cluster id with other frontends. - * If there is any difference, local fe will exit. This is designed to protect - * the consistency of the cluster. - */ - public static class CheckAction extends MetaBaseAction { - private static final Logger LOG = LogManager.getLogger(CheckAction.class); - - public CheckAction(ActionController controller, File imageDir) { - super(controller, imageDir); - } - - public static void registerAction(ActionController controller, File imageDir) - throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/check", - new CheckAction(controller, imageDir)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - try { - Storage storage = new Storage(imageDir.getAbsolutePath()); - response.updateHeader(MetaBaseAction.CLUSTER_ID, Integer.toString(storage.getClusterID())); - response.updateHeader(MetaBaseAction.TOKEN, storage.getToken()); - } catch (IOException e) { - LOG.error(e); - } - writeResponse(request, response); - } - } - - public static class DumpAction extends MetaBaseAction { - private static final Logger LOG = LogManager.getLogger(CheckAction.class); - - public DumpAction(ActionController controller, File imageDir) { - super(controller, imageDir); - } - - public static void registerAction (ActionController controller, File imageDir) - throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/dump", new DumpAction(controller, imageDir)); - } - - @Override - public boolean needAdmin() { - return true; - } - - @Override - protected boolean needCheckClientIsFe() { - return false; - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - /* - * Before dump, we acquired the catalog read lock and all databases' read lock and all - * the jobs' read lock. This will guarantee the consistency of database and job queues. - * But Backend may still inconsistent. - */ - - // TODO: Still need to lock ClusterInfoService to prevent add or drop Backends - String dumpFilePath = Catalog.getCurrentCatalog().dumpImage(); - if (dumpFilePath == null) { - response.appendContent("dump failed. " + dumpFilePath); - } - - response.appendContent("dump finished. " + dumpFilePath); - writeResponse(request, response); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/BootstrapFinishAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/BootstrapFinishAction.java deleted file mode 100644 index 7af15429eec000..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/BootstrapFinishAction.java +++ /dev/null @@ -1,162 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.Config; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import com.google.common.base.Strings; -import com.google.gson.Gson; - -import io.netty.handler.codec.http.HttpMethod; -import org.apache.doris.common.Version; - -/* - * fe_host:fe_http_port/api/bootstrap - * return: - * {"status":"OK","msg":"Success","replayedJournal"=123456, "queryPort"=9000, "rpcPort"=9001} - * {"status":"FAILED","msg":"err info..."} - */ -public class BootstrapFinishAction extends RestBaseAction { - private static final String CLUSTER_ID = "cluster_id"; - private static final String TOKEN = "token"; - - public static final String REPLAYED_JOURNAL_ID = "replayedJournalId"; - public static final String QUERY_PORT = "queryPort"; - public static final String RPC_PORT = "rpcPort"; - public static final String VERSION = "version"; - - public BootstrapFinishAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/bootstrap", new BootstrapFinishAction(controller)); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) throws DdlException { - boolean isReady = Catalog.getCurrentCatalog().isReady(); - - // to json response - BootstrapResult result = null; - if (isReady) { - result = new BootstrapResult(); - String clusterIdStr = request.getSingleParameter(CLUSTER_ID); - String token = request.getSingleParameter(TOKEN); - if (!Strings.isNullOrEmpty(clusterIdStr) && !Strings.isNullOrEmpty(token)) { - // cluster id or token is provided, return more info - int clusterId = 0; - try { - clusterId = Integer.valueOf(clusterIdStr); - } catch (NumberFormatException e) { - result.status = ActionStatus.FAILED; - result.msg = "invalid cluster id format: " + clusterIdStr; - } - - if (result.status == ActionStatus.OK) { - if (clusterId != Catalog.getCurrentCatalog().getClusterId()) { - result.status = ActionStatus.FAILED; - result.msg = "invalid cluster id: " + Catalog.getCurrentCatalog().getClusterId(); - } - } - - if (result.status == ActionStatus.OK) { - if (!token.equals(Catalog.getCurrentCatalog().getToken())) { - result.status = ActionStatus.FAILED; - result.msg = "invalid token: " + Catalog.getCurrentCatalog().getToken(); - } - } - - if (result.status == ActionStatus.OK) { - // cluster id and token are valid, return replayed journal id - long replayedJournalId = Catalog.getCurrentCatalog().getReplayedJournalId(); - result.setMaxReplayedJournal(replayedJournalId); - result.setQueryPort(Config.query_port); - result.setRpcPort(Config.rpc_port); - result.setVersion(Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH); - } - } - } else { - result = new BootstrapResult("not ready"); - } - - // send result - response.setContentType("application/json"); - response.getContent().append(result.toJson()); - sendResult(request, response); - } - - public static class BootstrapResult extends RestBaseResult { - private long replayedJournalId = 0; - private int queryPort = 0; - private int rpcPort = 0; - private String version = ""; - - public BootstrapResult() { - super(); - } - - public BootstrapResult(String msg) { - super(msg); - } - - public void setMaxReplayedJournal(long replayedJournalId) { - this.replayedJournalId = replayedJournalId; - } - - public long getMaxReplayedJournal() { - return replayedJournalId; - } - - public void setQueryPort(int queryPort) { - this.queryPort = queryPort; - } - - public int getQueryPort() { - return queryPort; - } - - public void setRpcPort(int rpcPort) { - this.rpcPort = rpcPort; - } - - public int getRpcPort() { - return rpcPort; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - - @Override - public String toJson() { - Gson gson = new Gson(); - return gson.toJson(this); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/CancelStreamLoad.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/CancelStreamLoad.java deleted file mode 100644 index 23f12f441a196f..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/CancelStreamLoad.java +++ /dev/null @@ -1,83 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.UserException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; - -public class CancelStreamLoad extends RestBaseAction { - public CancelStreamLoad(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) - throws IllegalArgException { - CancelStreamLoad action = new CancelStreamLoad(controller); - controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_cancel", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException { - - if (redirectToMaster(request, response)) { - return; - } - - final String clusterName = ConnectContext.get().getClusterName(); - if (Strings.isNullOrEmpty(clusterName)) { - throw new DdlException("No cluster selected."); - } - - String dbName = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(dbName)) { - throw new DdlException("No database selected."); - } - - String fullDbName = ClusterNamespace.getFullName(clusterName, dbName); - - String label = request.getSingleParameter(LABEL_KEY); - if (Strings.isNullOrEmpty(label)) { - throw new DdlException("No label selected."); - } - - // FIXME(cmy) - // checkWritePriv(authInfo.fullUserName, fullDbName); - - Database db = Catalog.getCurrentCatalog().getDbOrDdlException(fullDbName); - - try { - Catalog.getCurrentGlobalTransactionMgr().abortTransaction(db.getId(), label, "user cancel"); - } catch (UserException e) { - throw new DdlException(e.getMessage()); - } - - sendResult(request, response, new RestBaseResult()); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/CheckDecommissionAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/CheckDecommissionAction.java deleted file mode 100644 index 10a555c222c2d4..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/CheckDecommissionAction.java +++ /dev/null @@ -1,93 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.alter.SystemHandler; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.Pair; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.system.SystemInfoService; - -import com.google.common.base.Strings; -import com.google.common.collect.Lists; - -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; - -/* - * calc row count from replica to table - * fe_host:fe_http_port/api/check_decommission?host_ports=host:port,host2:port2... - * return: - * {"status":"OK","msg":"Success"} - * {"status":"FAILED","msg":"err info..."} - */ -public class CheckDecommissionAction extends RestBaseAction { - public static final String HOST_PORTS = "host_ports"; - - public CheckDecommissionAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/check_decommission", new CheckDecommissionAction(controller)); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.OPERATOR); - - String hostPorts = request.getSingleParameter(HOST_PORTS); - if (Strings.isNullOrEmpty(hostPorts)) { - throw new DdlException("No host:port specified."); - } - - String[] hostPortArr = hostPorts.split(","); - if (hostPortArr.length == 0) { - throw new DdlException("No host:port specified."); - } - - List> hostPortPairs = Lists.newArrayList(); - for (String hostPort : hostPortArr) { - Pair pair; - try { - pair = SystemInfoService.validateHostAndPort(hostPort); - } catch (AnalysisException e) { - throw new DdlException(e.getMessage()); - } - hostPortPairs.add(pair); - } - - SystemHandler.checkDecommission(hostPortPairs); - - // to json response - RestBaseResult result = new RestBaseResult(); - - // send result - response.setContentType("application/json"); - response.getContent().append(result.toJson()); - sendResult(request, response); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ConnectionAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/ConnectionAction.java deleted file mode 100644 index 3ac8f288be7fb5..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ConnectionAction.java +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; -import org.apache.doris.common.util.DebugUtil; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.service.ExecuteEnv; - -// This class is used to get current query_id of connection_id. -// Every connection holds at most one query at every point. -// Some we can get query_id firstly, and get query by query_id. -public class ConnectionAction extends RestBaseAction { - public ConnectionAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/connection", new ConnectionAction(controller)); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - String connStr = request.getSingleParameter("connection_id"); - if (connStr == null) { - response.getContent().append("not valid parameter"); - sendResult(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - int connectionId = Integer.valueOf(connStr.trim()); - ConnectContext context = ExecuteEnv.getInstance().getScheduler().getContext(connectionId); - if (context == null || context.queryId() == null) { - response.getContent().append("connection id " + connectionId + " not found."); - sendResult(request, response, HttpResponseStatus.NOT_FOUND); - return; - } - String queryId = DebugUtil.printId(context.queryId()); - response.setContentType("application/json"); - response.getContent().append("{\"query_id\" : \"" + queryId + "\"}"); - sendResult(request, response); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetDdlStmtAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetDdlStmtAction.java deleted file mode 100644 index e9c23429fa2451..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetDdlStmtAction.java +++ /dev/null @@ -1,108 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.Table; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.codehaus.jackson.map.ObjectMapper; - -import java.util.List; -import java.util.Map; - -import io.netty.handler.codec.http.HttpMethod; - -/* - * used to get a table's ddl stmt - * eg: - * fe_host:http_port/api/_get_ddl?db=xxx&tbl=yyy - */ -public class GetDdlStmtAction extends RestBaseAction { - private static final Logger LOG = LogManager.getLogger(GetDdlStmtAction.class); - private static final String DB_PARAM = "db"; - private static final String TABLE_PARAM = "tbl"; - - public GetDdlStmtAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - GetDdlStmtAction action = new GetDdlStmtAction(controller); - controller.registerHandler(HttpMethod.GET, "/api/_get_ddl", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); - - String dbName = request.getSingleParameter(DB_PARAM); - String tableName = request.getSingleParameter(TABLE_PARAM); - - if (Strings.isNullOrEmpty(dbName) || Strings.isNullOrEmpty(tableName)) { - throw new DdlException("Missing params. Need database name and Table name"); - } - - Database db = Catalog.getCurrentCatalog().getDbOrDdlException(dbName); - Table table = db.getTableOrDdlException(tableName); - - List createTableStmt = Lists.newArrayList(); - List addPartitionStmt = Lists.newArrayList(); - List createRollupStmt = Lists.newArrayList(); - - table.readLock(); - try { - Catalog.getDdlStmt(table, createTableStmt, addPartitionStmt, createRollupStmt, true, false /* show password */); - } finally { - table.readUnlock(); - } - - Map> results = Maps.newHashMap(); - results.put("TABLE", createTableStmt); - results.put("PARTITION", addPartitionStmt); - results.put("ROLLUP", createRollupStmt); - - // to json response - String result = ""; - ObjectMapper mapper = new ObjectMapper(); - try { - result = mapper.writeValueAsString(results); - } catch (Exception e) { - // do nothing - } - - // send result - response.setContentType("application/json"); - response.getContent().append(result); - sendResult(request, response); - } -} \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLoadInfoAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLoadInfoAction.java deleted file mode 100644 index b215702817fbb4..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLoadInfoAction.java +++ /dev/null @@ -1,88 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.common.DdlException; -import org.apache.doris.common.MetaNotFoundException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.load.Load; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; - -// Get load information of one load job -public class GetLoadInfoAction extends RestBaseAction { - public GetLoadInfoAction(ActionController controller, boolean isStreamLoad) { - super(controller); - } - - public static void registerAction(ActionController controller) - throws IllegalArgException { - GetLoadInfoAction action = new GetLoadInfoAction(controller, false); - controller.registerHandler(HttpMethod.GET, "/api/{" + DB_KEY + "}/_load_info", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - Load.JobInfo info = new Load.JobInfo(request.getSingleParameter(DB_KEY), - request.getSingleParameter(LABEL_KEY), - ConnectContext.get().getClusterName()); - if (Strings.isNullOrEmpty(info.dbName)) { - throw new DdlException("No database selected"); - } - if (Strings.isNullOrEmpty(info.label)) { - throw new DdlException("No label selected"); - } - if (Strings.isNullOrEmpty(info.clusterName)) { - throw new DdlException("No cluster name selected"); - } - - if (redirectToMaster(request, response)) { - return; - } - - try { - catalog.getLoadInstance().getJobInfo(info); - if (info.tblNames.isEmpty()) { - checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), info.dbName, PrivPredicate.LOAD); - } else { - for (String tblName : info.tblNames) { - checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), info.dbName, tblName, - PrivPredicate.LOAD); - } - } - } catch (DdlException | MetaNotFoundException e) { - catalog.getLoadManager().getLoadJobInfo(info); - } - sendResult(request, response, new Result(info)); - } - - private static class Result extends RestBaseResult { - private Load.JobInfo jobInfo; - public Result(Load.JobInfo info) { - jobInfo = info; - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLogFileAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLogFileAction.java deleted file mode 100644 index 9e80219e647a1f..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLogFileAction.java +++ /dev/null @@ -1,131 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.common.Config; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import com.google.common.base.Strings; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; - -import org.codehaus.jackson.map.ObjectMapper; - -import java.io.File; -import java.util.Map; -import java.util.Set; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -/* - * get log file infos: - * curl -I http://fe_host:http_port/api/get_log_file?type=fe.audit.log - * return: - * HTTP/1.1 200 OK - * file_infos: {"fe.audit.log":24759,"fe.audit.log.20190528.1":132934} - * content-type: text/html - * connection: keep-alive - * - * get log file: - * curl -X GET http://fe_host:http_port/api/get_log_file?type=fe.audit.log&file=fe.audit.log.20190528.1 - */ -public class GetLogFileAction extends RestBaseAction { - private final Set logFileTypes = Sets.newHashSet("fe.audit.log"); - - public GetLogFileAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/get_log_file", new GetLogFileAction(controller)); - controller.registerHandler(HttpMethod.HEAD, "/api/get_log_file", new GetLogFileAction(controller)); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) { - String logType = request.getSingleParameter("type"); - String logFile = request.getSingleParameter("file"); - - // check param empty - if (Strings.isNullOrEmpty(logType)) { - response.appendContent("Miss type parameter"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - - // check type valid or not - if (!logFileTypes.contains(logType)) { - response.appendContent("log type: " + logType + " is invalid!"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - - HttpMethod method = request.getRequest().method(); - if (method.equals(HttpMethod.HEAD)) { - String fileInfos = getFileInfos(logType); - response.updateHeader("file_infos", fileInfos); - writeResponse(request, response, HttpResponseStatus.OK); - return; - } else if (method.equals(HttpMethod.GET)) { - File log = getLogFile(logType, logFile); - if (!log.exists() || !log.isFile()) { - response.appendContent("Log file not exist: " + log.getName()); - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); - return; - } - writeObjectResponse(request, response, HttpResponseStatus.OK, log, log.getName(), true); - } else { - response.appendContent(new RestBaseResult("HTTP method is not allowed.").toJson()); - writeResponse(request, response, HttpResponseStatus.METHOD_NOT_ALLOWED); - } - } - - private String getFileInfos(String logType) { - Map fileInfos = Maps.newTreeMap(); - if (logType.equals("fe.audit.log")) { - File logDir = new File(Config.audit_log_dir); - File[] files = logDir.listFiles(); - for (int i = 0; i < files.length; i++) { - if (files[i].isFile() && files[i].getName().startsWith("fe.audit.log")) { - fileInfos.put(files[i].getName(), files[i].length()); - } - } - } - - String result = ""; - ObjectMapper mapper = new ObjectMapper(); - try { - result = mapper.writeValueAsString(fileInfos); - } catch (Exception e) { - // do nothing - } - return result; - } - - private File getLogFile(String logType, String logFile) { - String logPath = ""; - if ("fe.audit.log".equals(logType)) { - logPath = Config.audit_log_dir + "/" + logFile; - } - return new File(logPath); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetSmallFileAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetSmallFileAction.java deleted file mode 100644 index c678b8a10adeb3..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetSmallFileAction.java +++ /dev/null @@ -1,92 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.util.SmallFileMgr; -import org.apache.doris.common.util.SmallFileMgr.SmallFile; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import com.google.common.base.Strings; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -public class GetSmallFileAction extends RestBaseAction { - private static final Logger LOG = LogManager.getLogger(GetSmallFileAction.class); - - public GetSmallFileAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/get_small_file", new GetSmallFileAction(controller)); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - String token = request.getSingleParameter("token"); - String fileIdStr = request.getSingleParameter("file_id"); - - // check param empty - if (Strings.isNullOrEmpty(token) || Strings.isNullOrEmpty(fileIdStr)) { - response.appendContent("Missing parameter"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - - // check token - if (!token.equals(Catalog.getCurrentCatalog().getToken())) { - response.appendContent("Invalid token"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - - long fileId = -1; - try { - fileId = Long.valueOf(fileIdStr); - } catch (NumberFormatException e) { - response.appendContent("Invalid file id format: " + fileIdStr); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - - SmallFileMgr fileMgr = Catalog.getCurrentCatalog().getSmallFileMgr(); - SmallFile smallFile = fileMgr.getSmallFile(fileId); - if (smallFile == null || !smallFile.isContent) { - response.appendContent("File not found or is not content"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - - HttpMethod method = request.getRequest().method(); - if (method.equals(HttpMethod.GET)) { - writeObjectResponse(request, response, HttpResponseStatus.OK, smallFile.getContentBytes(), - smallFile.name, true); - } else { - response.appendContent(new RestBaseResult("HTTP method is not allowed.").toJson()); - writeResponse(request, response, HttpResponseStatus.METHOD_NOT_ALLOWED); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetStreamLoadState.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetStreamLoadState.java deleted file mode 100644 index 6be12368eb263c..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetStreamLoadState.java +++ /dev/null @@ -1,86 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; - -public class GetStreamLoadState extends RestBaseAction { - public GetStreamLoadState(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) - throws IllegalArgException { - GetStreamLoadState action = new GetStreamLoadState(controller); - controller.registerHandler(HttpMethod.GET, "/api/{" + DB_KEY + "}/get_load_state", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - - if (redirectToMaster(request, response)) { - return; - } - - final String clusterName = ConnectContext.get().getClusterName(); - if (Strings.isNullOrEmpty(clusterName)) { - throw new DdlException("No cluster selected."); - } - - String dbName = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(dbName)) { - throw new DdlException("No database selected."); - } - - String fullDbName = ClusterNamespace.getFullName(clusterName, dbName); - - String label = request.getSingleParameter(LABEL_KEY); - if (Strings.isNullOrEmpty(label)) { - throw new DdlException("No label selected."); - } - - // FIXME(cmy) - // checkReadPriv(authInfo.fullUserName, fullDbName); - - Database db = Catalog.getCurrentCatalog().getDbOrDdlException(fullDbName); - - String state = Catalog.getCurrentGlobalTransactionMgr().getLabelState(db.getId(), label).toString(); - - sendResult(request, response, new Result(state)); - } - - private static class Result extends RestBaseResult { - private String state; - public Result(String state) { - this.state = state; - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/HealthAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/HealthAction.java deleted file mode 100644 index 78322a68ad563f..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/HealthAction.java +++ /dev/null @@ -1,47 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import io.netty.handler.codec.http.HttpMethod; - -public class HealthAction extends RestBaseAction { - public HealthAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) - throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/health", new HealthAction(controller)); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - response.setContentType("application/json"); - - RestResult result = new RestResult(); - result.addResultEntry("total_backend_num", Catalog.getCurrentSystemInfo().getBackendIds(false).size()); - result.addResultEntry("online_backend_num", Catalog.getCurrentSystemInfo().getBackendIds(true).size()); - sendResult(request, response, result); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/LoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/LoadAction.java deleted file mode 100644 index f0d51065eb89be..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/LoadAction.java +++ /dev/null @@ -1,141 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.service.ExecuteEnv; -import org.apache.doris.system.Backend; -import org.apache.doris.system.SystemInfoService; -import org.apache.doris.thrift.TNetworkAddress; - -import com.google.common.base.Strings; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.List; - -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpMethod; - -public class LoadAction extends RestBaseAction { - private static final Logger LOG = LogManager.getLogger(LoadAction.class); - - public static final String SUB_LABEL_NAME_PARAM = "sub_label"; - - private ExecuteEnv execEnv; - private boolean isStreamLoad = false; - - public LoadAction(ActionController controller, ExecuteEnv execEnv) { - this(controller, execEnv, false); - } - - public LoadAction(ActionController controller, ExecuteEnv execEnv, boolean isStreamLoad) { - super(controller); - this.execEnv = execEnv; - this.isStreamLoad = isStreamLoad; - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - ExecuteEnv execEnv = ExecuteEnv.getInstance(); - LoadAction action = new LoadAction(controller, execEnv); - controller.registerHandler(HttpMethod.PUT, - "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_load", action); - - controller.registerHandler(HttpMethod.PUT, - "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_stream_load", - new LoadAction(controller, execEnv, true)); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException { - - // A 'Load' request must have 100-continue header - if (!request.getRequest().headers().contains(HttpHeaders.Names.EXPECT)) { - throw new DdlException("There is no 100-continue header"); - } - - final String clusterName = ConnectContext.get().getClusterName(); - if (Strings.isNullOrEmpty(clusterName)) { - throw new DdlException("No cluster selected."); - } - - String dbName = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(dbName)) { - throw new DdlException("No database selected."); - } - - String tableName = request.getSingleParameter(TABLE_KEY); - if (Strings.isNullOrEmpty(tableName)) { - throw new DdlException("No table selected."); - } - - String fullDbName = ClusterNamespace.getFullName(clusterName, dbName); - - String label = request.getSingleParameter(LABEL_KEY); - if (!isStreamLoad) { - if (Strings.isNullOrEmpty(label)) { - throw new DdlException("No label selected."); - } - } else { - label = request.getRequest().headers().get(LABEL_KEY); - } - - // check auth - checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tableName, PrivPredicate.LOAD); - - TNetworkAddress redirectAddr; - if (!isStreamLoad && !Strings.isNullOrEmpty(request.getSingleParameter(SUB_LABEL_NAME_PARAM))) { - // only multi mini load need to redirect to Master, because only Master has the info of table to - // the Backend which the file exists. - if (redirectToMaster(request, response)) { - return; - } - redirectAddr = execEnv.getMultiLoadMgr().redirectAddr(fullDbName, label); - } else { - // Choose a backend sequentially. - SystemInfoService.BeAvailablePredicate beAvailablePredicate = - new SystemInfoService.BeAvailablePredicate(false, false, true); - List backendIds = Catalog.getCurrentSystemInfo().seqChooseBackendIdsByStorageMediumAndTag( - 1, beAvailablePredicate, false, clusterName, null, null); - if (backendIds == null) { - throw new DdlException(SystemInfoService.NO_BACKEND_LOAD_AVAILABLE_MSG); - } - - Backend backend = Catalog.getCurrentSystemInfo().getBackend(backendIds.get(0)); - if (backend == null) { - throw new DdlException(SystemInfoService.NO_BACKEND_LOAD_AVAILABLE_MSG); - } - - redirectAddr = new TNetworkAddress(backend.getHost(), backend.getHttpPort()); - } - - LOG.info("redirect load action to destination={}, stream: {}, db: {}, tbl: {}, label: {}", - redirectAddr.toString(), isStreamLoad, dbName, tableName, label); - redirectTo(request, response, redirectAddr); - } -} - diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MetaReplayerCheckAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MetaReplayerCheckAction.java deleted file mode 100644 index f874ecf9bd716a..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MetaReplayerCheckAction.java +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import org.codehaus.jackson.map.ObjectMapper; - -import java.util.Map; - -import io.netty.handler.codec.http.HttpMethod; - -/* - * used to get meta replay info - * eg: - * fe_host:http_port/api/_meta_replay_state - */ -public class MetaReplayerCheckAction extends RestBaseAction { - public MetaReplayerCheckAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - MetaReplayerCheckAction action = new MetaReplayerCheckAction(controller); - controller.registerHandler(HttpMethod.GET, "/api/_meta_replay_state", action); - } - - @Override - protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException { - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); - - Map resultMap = Catalog.getCurrentCatalog().getMetaReplayState().getInfo(); - - // to json response - String result = ""; - ObjectMapper mapper = new ObjectMapper(); - try { - result = mapper.writeValueAsString(resultMap); - } catch (Exception e) { - // do nothing - } - - // send result - response.setContentType("application/json"); - response.getContent().append(result); - sendResult(request, response); - } -} \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MetricsAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MetricsAction.java deleted file mode 100644 index 34bc7854e77be0..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MetricsAction.java +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.metric.MetricRepo; -import org.apache.doris.metric.MetricVisitor; -import org.apache.doris.metric.JsonMetricVisitor; -import org.apache.doris.metric.PrometheusMetricVisitor; -import org.apache.doris.metric.SimpleCoreMetricVisitor; - -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; - -//fehost:port/metrics -//fehost:port/metrics?type=core -//fehost:port/metrics?type=json -public class MetricsAction extends RestBaseAction { - - private static final String TYPE_PARAM = "type"; - - public MetricsAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/metrics", new MetricsAction(controller)); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - String type = request.getSingleParameter(TYPE_PARAM); - MetricVisitor visitor = null; - if (!Strings.isNullOrEmpty(type) && type.equalsIgnoreCase("core")) { - visitor = new SimpleCoreMetricVisitor("doris_fe"); - } else if (!Strings.isNullOrEmpty(type) && type.equalsIgnoreCase("json")) { - visitor = new JsonMetricVisitor("doris_fe"); - } else { - visitor = new PrometheusMetricVisitor("doris_fe"); - } - - response.setContentType("text/plain"); - response.getContent().append(MetricRepo.getMetric(visitor)); - sendResult(request, response); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MigrationAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MigrationAction.java deleted file mode 100644 index 6f36c5736fe9c3..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MigrationAction.java +++ /dev/null @@ -1,164 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.MaterializedIndex; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.Replica; -import org.apache.doris.catalog.Table; -import org.apache.doris.catalog.Table.TableType; -import org.apache.doris.catalog.Tablet; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.util.ListComparator; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.base.Strings; -import com.google.common.collect.Lists; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.codehaus.jackson.map.ObjectMapper; - -import java.util.Collections; -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; - -/* - * used to get table's sorted tablet info - * eg: - * fe_host:http_port/api/_migration?db=xxx&tbl=yyy - */ -public class MigrationAction extends RestBaseAction { - private static final Logger LOG = LogManager.getLogger(MigrationAction.class); - private static final String DB_PARAM = "db"; - private static final String TABLE_PARAM = "tbl"; - - public MigrationAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - MigrationAction action = new MigrationAction(controller); - controller.registerHandler(HttpMethod.GET, "/api/_migration", action); - } - - @Override - protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException { - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); - - String dbName = request.getSingleParameter(DB_PARAM); - String tableName = request.getSingleParameter(TABLE_PARAM); - - if (Strings.isNullOrEmpty(dbName)) { - throw new DdlException("Missing params. Need database name"); - } - - Database db = Catalog.getCurrentCatalog().getDbOrDdlException(dbName); - - List> rows = Lists.newArrayList(); - - - - if (!Strings.isNullOrEmpty(tableName)) { - OlapTable olapTable = db.getOlapTableOrDdlException(tableName); - olapTable.readLock(); - try { - for (Partition partition : olapTable.getPartitions()) { - String partitionName = partition.getName(); - MaterializedIndex baseIndex = partition.getBaseIndex(); - for (Tablet tablet : baseIndex.getTablets()) { - List row = Lists.newArrayList(); - row.add(tableName); - row.add(partitionName); - row.add(tablet.getId()); - row.add(olapTable.getSchemaHashByIndexId(baseIndex.getId())); - for (Replica replica : tablet.getReplicas()) { - row.add(replica.getBackendId()); - break; - } - rows.add(row); - } - } - } finally { - olapTable.readUnlock(); - } - } else { - List tableList = db.getTables(); - - // get all olap table - for (Table table : tableList) { - if (table.getType() != TableType.OLAP) { - continue; - } - - OlapTable olapTable = (OlapTable) table; - table.readLock(); - try { - tableName = table.getName(); - for (Partition partition : olapTable.getPartitions()) { - String partitionName = partition.getName(); - MaterializedIndex baseIndex = partition.getBaseIndex(); - for (Tablet tablet : baseIndex.getTablets()) { - List row = Lists.newArrayList(); - row.add(tableName); - row.add(partitionName); - row.add(tablet.getId()); - row.add(olapTable.getSchemaHashByIndexId(baseIndex.getId())); - for (Replica replica : tablet.getReplicas()) { - row.add(replica.getBackendId()); - break; - } - rows.add(row); - } - } - } finally { - table.readUnlock(); - } - } - } - ListComparator> comparator = new ListComparator>(0, 1, 2); - Collections.sort(rows, comparator); - - // to json response - String result = ""; - ObjectMapper mapper = new ObjectMapper(); - try { - result = mapper.writeValueAsString(rows); - } catch (Exception e) { - // do nothing - } - - // send result - response.setContentType("application/json"); - response.getContent().append(result); - sendResult(request, response); - } - - public static void print(String msg) { - System.out.println(System.currentTimeMillis() + " " + msg); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiAbort.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiAbort.java deleted file mode 100644 index 893a47b92b3420..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiAbort.java +++ /dev/null @@ -1,72 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.service.ExecuteEnv; - -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; - -public class MultiAbort extends RestBaseAction { - private ExecuteEnv execEnv; - - public MultiAbort(ActionController controller, ExecuteEnv execEnv) { - super(controller); - this.execEnv = execEnv; - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - ExecuteEnv executeEnv = ExecuteEnv.getInstance(); - MultiAbort action = new MultiAbort(controller, executeEnv); - controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_abort", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - String db = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(db)) { - throw new DdlException("No database selected"); - } - String label = request.getSingleParameter(LABEL_KEY); - if (Strings.isNullOrEmpty(label)) { - throw new DdlException("No label selected"); - } - - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db); - checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); - - // only Master has these load info - if (redirectToMaster(request, response)) { - return; - } - - execEnv.getMultiLoadMgr().abort(fullDbName, label); - sendResult(request, response, RestBaseResult.getOk()); - } -} - diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiCommit.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiCommit.java deleted file mode 100644 index 956c73dc3957f0..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiCommit.java +++ /dev/null @@ -1,77 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.service.ExecuteEnv; - -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; - -public class MultiCommit extends RestBaseAction { - private ExecuteEnv execEnv; - - public MultiCommit(ActionController controller, ExecuteEnv execEnv) { - super(controller); - this.execEnv = execEnv; - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - ExecuteEnv executeEnv = ExecuteEnv.getInstance(); - MultiCommit action = new MultiCommit(controller, executeEnv); - controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_commit", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - String db = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(db)) { - throw new DdlException("No database selected"); - } - String label = request.getSingleParameter(LABEL_KEY); - if (Strings.isNullOrEmpty(label)) { - throw new DdlException("No label selected"); - } - - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db); - checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); - - // only Master has these load info - if (redirectToMaster(request, response)) { - return; - } - RestBaseResult result = new RestBaseResult(); - try { - execEnv.getMultiLoadMgr().commit(fullDbName, label); - } catch (Exception e) { - result.msg = e.getMessage(); - result.status = ActionStatus.FAILED; - } - sendResult(request, response, result); - } -} - diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiDesc.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiDesc.java deleted file mode 100644 index 5f023c7a42aabe..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiDesc.java +++ /dev/null @@ -1,85 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.service.ExecuteEnv; - -import com.google.common.base.Strings; -import com.google.common.collect.Lists; - -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; - -// List all labels of one multi-load -public class MultiDesc extends RestBaseAction { - private ExecuteEnv execEnv; - - public MultiDesc(ActionController controller, ExecuteEnv execEnv) { - super(controller); - this.execEnv = execEnv; - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - ExecuteEnv executeEnv = ExecuteEnv.getInstance(); - MultiDesc action = new MultiDesc(controller, executeEnv); - controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_desc", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - String db = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(db)) { - throw new DdlException("No database selected"); - } - String label = request.getSingleParameter(LABEL_KEY); - if (Strings.isNullOrEmpty(label)) { - throw new DdlException("No label selected"); - } - - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db); - checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); - - // only Master has these load info - if (redirectToMaster(request, response)) { - return; - } - - final List labels = Lists.newArrayList(); - execEnv.getMultiLoadMgr().desc(fullDbName, label, labels); - sendResult(request, response, new Result(labels)); - } - - private static class Result extends RestBaseResult { - private List labels; - - public Result(List labels) { - this.labels = labels; - } - } -} - diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiList.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiList.java deleted file mode 100644 index 407edca9c20fca..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiList.java +++ /dev/null @@ -1,82 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.service.ExecuteEnv; - -import com.google.common.base.Strings; -import com.google.common.collect.Lists; - -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; - -// list all multi load before commit -public class MultiList extends RestBaseAction { - - private ExecuteEnv execEnv; - - public MultiList(ActionController controller, ExecuteEnv execEnv) { - super(controller); - this.execEnv = execEnv; - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - ExecuteEnv executeEnv = ExecuteEnv.getInstance(); - MultiList action = new MultiList(controller, executeEnv); - controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_list", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - String db = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(db)) { - throw new DdlException("No database selected"); - } - - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db); - checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); - - // only Master has these load info - if (redirectToMaster(request, response)) { - return; - } - - final List labels = Lists.newArrayList(); - execEnv.getMultiLoadMgr().list(fullDbName, labels); - sendResult(request, response, new Result(labels)); - } - - private static class Result extends RestBaseResult { - private List labels; - - public Result(List labels) { - this.labels = labels; - } - } -} - diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiStart.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiStart.java deleted file mode 100644 index da8581dbd0de1c..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiStart.java +++ /dev/null @@ -1,92 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.analysis.LoadStmt; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.service.ExecuteEnv; - -import com.google.common.base.Strings; -import com.google.common.collect.Maps; - -import java.util.Map; - -import io.netty.handler.codec.http.HttpMethod; - -// Start multi action -public class MultiStart extends RestBaseAction { - private ExecuteEnv execEnv; - - public MultiStart(ActionController controller, ExecuteEnv execEnv) { - super(controller); - this.execEnv = execEnv; - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - ExecuteEnv executeEnv = ExecuteEnv.getInstance(); - MultiStart action = new MultiStart(controller, executeEnv); - controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_start", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - String db = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(db)) { - throw new DdlException("No database selected"); - } - String label = request.getSingleParameter(LABEL_KEY); - if (Strings.isNullOrEmpty(label)) { - throw new DdlException("No label selected"); - } - - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db); - checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); - - // Multi start request must redirect to master, because all following sub requests will be handled - // on Master - if (redirectToMaster(request, response)) { - return; - } - - Map properties = Maps.newHashMap(); - String[] keys = {LoadStmt.TIMEOUT_PROPERTY, LoadStmt.MAX_FILTER_RATIO_PROPERTY}; - for (String key : keys) { - String value = request.getSingleParameter(key); - if (!Strings.isNullOrEmpty(value)) { - properties.put(key, value); - } - } - for (String key : keys) { - String value = request.getRequest().headers().get(key); - if (!Strings.isNullOrEmpty(value)) { - properties.put(key, value); - } - } - execEnv.getMultiLoadMgr().startMulti(fullDbName, label, properties); - sendResult(request, response, RestBaseResult.getOk()); - } -} - diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiUnload.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiUnload.java deleted file mode 100644 index 8e5f89ad5cb088..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiUnload.java +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.service.ExecuteEnv; - -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; - -public class MultiUnload extends RestBaseAction { - private static final String SUB_LABEL_KEY = "sub_label"; - - private ExecuteEnv execEnv; - - public MultiUnload(ActionController controller, ExecuteEnv execEnv) { - super(controller); - this.execEnv = execEnv; - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - ExecuteEnv executeEnv = ExecuteEnv.getInstance(); - MultiUnload action = new MultiUnload(controller, executeEnv); - controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_unload", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - String db = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(db)) { - throw new DdlException("No database selected"); - } - String label = request.getSingleParameter(LABEL_KEY); - if (Strings.isNullOrEmpty(label)) { - throw new DdlException("No label selected"); - } - String subLabel = request.getSingleParameter(SUB_LABEL_KEY); - if (Strings.isNullOrEmpty(subLabel)) { - throw new DdlException("No sub_label selected"); - } - - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db); - checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); - - if (redirectToMaster(request, response)) { - return; - } - - execEnv.getMultiLoadMgr().unload(fullDbName, label, subLabel); - sendResult(request, response, RestBaseResult.getOk()); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ProfileAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/ProfileAction.java deleted file mode 100644 index 0bb35ed7fbfa8c..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ProfileAction.java +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.common.util.ProfileManager; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -// This class is a RESTFUL interface to get query profile. -// It will be used in query monitor to collect profiles. -// Usage: -// wget http://fe_host:fe_http_port/api/profile?query_id=123456 -public class ProfileAction extends RestBaseAction { - - public ProfileAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/profile", new ProfileAction(controller)); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - String queryId = request.getSingleParameter("query_id"); - if (queryId == null) { - response.getContent().append("not valid parameter"); - sendResult(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - String queryProfileStr = ProfileManager.getInstance().getProfile(queryId); - if (queryProfileStr != null) { - response.getContent().append(queryProfileStr); - sendResult(request, response); - } else { - response.getContent().append("query id " + queryId + " not found."); - sendResult(request, response, HttpResponseStatus.NOT_FOUND); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/QueryDetailAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/QueryDetailAction.java deleted file mode 100644 index c12a640e4a16e9..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/QueryDetailAction.java +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import com.google.gson.Gson; -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; -import java.util.List; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.qe.QueryDetail; -import org.apache.doris.qe.QueryDetailQueue; - -public class QueryDetailAction extends RestBaseAction { - - public QueryDetailAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/query_detail", new QueryDetailAction(controller)); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - String eventTimeStr = request.getSingleParameter("event_time"); - if (eventTimeStr == null) { - response.getContent().append("not valid parameter"); - sendResult(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - long eventTime = Long.valueOf(eventTimeStr.trim()); - List queryDetails = QueryDetailQueue.getQueryDetails(eventTime); - Gson gson = new Gson(); - String json_string = gson.toJson(queryDetails); - response.getContent().append(json_string); - sendResult(request, response); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseAction.java deleted file mode 100644 index 8fea6046b24e8f..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseAction.java +++ /dev/null @@ -1,127 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.analysis.UserIdentity; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseAction; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.UnauthorizedException; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.thrift.TNetworkAddress; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.net.URI; -import java.net.URISyntaxException; - -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpResponseStatus; - -public class RestBaseAction extends BaseAction { - protected static final String DB_KEY = "db"; - protected static final String TABLE_KEY = "table"; - protected static final String LABEL_KEY = "label"; - private static final Logger LOG = LogManager.getLogger(RestBaseAction.class); - - public RestBaseAction(ActionController controller) { - super(controller); - } - - @Override - public void handleRequest(BaseRequest request) throws Exception { - LOG.debug("receive http request. url={}", request.getRequest().uri()); - BaseResponse response = new BaseResponse(); - try { - execute(request, response); - } catch (DdlException e) { - if (e instanceof UnauthorizedException) { - response.appendContent(e.getMessage()); - response.updateHeader(HttpHeaderNames.WWW_AUTHENTICATE.toString(), "Basic realm=\"\""); - writeResponse(request, response, HttpResponseStatus.UNAUTHORIZED); - } else { - sendResult(request, response, new RestBaseResult(e.getMessage())); - } - } - } - - @Override - public void execute(BaseRequest request, BaseResponse response) throws DdlException { - ActionAuthorizationInfo authInfo = getAuthorizationInfo(request); - // check password - UserIdentity currentUser = checkPassword(authInfo); - ConnectContext ctx = new ConnectContext(null); - ctx.setCatalog(Catalog.getCurrentCatalog()); - ctx.setQualifiedUser(authInfo.fullUserName); - ctx.setRemoteIP(authInfo.remoteIp); - ctx.setCurrentUserIdentity(currentUser); - ctx.setCluster(authInfo.cluster); - ctx.setThreadLocalInfo(); - executeWithoutPassword(request, response); - } - - // If user password should be checked, the derived class should implement this method, NOT 'execute()', - // otherwise, override 'execute()' directly - protected void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - throw new DdlException("Not implemented"); - } - - public void sendResult(BaseRequest request, BaseResponse response, RestBaseResult result) { - response.appendContent(result.toJson()); - writeResponse(request, response, HttpResponseStatus.OK); - } - - public void sendResult(BaseRequest request, BaseResponse response, HttpResponseStatus status) { - writeResponse(request, response, status); - } - - public void sendResult(BaseRequest request, BaseResponse response) { - writeResponse(request, response, HttpResponseStatus.OK); - } - - public void redirectTo(BaseRequest request, BaseResponse response, TNetworkAddress addr) - throws DdlException { - String urlStr = request.getRequest().uri(); - URI urlObj = null; - URI resultUriObj = null; - try { - urlObj = new URI(urlStr); - resultUriObj = new URI("http", null, addr.getHostname(), - addr.getPort(), urlObj.getPath(), urlObj.getQuery(), null); - } catch (URISyntaxException e) { - LOG.warn(e.getMessage()); - throw new DdlException(e.getMessage()); - } - response.updateHeader(HttpHeaderNames.LOCATION.toString(), resultUriObj.toString()); - writeResponse(request, response, HttpResponseStatus.TEMPORARY_REDIRECT); - } - - public boolean redirectToMaster(BaseRequest request, BaseResponse response) throws DdlException { - Catalog catalog = Catalog.getCurrentCatalog(); - if (catalog.isMaster()) { - return false; - } - redirectTo(request, response, new TNetworkAddress(catalog.getMasterIp(), catalog.getMasterHttpPort())); - return true; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RowCountAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/RowCountAction.java deleted file mode 100644 index c343607148d557..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RowCountAction.java +++ /dev/null @@ -1,115 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.MaterializedIndex; -import org.apache.doris.catalog.MaterializedIndex.IndexExtState; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.Replica; -import org.apache.doris.catalog.Tablet; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.base.Strings; -import com.google.common.collect.Maps; - -import org.codehaus.jackson.map.ObjectMapper; - -import java.util.Map; - -import io.netty.handler.codec.http.HttpMethod; - -/* - * calc row count from replica to table - * fe_host:fe_http_port/api/rowcount?db=dbname&table=tablename - */ -public class RowCountAction extends RestBaseAction { - - public RowCountAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/rowcount", new RowCountAction(controller)); - } - - @Override - protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException { - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); - - String dbName = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(dbName)) { - throw new DdlException("No database selected."); - } - - String tableName = request.getSingleParameter(TABLE_KEY); - if (Strings.isNullOrEmpty(tableName)) { - throw new DdlException("No table selected."); - } - - Map indexRowCountMap = Maps.newHashMap(); - Catalog catalog = Catalog.getCurrentCatalog(); - Database db = catalog.getDbOrDdlException(dbName); - OlapTable olapTable = db.getOlapTableOrDdlException(tableName); - olapTable.writeLockOrDdlException(); - try { - for (Partition partition : olapTable.getAllPartitions()) { - long version = partition.getVisibleVersion(); - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { - long indexRowCount = 0L; - for (Tablet tablet : index.getTablets()) { - long tabletRowCount = 0L; - for (Replica replica : tablet.getReplicas()) { - if (replica.checkVersionCatchUp(version, false) - && replica.getRowCount() > tabletRowCount) { - tabletRowCount = replica.getRowCount(); - } - } - indexRowCount += tabletRowCount; - } // end for tablets - index.setRowCount(indexRowCount); - indexRowCountMap.put(olapTable.getIndexNameById(index.getId()), indexRowCount); - } // end for indices - } // end for partitions - } finally { - olapTable.writeUnlock(); - } - - // to json response - String result = ""; - ObjectMapper mapper = new ObjectMapper(); - try { - result = mapper.writeValueAsString(indexRowCountMap); - } catch (Exception e) { - // do nothing - } - - // send result - response.setContentType("application/json"); - response.getContent().append(result); - sendResult(request, response); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java deleted file mode 100644 index f096b10bf7da93..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java +++ /dev/null @@ -1,136 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import io.netty.handler.codec.http.HttpMethod; - -import org.apache.doris.common.ConfigBase; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.collect.Maps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.codehaus.jackson.map.ObjectMapper; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -/* - * used to set fe config - * eg: - * fe_host:http_port/api/_set_config?config_key1=config_value1&config_key2=config_value2&... - */ -public class SetConfigAction extends RestBaseAction { - private static final Logger LOG = LogManager.getLogger(SetConfigAction.class); - - private static final String PERSIST_PARAM = "persist"; - private static final String RESET_PERSIST = "reset_persist"; - - public SetConfigAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - SetConfigAction action = new SetConfigAction(controller); - controller.registerHandler(HttpMethod.GET, "/api/_set_config", action); - } - - @Override - protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException { - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); - - boolean needPersist = false; - boolean resetPersist = true; - Map> configs = request.getAllParameters(); - if (configs.containsKey(PERSIST_PARAM)) { - List val = configs.remove(PERSIST_PARAM); - if (val.size() == 1 && val.get(0).equals("true")) { - needPersist = true; - } - } - if (configs.containsKey(RESET_PERSIST)) { - List val = configs.remove(RESET_PERSIST); - if (val.size() == 1 && val.get(0).equals("false")) { - resetPersist = false; - } - } - - Map setConfigs = Maps.newHashMap(); - Map errConfigs = Maps.newHashMap(); - - LOG.debug("get config from url: {}, need persist: {}", configs, needPersist); - - for (Map.Entry> config : configs.entrySet()) { - String confKey = config.getKey(); - List confValue = config.getValue(); - try { - if (confValue != null && confValue.size() == 1) { - ConfigBase.setMutableConfig(confKey, confValue.get(0)); - setConfigs.put(confKey, confValue.get(0)); - } else { - throw new DdlException("conf value size != 1"); - } - } catch (DdlException e) { - LOG.warn("failed to set config {}:{}", confKey, confValue, e); - errConfigs.put(confKey, String.valueOf(confValue)); - } - } - - String persistMsg = ""; - if (needPersist) { - try { - ConfigBase.persistConfig(setConfigs, resetPersist); - persistMsg = "ok"; - } catch (IOException e) { - LOG.warn("failed to persist config", e); - persistMsg = e.getMessage(); - } - } - - Map resultMap = Maps.newHashMap(); - resultMap.put("set", setConfigs); - resultMap.put("err", errConfigs); - resultMap.put("persist", persistMsg); - - // to json response - String result = ""; - ObjectMapper mapper = new ObjectMapper(); - try { - result = mapper.writeValueAsString(resultMap); - } catch (Exception e) { - // do nothing - } - - // send result - response.setContentType("application/json"); - response.getContent().append(result); - sendResult(request, response); - } - - public static void print(String msg) { - System.out.println(System.currentTimeMillis() + " " + msg); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowDataAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowDataAction.java deleted file mode 100644 index af8036eb8b8c38..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowDataAction.java +++ /dev/null @@ -1,91 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Table; -import org.apache.doris.catalog.Table.TableType; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import io.netty.handler.codec.http.HttpMethod; -import java.util.concurrent.ConcurrentHashMap; -import java.util.List; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class ShowDataAction extends RestBaseAction { - private static final Logger LOG = LogManager.getLogger(ShowDataAction.class); - - public ShowDataAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/show_data", new ShowDataAction(controller)); - } - - public long getDataSizeOfDatabase(Database db) { - long totalSize = 0; - long tableSize = 0; - // sort by table name - List
    tables = db.getTables(); - for (Table table : tables) { - if (table.getType() != TableType.OLAP) { - continue; - } - table.readLock(); - try { - tableSize = ((OlapTable)table).getDataSize(); - } finally { - table.readUnlock(); - } - totalSize += tableSize; - } // end for tables - return totalSize; - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - String dbName = request.getSingleParameter("db"); - ConcurrentHashMap fullNameToDb = Catalog.getCurrentCatalog().getFullNameToDb(); - long totalSize = 0; - if (dbName != null) { - Database db = fullNameToDb.get("default_cluster:"+dbName); - if (db == null) { - response.getContent().append("database " + dbName + " not found."); - sendResult(request, response, HttpResponseStatus.NOT_FOUND); - return; - } - totalSize = getDataSizeOfDatabase(db); - } else { - for (Database db : fullNameToDb.values()) { - LOG.info("database name: {}", db.getFullName()); - totalSize += getDataSizeOfDatabase(db); - } - } - response.getContent().append(String.valueOf(totalSize)); - sendResult(request, response); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowMetaInfoAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowMetaInfoAction.java deleted file mode 100644 index e3bbef3c405724..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowMetaInfoAction.java +++ /dev/null @@ -1,203 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.MaterializedIndex; -import org.apache.doris.catalog.MaterializedIndex.IndexExtState; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.Replica; -import org.apache.doris.catalog.Replica.ReplicaState; -import org.apache.doris.catalog.Table; -import org.apache.doris.catalog.Table.TableType; -import org.apache.doris.catalog.Tablet; -import org.apache.doris.common.Config; -import org.apache.doris.ha.HAProtocol; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.persist.Storage; - -import com.google.gson.Gson; - -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import io.netty.handler.codec.http.HttpMethod; - -public class ShowMetaInfoAction extends RestBaseAction { - private enum Action { - SHOW_DB_SIZE, - SHOW_HA, - INVALID; - - public static Action getAction(String str) { - try { - return valueOf(str); - } catch (Exception ex) { - return INVALID; - } - } - } - - private static final Logger LOG = LogManager.getLogger(ShowMetaInfoAction.class); - - public ShowMetaInfoAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/show_meta_info", - new ShowMetaInfoAction(controller)); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - String action = request.getSingleParameter("action"); - Gson gson = new Gson(); - response.setContentType("application/json"); - - switch (Action.getAction(action.toUpperCase())) { - case SHOW_DB_SIZE: - response.getContent().append(gson.toJson(getDataSize())); - break; - case SHOW_HA: - response.getContent().append(gson.toJson(getHaInfo())); - break; - default: - break; - } - sendResult(request, response); - } - - public Map getHaInfo() { - HashMap feInfo = new HashMap(); - feInfo.put("role", Catalog.getCurrentCatalog().getFeType().toString()); - if (Catalog.getCurrentCatalog().isMaster()) { - feInfo.put("current_journal_id", - String.valueOf(Catalog.getCurrentCatalog().getEditLog().getMaxJournalId())); - } else { - feInfo.put("current_journal_id", - String.valueOf(Catalog.getCurrentCatalog().getReplayedJournalId())); - } - - HAProtocol haProtocol = Catalog.getCurrentCatalog().getHaProtocol(); - if (haProtocol != null) { - - InetSocketAddress master = null; - try { - master = haProtocol.getLeader(); - } catch (Exception e) { - // this may happen when majority of FOLLOWERS are down and no MASTER right now. - LOG.warn("failed to get leader: {}", e.getMessage()); - } - if (master != null) { - feInfo.put("master", master.getHostString()); - } else { - feInfo.put("master", "unknown"); - } - - List electableNodes = haProtocol.getElectableNodes(false); - ArrayList electableNodeNames = new ArrayList(); - if (!electableNodes.isEmpty()) { - for (InetSocketAddress node : electableNodes) { - electableNodeNames.add(node.getHostString()); - } - feInfo.put("electable_nodes", StringUtils.join(electableNodeNames.toArray(), ",")); - } - - List observerNodes = haProtocol.getObserverNodes(); - ArrayList observerNodeNames = new ArrayList(); - if (observerNodes != null) { - for (InetSocketAddress node : observerNodes) { - observerNodeNames.add(node.getHostString()); - } - feInfo.put("observer_nodes", StringUtils.join(observerNodeNames.toArray(), ",")); - } - } - - feInfo.put("can_read", String.valueOf(Catalog.getCurrentCatalog().canRead())); - feInfo.put("is_ready", String.valueOf(Catalog.getCurrentCatalog().isReady())); - try { - Storage storage = new Storage(Config.meta_dir + "/image"); - feInfo.put("last_checkpoint_version", String.valueOf(storage.getImageSeq())); - long lastCheckpointTime = storage.getCurrentImageFile().lastModified(); - feInfo.put("last_checkpoint_time", String.valueOf(lastCheckpointTime)); - } catch (IOException e) { - LOG.warn(e.getMessage()); - } - return feInfo; - } - - public Map getDataSize() { - Map result = new HashMap(); - List dbNames = Catalog.getCurrentCatalog().getDbNames(); - - for (String dbName : dbNames) { - Database db = Catalog.getCurrentCatalog().getDbNullable(dbName); - if (db == null) { - continue; - } - - long totalSize = 0; - List
    tables = db.getTables(); - for (Table table : tables) { - if (table.getType() != TableType.OLAP) { - continue; - } - - OlapTable olapTable = (OlapTable) table; - long tableSize = 0; - for (Partition partition : olapTable.getAllPartitions()) { - long partitionSize = 0; - for (MaterializedIndex mIndex : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { - long indexSize = 0; - for (Tablet tablet : mIndex.getTablets()) { - long maxReplicaSize = 0; - for (Replica replica : tablet.getReplicas()) { - if (replica.getState() == ReplicaState.NORMAL - || replica.getState() == ReplicaState.SCHEMA_CHANGE) { - if (replica.getDataSize() > maxReplicaSize) { - maxReplicaSize = replica.getDataSize(); - } - } - } // end for replicas - indexSize += maxReplicaSize; - } // end for tablets - partitionSize += indexSize; - } // end for tables - tableSize += partitionSize; - } // end for partitions - totalSize += tableSize; - } // end for tables - result.put(dbName, totalSize); - } // end for dbs - return result; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowProcAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowProcAction.java deleted file mode 100644 index bb53d9c7e7b3b5..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowProcAction.java +++ /dev/null @@ -1,136 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.analysis.RedirectStatus; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.proc.ProcNodeInterface; -import org.apache.doris.common.proc.ProcResult; -import org.apache.doris.common.proc.ProcService; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.MasterOpExecutor; -import org.apache.doris.qe.OriginStatement; -import org.apache.doris.qe.ShowResultSet; -import org.apache.doris.system.SystemInfoService; - -import com.google.common.base.Strings; -import com.google.gson.Gson; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; - -// Format: -// http://username:password@192.168.1.1:8030/api/show_proc?path=/ -public class ShowProcAction extends RestBaseAction { - private static final Logger LOG = LogManager.getLogger(ShowProcAction.class); - - public ShowProcAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/show_proc", new ShowProcAction(controller)); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException { - // check authority - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); - - String path = request.getSingleParameter("path"); - String forward = request.getSingleParameter("forward"); - boolean isForward = false; - if (!Strings.isNullOrEmpty(forward) && forward.equals("true")) { - isForward = true; - } - - // forward to master if necessary - if (!Catalog.getCurrentCatalog().isMaster() && isForward) { - String showProcStmt = "SHOW PROC \"" + path + "\""; - ConnectContext context = new ConnectContext(null); - context.setCatalog(Catalog.getCurrentCatalog()); - context.setCluster(SystemInfoService.DEFAULT_CLUSTER); - context.setQualifiedUser(ConnectContext.get().getQualifiedUser()); - context.setRemoteIP(ConnectContext.get().getRemoteIP()); - MasterOpExecutor masterOpExecutor = new MasterOpExecutor(new OriginStatement(showProcStmt, 0), context, - RedirectStatus.FORWARD_NO_SYNC, true); - LOG.debug("need to transfer to Master. stmt: {}", context.getStmtId()); - - try { - masterOpExecutor.execute(); - } catch (Exception e) { - response.appendContent("Failed to forward stmt: " + e.getMessage()); - sendResult(request, response); - return; - } - - ShowResultSet resultSet = masterOpExecutor.getProxyResultSet(); - if (resultSet == null) { - response.appendContent("Failed to get result set"); - sendResult(request, response); - return; - } - - Gson gson = new Gson(); - response.setContentType("application/json"); - response.getContent().append(gson.toJson(resultSet.getResultRows())); - - } else { - ProcNodeInterface procNode = null; - ProcService instance = ProcService.getInstance(); - try { - if (Strings.isNullOrEmpty(path)) { - procNode = instance.open("/"); - } else { - procNode = instance.open(path); - } - } catch (AnalysisException e) { - LOG.warn(e.getMessage()); - response.getContent().append("[]"); - } - - if (procNode != null) { - ProcResult result; - try { - result = procNode.fetchResult(); - List> rows = result.getRows(); - - Gson gson = new Gson(); - response.setContentType("application/json"); - response.getContent().append(gson.toJson(rows)); - } catch (AnalysisException e) { - LOG.warn(e.getMessage()); - response.getContent().append("[]"); - } - } - } - - sendResult(request, response); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowRuntimeInfoAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowRuntimeInfoAction.java deleted file mode 100644 index 66ec5c4c3500da..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowRuntimeInfoAction.java +++ /dev/null @@ -1,69 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import com.google.gson.Gson; - -import io.netty.handler.codec.http.HttpMethod; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.HashMap; - -public class ShowRuntimeInfoAction extends RestBaseAction { - private static final Logger LOG = LogManager.getLogger(ShowRuntimeInfoAction.class); - - public ShowRuntimeInfoAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/show_runtime_info", - new ShowRuntimeInfoAction(controller)); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - HashMap feInfo = new HashMap(); - - // Get memory info - Runtime r = Runtime.getRuntime(); - feInfo.put("free_mem", String.valueOf(r.freeMemory())); - feInfo.put("total_mem", String.valueOf(r.totalMemory())); - feInfo.put("max_mem", String.valueOf(r.maxMemory())); - - // Get thread count - ThreadGroup parentThread; - for (parentThread = Thread.currentThread().getThreadGroup(); - parentThread.getParent() != null; - parentThread = parentThread.getParent()) { - }; - feInfo.put("thread_cnt", String.valueOf(parentThread.activeCount())); - - Gson gson = new Gson(); - response.setContentType("application/json"); - response.getContent().append(gson.toJson(feInfo)); - - sendResult(request, response); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/StorageTypeCheckAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/StorageTypeCheckAction.java deleted file mode 100644 index bf6d271ebe2443..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/StorageTypeCheckAction.java +++ /dev/null @@ -1,98 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.MaterializedIndexMeta; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Table; -import org.apache.doris.catalog.Table.TableType; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.thrift.TStorageType; - -import com.google.common.base.Strings; - -import org.json.simple.JSONObject; - -import java.util.List; -import java.util.Map; - -import io.netty.handler.codec.http.HttpMethod; - -public class StorageTypeCheckAction extends RestBaseAction { - public StorageTypeCheckAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - StorageTypeCheckAction action = new StorageTypeCheckAction(controller); - controller.registerHandler(HttpMethod.GET, "/api/_check_storagetype", action); - } - - @Override - protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException { - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); - - String dbName = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(dbName)) { - throw new DdlException("Parameter db is missing"); - } - - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), dbName); - Database db = catalog.getDbOrDdlException(fullDbName); - - JSONObject root = new JSONObject(); - List
    tableList = db.getTables(); - - for (Table tbl : tableList) { - if (tbl.getType() != TableType.OLAP) { - continue; - } - - OlapTable olapTbl = (OlapTable) tbl; - olapTbl.readLock(); - try { - JSONObject indexObj = new JSONObject(); - for (Map.Entry entry : olapTbl.getIndexIdToMeta().entrySet()) { - MaterializedIndexMeta indexMeta = entry.getValue(); - if (indexMeta.getStorageType() == TStorageType.ROW) { - indexObj.put(olapTbl.getIndexNameById(entry.getKey()), indexMeta.getStorageType().name()); - } - } - root.put(tbl.getName(), indexObj); - } finally { - olapTbl.readUnlock(); - } - } - - // to json response - String result = root.toString(); - - // send result - response.setContentType("application/json"); - response.getContent().append(result); - sendResult(request, response); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java deleted file mode 100644 index d252ba36dbcbfe..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java +++ /dev/null @@ -1,306 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.analysis.InlineViewRef; -import org.apache.doris.analysis.SelectStmt; -import org.apache.doris.analysis.StatementBase; -import org.apache.doris.analysis.TableName; -import org.apache.doris.analysis.TableRef; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.Table; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.DorisHttpException; -import org.apache.doris.common.MetaNotFoundException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.planner.PlanFragment; -import org.apache.doris.planner.Planner; -import org.apache.doris.planner.ScanNode; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.OriginStatement; -import org.apache.doris.qe.StmtExecutor; -import org.apache.doris.thrift.TDataSink; -import org.apache.doris.thrift.TDataSinkType; -import org.apache.doris.thrift.TMemoryScratchSink; -import org.apache.doris.thrift.TNetworkAddress; -import org.apache.doris.thrift.TPaloScanRange; -import org.apache.doris.thrift.TPlanFragment; -import org.apache.doris.thrift.TQueryOptions; -import org.apache.doris.thrift.TQueryPlanInfo; -import org.apache.doris.thrift.TScanRangeLocations; -import org.apache.doris.thrift.TTabletVersionInfo; -import org.apache.doris.thrift.TUniqueId; - -import com.google.common.base.Strings; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.thrift.TException; -import org.apache.thrift.TSerializer; -import org.codehaus.jackson.map.ObjectMapper; -import org.json.simple.JSONObject; -import org.json.simple.JSONValue; - -import java.util.ArrayList; -import java.util.Base64; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -/** - * This class responsible for parse the sql and generate the query plan fragment for a (only one) table{@see OlapTable} - * the related tablet maybe pruned by query planer according the `where` predicate. - */ -public class TableQueryPlanAction extends RestBaseAction { - - public static final Logger LOG = LogManager.getLogger(TableQueryPlanAction.class); - - public TableQueryPlanAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.POST, - "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_query_plan", - new TableQueryPlanAction(controller)); - controller.registerHandler(HttpMethod.GET, - "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_query_plan", - new TableQueryPlanAction(controller)); - } - - @Override - protected void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - // just allocate 2 slot for top holder map - Map resultMap = new HashMap<>(4); - String dbName = request.getSingleParameter(DB_KEY); - String tableName = request.getSingleParameter(TABLE_KEY); - String postContent = request.getContent(); - try { - // may be these common validate logic should be moved to one base class - if (Strings.isNullOrEmpty(dbName) - || Strings.isNullOrEmpty(tableName)) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "{database}/{table} must be selected"); - } - if (Strings.isNullOrEmpty(postContent)) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "POST body must contains [sql] root object"); - } - JSONObject jsonObject = (JSONObject) JSONValue.parse(postContent); - if (jsonObject == null) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "malformed json [ " + postContent + " ]"); - } - String sql = (String) jsonObject.get("sql"); - if (Strings.isNullOrEmpty(sql)) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "POST body must contains [sql] root object"); - } - LOG.info("receive SQL statement [{}] from external service [ user [{}]] for database [{}] table [{}]", - sql, ConnectContext.get().getCurrentUserIdentity(), dbName, tableName); - - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), dbName); - // check privilege for select, otherwise return HTTP 401 - checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tableName, PrivPredicate.SELECT); - Table table; - try { - Database db = Catalog.getCurrentCatalog().getDbOrMetaException(fullDbName); - table = db.getTableOrMetaException(tableName, Table.TableType.OLAP); - } catch (MetaNotFoundException e) { - throw new DorisHttpException(HttpResponseStatus.FORBIDDEN, e.getMessage()); - } - - // may be should acquire writeLock - table.readLock(); - try { - // parse/analysis/plan the sql and acquire tablet distributions - handleQuery(ConnectContext.get(), fullDbName, tableName, sql, resultMap); - } finally { - table.readUnlock(); - } - } catch (DorisHttpException e) { - // status code should conforms to HTTP semantic - resultMap.put("status", e.getCode().code()); - resultMap.put("exception", e.getMessage()); - } - ObjectMapper mapper = new ObjectMapper(); - try { - String result = mapper.writeValueAsString(resultMap); - // send result with extra information - response.setContentType("application/json"); - response.getContent().append(result); - sendResult(request, response, HttpResponseStatus.valueOf(Integer.parseInt(String.valueOf(resultMap.get("status"))))); - } catch (Exception e) { - // may be this never happen - response.getContent().append(e.getMessage()); - sendResult(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR); - } - } - - - /** - * process the sql syntax and return the resolved pruned tablet - * - * @param context context for analyzer - * @param sql the single table select statement - * @param result the acquired results - * @return - * @throws DorisHttpException - */ - private void handleQuery(ConnectContext context, String requestDb, String requestTable, String sql, - Map result) throws DorisHttpException { - // use SE to resolve sql - StmtExecutor stmtExecutor = new StmtExecutor(context, new OriginStatement(sql, 0), false); - try { - TQueryOptions tQueryOptions = context.getSessionVariable().toThrift(); - // Conduct Planner create SingleNodePlan#createPlanFragments - tQueryOptions.num_nodes = 1; - // analyze sql - stmtExecutor.analyze(tQueryOptions); - } catch (Exception e) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, e.getMessage()); - } - // the parsed logical statement - StatementBase query = stmtExecutor.getParsedStmt(); - // only process select semantic - if (!(query instanceof SelectStmt)) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "Select statement needed, but found [" + sql + " ]"); - } - SelectStmt stmt = (SelectStmt) query; - // just only process sql like `select * from table where `, only support executing scan semantic - if (stmt.hasAggInfo() || stmt.hasAnalyticInfo() - || stmt.hasOrderByClause() || stmt.hasOffset() || stmt.hasLimit() || stmt.isExplain()) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "only support single table filter-prune-scan, but found [ " + sql + "]"); - } - // process only one table by one http query - List fromTables = stmt.getTableRefs(); - if (fromTables.size() != 1) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "Select statement must have only one table"); - } - - TableRef fromTable = fromTables.get(0); - if (fromTable instanceof InlineViewRef) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "Select statement must not embed another statement"); - } - // check consistent http requested resource with sql referenced - // if consistent in this way, can avoid check privilege - TableName tableAndDb = fromTables.get(0).getName(); - if (!(tableAndDb.getDb().equals(requestDb) && tableAndDb.getTbl().equals(requestTable))) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "requested database and table must consistent with sql: request [ " - + requestDb + "." + requestTable + "]" + "and sql [" + tableAndDb.toString() + "]"); - } - - // acquired Planner to get PlanNode and fragment templates - Planner planner = stmtExecutor.planner(); - // acquire ScanNode to obtain pruned tablet - // in this way, just retrieve only one scannode - List scanNodes = planner.getScanNodes(); - if (scanNodes.size() != 1) { - throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Planner should plan just only one ScanNode but found [ " + scanNodes.size() + "]"); - } - List scanRangeLocations = scanNodes.get(0).getScanRangeLocations(0); - // acquire the PlanFragment which the executable template - List fragments = planner.getFragments(); - if (fragments.size() != 1) { - throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Planner should plan just only one PlanFragment but found [ " + fragments.size() + "]"); - } - - TQueryPlanInfo tQueryPlanInfo = new TQueryPlanInfo(); - - - // acquire TPlanFragment - TPlanFragment tPlanFragment = fragments.get(0).toThrift(); - // set up TMemoryScratchSink - TDataSink tDataSink = new TDataSink(); - tDataSink.type = TDataSinkType.MEMORY_SCRATCH_SINK; - tDataSink.memory_scratch_sink = new TMemoryScratchSink(); - tPlanFragment.output_sink = tDataSink; - - tQueryPlanInfo.plan_fragment = tPlanFragment; - tQueryPlanInfo.desc_tbl = query.getAnalyzer().getDescTbl().toThrift(); - // set query_id - UUID uuid = UUID.randomUUID(); - tQueryPlanInfo.query_id = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()); - - Map tablet_info = new HashMap<>(); - // acquire resolved tablet distribution - Map tabletRoutings = assemblePrunedPartitions(scanRangeLocations); - tabletRoutings.forEach((tabletId, node) -> { - long tablet = Long.parseLong(tabletId); - tablet_info.put(tablet, new TTabletVersionInfo(tablet, node.version, 0l /*versionHash*/, node.schemaHash)); - }); - tQueryPlanInfo.tablet_info = tablet_info; - - // serialize TQueryPlanInfo and encode plan with Base64 to string in order to translate by json format - TSerializer serializer = new TSerializer(); - String opaqued_query_plan; - try { - byte[] query_plan_stream = serializer.serialize(tQueryPlanInfo); - opaqued_query_plan = Base64.getEncoder().encodeToString(query_plan_stream); - } catch (TException e) { - throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "TSerializer failed to serialize PlanFragment, reason [ " + e.getMessage() + " ]"); - } - result.put("partitions", tabletRoutings); - result.put("opaqued_query_plan", opaqued_query_plan); - result.put("status", 200); - } - - /** - * acquire all involved (already pruned) tablet routing - * - * @param scanRangeLocationsList - * @return - */ - private Map assemblePrunedPartitions(List scanRangeLocationsList) { - Map result = new HashMap<>(); - for (TScanRangeLocations scanRangeLocations : scanRangeLocationsList) { - // only process palo(doris) scan range - TPaloScanRange scanRange = scanRangeLocations.scan_range.palo_scan_range; - Node tabletRouting = new Node(Long.parseLong(scanRange.version), Integer.parseInt(scanRange.schema_hash)); - for (TNetworkAddress address : scanRange.hosts) { - tabletRouting.addRouting(address.hostname + ":" + address.port); - } - result.put(String.valueOf(scanRange.tablet_id), tabletRouting); - } - return result; - } - - // helper class for json transformation - final class Node { - // ["host1:port1", "host2:port2", "host3:port3"] - public List routings = new ArrayList<>(); - public long version; - public int schemaHash; - - public Node(long version, int schemaHash) { - this.version = version; - this.schemaHash = schemaHash; - } - - private void addRouting(String routing) { - routings.add(routing); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableRowCountAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableRowCountAction.java deleted file mode 100644 index 1b4a8edece5329..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableRowCountAction.java +++ /dev/null @@ -1,113 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Table; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.DorisHttpException; -import org.apache.doris.common.MetaNotFoundException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.base.Strings; - -import org.codehaus.jackson.map.ObjectMapper; - -import java.util.HashMap; -import java.util.Map; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -/** - * This class is responsible for fetch the approximate row count of the specified table from cluster-meta data, - * the approximate row maybe used for some computing system to decide use which compute-algorithm can be used - * such as shuffle join or broadcast join. - *

    - * This API is not intended to compute the exact row count of the specified table, if you need the exact row count, - * please consider using the sql syntax `select count(*) from {table}` - */ -public class TableRowCountAction extends RestBaseAction { - public TableRowCountAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, - "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_count", - new TableRowCountAction(controller)); - } - - @Override - protected void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - // just allocate 2 slot for top holder map - Map resultMap = new HashMap<>(4); - String dbName = request.getSingleParameter(DB_KEY); - String tableName = request.getSingleParameter(TABLE_KEY); - try { - if (Strings.isNullOrEmpty(dbName) - || Strings.isNullOrEmpty(tableName)) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "{database}/{table} must be selected"); - } - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), dbName); - // check privilege for select, otherwise return HTTP 401 - checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tableName, PrivPredicate.SELECT); - Table table; - try { - Database db = Catalog.getCurrentCatalog().getDbOrMetaException(fullDbName); - table = db.getTableOrMetaException(tableName, Table.TableType.OLAP); - } catch (MetaNotFoundException e) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, e.getMessage()); - } - - table.readLock(); - try { - OlapTable olapTable = (OlapTable) table; - resultMap.put("status", 200); - resultMap.put("size", olapTable.proximateRowCount()); - } finally { - table.readUnlock(); - } - } catch (DorisHttpException e) { - // status code should conforms to HTTP semantic - resultMap.put("status", e.getCode().code()); - resultMap.put("exception", e.getMessage()); - } - ObjectMapper mapper = new ObjectMapper(); - try { - String result = mapper.writeValueAsString(resultMap); - // send result with extra information - response.setContentType("application/json"); - response.getContent().append(result); - sendResult(request, response, HttpResponseStatus.valueOf(Integer.parseInt(String.valueOf(resultMap.get("status"))))); - } catch (Exception e) { - // may be this never happen - response.getContent().append(e.getMessage()); - sendResult(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableSchemaAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableSchemaAction.java deleted file mode 100644 index 9000ae4890f9c4..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableSchemaAction.java +++ /dev/null @@ -1,135 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.PrimitiveType; -import org.apache.doris.catalog.ScalarType; -import org.apache.doris.catalog.Table; -import org.apache.doris.catalog.Type; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.DorisHttpException; -import org.apache.doris.common.MetaNotFoundException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.base.Strings; - -import org.codehaus.jackson.map.ObjectMapper; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -/** - * Get table schema for specified cluster.database.table with privilege checking - */ -public class TableSchemaAction extends RestBaseAction { - - public TableSchemaAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - // the extra `/api` path is so disgusting - controller.registerHandler(HttpMethod.GET, - "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_schema", new TableSchemaAction - (controller)); - } - - @Override - protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException { - // just allocate 2 slot for top holder map - Map resultMap = new HashMap<>(2); - String dbName = request.getSingleParameter(DB_KEY); - String tableName = request.getSingleParameter(TABLE_KEY); - try { - if (Strings.isNullOrEmpty(dbName) - || Strings.isNullOrEmpty(tableName)) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "No database or table selected."); - } - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), dbName); - // check privilege for select, otherwise return 401 HTTP status - checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tableName, PrivPredicate.SELECT); - Table table; - try { - Database db = Catalog.getCurrentCatalog().getDbOrMetaException(fullDbName); - table = db.getTableOrMetaException(tableName, Table.TableType.OLAP); - } catch (MetaNotFoundException e) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, e.getMessage()); - } - - table.readLock(); - try { - List columns = table.getBaseSchema(); - List> propList = new ArrayList(columns.size()); - for (Column column : columns) { - Map baseInfo = new HashMap<>(2); - Type colType = column.getOriginType(); - PrimitiveType primitiveType = colType.getPrimitiveType(); - if (primitiveType == PrimitiveType.DECIMALV2) { - ScalarType scalarType = (ScalarType) colType; - baseInfo.put("precision", scalarType.getPrecision() + ""); - baseInfo.put("scale", scalarType.getScalarScale() + ""); - } - baseInfo.put("type", primitiveType.toString()); - baseInfo.put("comment", column.getComment()); - baseInfo.put("name", column.getDisplayName()); - Optional aggregationType = Optional.ofNullable(column.getAggregationType()); - baseInfo.put("aggregation_type", aggregationType.isPresent() ? column.getAggregationType().toSql() : ""); - propList.add(baseInfo); - } - resultMap.put("status", 200); - resultMap.put("properties", propList); - } catch (Exception e) { - // Transform the general Exception to custom DorisHttpException - throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, e.getMessage() == null ? "Null Pointer Exception" : e.getMessage()); - } finally { - table.readUnlock(); - } - } catch (DorisHttpException e) { - // status code should conforms to HTTP semantic - resultMap.put("status", e.getCode().code()); - resultMap.put("exception", e.getMessage()); - } - ObjectMapper mapper = new ObjectMapper(); - try { - String result = mapper.writeValueAsString(resultMap); - // send result with extra information - response.setContentType("application/json"); - response.getContent().append(result); - sendResult(request, response, HttpResponseStatus.valueOf(Integer.parseInt(String.valueOf(resultMap.get("status"))))); - } catch (Exception e) { - // may be this never happen - response.getContent().append(e.getMessage()); - sendResult(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/IllegalArgException.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/IllegalArgException.java similarity index 72% rename from fe/fe-core/src/main/java/org/apache/doris/http/IllegalArgException.java rename to fe/fe-core/src/main/java/org/apache/doris/httpv2/IllegalArgException.java index 9e82202d8344a5..b77cf0844940c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/http/IllegalArgException.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/IllegalArgException.java @@ -14,21 +14,20 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - -package org.apache.doris.http; +package org.apache.doris.httpv2; public class IllegalArgException extends Exception { - private static final long serialVersionUID = 3344697787301861667L; + private static final long serialVersionUID = 3344697787301861667L; - public IllegalArgException() { - super(""); - } + public IllegalArgException() { + super(""); + } - public IllegalArgException(String msg) { - super(msg); - } + public IllegalArgException(String msg) { + super(msg); + } - public IllegalArgException(String msg, Throwable cause) { - super(msg, cause); - } -} + public IllegalArgException(String msg, Throwable cause) { + super(msg, cause); + } +} \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ActionStatus.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ActionStatus.java similarity index 93% rename from fe/fe-core/src/main/java/org/apache/doris/http/rest/ActionStatus.java rename to fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ActionStatus.java index a72b94cd946adb..a713958c773fb1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ActionStatus.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ActionStatus.java @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.http.rest; +package org.apache.doris.httpv2.entity; // Status of RESTful action public enum ActionStatus { - OK, - FAILED + OK, + FAILED } diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseResult.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestBaseResult.java similarity index 62% rename from fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseResult.java rename to fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestBaseResult.java index 63cd07cc02f6e6..e9f96ec946454a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestBaseResult.java @@ -14,34 +14,34 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - -package org.apache.doris.http.rest; +package org.apache.doris.httpv2.entity; import com.google.gson.Gson; import com.google.gson.GsonBuilder; // Base restful result public class RestBaseResult { - private static final RestBaseResult OK = new RestBaseResult(); - public ActionStatus status; - public String msg; - public RestBaseResult() { - status = ActionStatus.OK; - msg = "Success"; - } + private static final RestBaseResult OK = new RestBaseResult(); + public ActionStatus status; + public String msg; + + public RestBaseResult() { + status = ActionStatus.OK; + msg = "Success"; + } - public RestBaseResult(String msg) { - status = ActionStatus.FAILED; - this.msg = msg; - } + public RestBaseResult(String msg) { + status = ActionStatus.FAILED; + this.msg = msg; + } - public static RestBaseResult getOk() { - return OK; - } + public static RestBaseResult getOk() { + return OK; + } - public String toJson() { - Gson gson = new GsonBuilder().disableHtmlEscaping().create(); - return gson.toJson(this); - } + public String toJson() { + Gson gson = new GsonBuilder().disableHtmlEscaping().create(); + return gson.toJson(this); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestResult.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestResult.java similarity index 65% rename from fe/fe-core/src/main/java/org/apache/doris/http/rest/RestResult.java rename to fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestResult.java index d8819848c151ae..ac28668f5e77e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestResult.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.http.rest; +package org.apache.doris.httpv2.entity; import java.util.Map; @@ -23,23 +23,24 @@ import com.google.gson.Gson; public class RestResult extends RestBaseResult { - private Map resultMap; - public RestResult() { - super(); - resultMap = Maps.newHashMap(); - } + private Map resultMap; - public void addResultEntry(String key, Object value) { - resultMap.put(key, value); - } + public RestResult() { + super(); + resultMap = Maps.newHashMap(); + } + + public void addResultEntry(String key, Object value) { + resultMap.put(key, value); + } - public String toJson() { - Gson gson = new Gson(); - addResultEntry("status", status); - if (status != ActionStatus.OK) { - addResultEntry("msg", msg); - } - return gson.toJson(resultMap); + public String toJson() { + Gson gson = new Gson(); + addResultEntry("status", status); + if (status != ActionStatus.OK) { + addResultEntry("msg", msg); } + return gson.toJson(resultMap); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java index d79cd5b9976214..2213034cfe00df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java @@ -20,7 +20,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; -import org.apache.doris.http.rest.RestBaseResult; +import org.apache.doris.httpv2.entity.RestBaseResult; import org.apache.doris.load.Load; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java index 8cacb0832a8ea4..d5b2b71e589637 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java @@ -20,7 +20,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.DdlException; -import org.apache.doris.http.rest.RestBaseResult; +import org.apache.doris.httpv2.entity.RestBaseResult; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; import org.apache.doris.service.ExecuteEnv; diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java index 943aa26bac3c54..86983901d8df76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java @@ -19,7 +19,7 @@ import org.apache.doris.analysis.LoadStmt; import org.apache.doris.common.DdlException; -import org.apache.doris.http.rest.RestBaseResult; +import org.apache.doris.httpv2.entity.RestBaseResult; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; import org.apache.doris.service.ExecuteEnv; diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java index 1dc7e77be780c2..35533433d4d615 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java @@ -20,8 +20,8 @@ import org.apache.doris.common.Config; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.ThreadPoolManager; -import org.apache.doris.http.HttpServer; -import org.apache.doris.http.IllegalArgException; +import org.apache.doris.httpv2.HttpServer; +import org.apache.doris.httpv2.IllegalArgException; import org.apache.doris.journal.JournalEntity; import org.apache.doris.meta.MetaContext; import org.apache.doris.qe.QeService; @@ -86,20 +86,10 @@ public void startDebugMode(String dorisHomeDir) { // Only start MySQL and HttpServer private void startService(String dorisHomeDir) throws IllegalArgException, IOException { // HTTP server - if (!Config.enable_http_server_v2) { - HttpServer httpServer = new HttpServer( - Config.http_port, - Config.http_max_line_length, - Config.http_max_header_size, - Config.http_max_chunk_size - ); - httpServer.setup(); - httpServer.start(); - } else { - org.apache.doris.httpv2.HttpServer httpServer2 = new org.apache.doris.httpv2.HttpServer(); - httpServer2.setPort(Config.http_port); - httpServer2.start(dorisHomeDir); - } + + HttpServer httpServer = new HttpServer(); + httpServer.setPort(Config.http_port); + httpServer.start(dorisHomeDir); // MySQl server QeService qeService = new QeService(Config.query_port, Config.mysql_service_nio_enabled, ExecuteEnv.getInstance().getScheduler()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/path/PathTrieTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/path/PathTrieTest.java index 66531d6ed3e413..afeef0db1fc297 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/path/PathTrieTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/path/PathTrieTest.java @@ -19,11 +19,6 @@ import static com.google.common.collect.Maps.newHashMap; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.rest.MultiStart; -import org.apache.doris.http.rest.TableQueryPlanAction; -import org.apache.doris.http.rest.TableRowCountAction; -import org.apache.doris.http.rest.TableSchemaAction; import org.apache.doris.service.ExecuteEnv; import com.google.common.collect.Maps; @@ -165,18 +160,4 @@ public void testNamedWildcardAndLookupWithWildcard() { Assert.assertEquals(params.get("test"), "*"); } - @Test - public void testInsert(@Injectable ActionController controller, - @Injectable ExecuteEnv execEnv) { - PathTrie pathTrie = new PathTrie(); - pathTrie.insert("/api/{db}/_multi_start", new MultiStart(controller, execEnv)); - pathTrie.insert("/api/{db}/{table}/_count", new TableRowCountAction(controller)); - pathTrie.insert("/api/{db}/{table}/_schema", new TableSchemaAction(controller)); - pathTrie.insert("/api/{db}/{table}/_query_plan", new TableQueryPlanAction(controller)); - Map params = Maps.newHashMap(); - pathTrie.retrieve("/api/test/_multi_start", params); - Assert.assertEquals("test", params.get("db")); - - - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java b/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java index 8f2a1c444ce627..71f4adaf247277 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java +++ b/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java @@ -38,9 +38,12 @@ import org.apache.doris.catalog.TabletInvertedIndex; import org.apache.doris.catalog.TabletMeta; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.ExceptionChecker.ThrowingRunnable; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.httpv2.HttpServer; +import org.apache.doris.httpv2.IllegalArgException; import org.apache.doris.load.Load; import org.apache.doris.mysql.privilege.PaloAuth; import org.apache.doris.persist.EditLog; @@ -110,6 +113,8 @@ abstract public class DorisHttpTestCase { protected String rootAuth = Credentials.basic("root", ""); + public static final String DORIS_HOME_DIR = System.getenv("DORIS_HOME"); + @Mocked private static EditLog editLog; @@ -291,13 +296,13 @@ public static void initHttpServer() throws IllegalArgException, InterruptedExcep } } - httpServer = new HttpServer(HTTP_PORT); - httpServer.setup(); - httpServer.start(); - // must ensure the http server started before any unit test - while (!httpServer.isStarted()) { - Thread.sleep(500); - } + httpServer = new HttpServer(); + httpServer.setPort(Config.http_port); + httpServer.setMaxHttpPostSize(Config.jetty_server_max_http_post_size); + httpServer.setAcceptors(Config.jetty_server_acceptors); + httpServer.setSelectors(Config.jetty_server_selectors); + httpServer.setWorkers(Config.jetty_server_workers); + httpServer.start(DORIS_HOME_DIR); } @@ -337,10 +342,6 @@ TabletInvertedIndex getCurrentInvertedIndex() { public void tearDown() { } - @AfterClass - public static void closeHttpServer() { - httpServer.shutDown(); - } public void doSetUp() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/http/HttpAuthManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/http/HttpAuthManagerTest.java index a6ed325d2ba5d7..2cd23dc34d0b37 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/http/HttpAuthManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/http/HttpAuthManagerTest.java @@ -17,9 +17,12 @@ package org.apache.doris.http; +import java.util.ArrayList; +import java.util.List; import org.apache.doris.analysis.UserIdentity; -import org.apache.doris.http.HttpAuthManager.SessionValue; +import org.apache.doris.httpv2.HttpAuthManager.SessionValue; +import org.apache.doris.httpv2.HttpAuthManager; import org.junit.Assert; import org.junit.Test; @@ -31,15 +34,20 @@ public void testNormal() { HttpAuthManager authMgr = HttpAuthManager.getInstance(); String sessionId = "test_session_id"; String username = "test-user"; - SessionValue sessionValue = new SessionValue(); + HttpAuthManager.SessionValue sessionValue = new HttpAuthManager.SessionValue(); sessionValue.currentUser = UserIdentity.createAnalyzedUserIdentWithIp(username, "%"); authMgr.addSessionValue(sessionId, sessionValue); Assert.assertEquals(1, authMgr.getAuthSessions().size()); - System.out.println("username in test: " + authMgr.getSessionValue(sessionId).currentUser); - Assert.assertEquals(username, authMgr.getSessionValue(sessionId).currentUser.getQualifiedUser()); + List sessionIds = new ArrayList<>(); + sessionIds.add (sessionId); + System.out.println("username in test: " + authMgr.getSessionValue(sessionIds).currentUser); + Assert.assertEquals(username, authMgr.getSessionValue(sessionIds).currentUser.getQualifiedUser()); + String noExistSession = "no-exist-session-id"; - Assert.assertNull(authMgr.getSessionValue(noExistSession)); + sessionIds.clear(); + sessionIds.add (noExistSession); + Assert.assertNull(authMgr.getSessionValue(sessionIds)); Assert.assertEquals(1, authMgr.getAuthSessions().size()); } } From c6cda72d6cbb07550991642e98e4fc6f8b0e3d20 Mon Sep 17 00:00:00 2001 From: "jiafeng.zhang" Date: Wed, 6 Apr 2022 09:58:21 +0800 Subject: [PATCH 2/3] code style code style --- .../doris/httpv2/entity/ActionStatus.java | 4 +-- .../doris/httpv2/entity/RestBaseResult.java | 32 ++++++++--------- .../doris/httpv2/entity/RestResult.java | 36 +++++++++---------- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ActionStatus.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ActionStatus.java index a713958c773fb1..23cd6f7fbca1d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ActionStatus.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ActionStatus.java @@ -19,6 +19,6 @@ // Status of RESTful action public enum ActionStatus { - OK, - FAILED + OK, + FAILED } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestBaseResult.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestBaseResult.java index e9f96ec946454a..973d5951b333ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestBaseResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestBaseResult.java @@ -22,26 +22,26 @@ // Base restful result public class RestBaseResult { - private static final RestBaseResult OK = new RestBaseResult(); - public ActionStatus status; - public String msg; + private static final RestBaseResult OK = new RestBaseResult(); + public ActionStatus status; + public String msg; - public RestBaseResult() { - status = ActionStatus.OK; - msg = "Success"; - } + public RestBaseResult() { + status = ActionStatus.OK; + msg = "Success"; + } - public RestBaseResult(String msg) { - status = ActionStatus.FAILED; - this.msg = msg; - } + public RestBaseResult(String msg) { + status = ActionStatus.FAILED; + this.msg = msg; + } - public static RestBaseResult getOk() { + public static RestBaseResult getOk() { return OK; } - public String toJson() { - Gson gson = new GsonBuilder().disableHtmlEscaping().create(); - return gson.toJson(this); - } + public String toJson() { + Gson gson = new GsonBuilder().disableHtmlEscaping().create(); + return gson.toJson(this); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestResult.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestResult.java index ac28668f5e77e7..3dda63e4ca5575 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestResult.java @@ -24,23 +24,23 @@ public class RestResult extends RestBaseResult { - private Map resultMap; - - public RestResult() { - super(); - resultMap = Maps.newHashMap(); - } - - public void addResultEntry(String key, Object value) { - resultMap.put(key, value); - } - - public String toJson() { - Gson gson = new Gson(); - addResultEntry("status", status); - if (status != ActionStatus.OK) { - addResultEntry("msg", msg); + private Map resultMap; + + public RestResult() { + super(); + resultMap = Maps.newHashMap(); + } + + public void addResultEntry(String key, Object value) { + resultMap.put(key, value); + } + + public String toJson() { + Gson gson = new Gson(); + addResultEntry("status", status); + if (status != ActionStatus.OK) { + addResultEntry("msg", msg); + } + return gson.toJson(resultMap); } - return gson.toJson(resultMap); - } } From c38666ab0d4b5f3b85128eecf325856a667374ed Mon Sep 17 00:00:00 2001 From: "jiafeng.zhang" Date: Wed, 6 Apr 2022 10:11:10 +0800 Subject: [PATCH 3/3] 1 1 --- .../doris/httpv2/entity/RestBaseResult.java | 16 ++++++++-------- .../apache/doris/httpv2/entity/RestResult.java | 18 +++++++++--------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestBaseResult.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestBaseResult.java index 973d5951b333ff..5d27d9e6915c68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestBaseResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestBaseResult.java @@ -27,21 +27,21 @@ public class RestBaseResult { public String msg; public RestBaseResult() { - status = ActionStatus.OK; - msg = "Success"; + status = ActionStatus.OK; + msg = "Success"; } public RestBaseResult(String msg) { - status = ActionStatus.FAILED; - this.msg = msg; + status = ActionStatus.FAILED; + this.msg = msg; } public static RestBaseResult getOk() { - return OK; - } + return OK; + } public String toJson() { - Gson gson = new GsonBuilder().disableHtmlEscaping().create(); - return gson.toJson(this); + Gson gson = new GsonBuilder().disableHtmlEscaping().create(); + return gson.toJson(this); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestResult.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestResult.java index 3dda63e4ca5575..d501bfdc59ae77 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestResult.java @@ -27,20 +27,20 @@ public class RestResult extends RestBaseResult { private Map resultMap; public RestResult() { - super(); - resultMap = Maps.newHashMap(); + super(); + resultMap = Maps.newHashMap(); } public void addResultEntry(String key, Object value) { - resultMap.put(key, value); + resultMap.put(key, value); } public String toJson() { - Gson gson = new Gson(); - addResultEntry("status", status); - if (status != ActionStatus.OK) { - addResultEntry("msg", msg); - } - return gson.toJson(resultMap); + Gson gson = new Gson(); + addResultEntry("status", status); + if (status != ActionStatus.OK) { + addResultEntry("msg", msg); + } + return gson.toJson(resultMap); } }