diff --git a/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java b/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java index c954055c5a08b5..c8ee60a4e1bd34 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java +++ b/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java @@ -26,7 +26,7 @@ import org.apache.doris.common.Version; import org.apache.doris.common.util.JdkUtils; import org.apache.doris.common.util.NetUtils; -import org.apache.doris.http.HttpServer; +import org.apache.doris.httpv2.HttpServer; import org.apache.doris.journal.bdbje.BDBDebugger; import org.apache.doris.journal.bdbje.BDBTool; import org.apache.doris.journal.bdbje.BDBToolOptions; @@ -134,25 +134,14 @@ public static void start(String dorisHomeDir, String pidDir, String[] args) { feServer.start(); - if (!Config.enable_http_server_v2) { - HttpServer httpServer = new HttpServer( - Config.http_port, - Config.http_max_line_length, - Config.http_max_header_size, - Config.http_max_chunk_size - ); - httpServer.setup(); - httpServer.start(); - } else { - org.apache.doris.httpv2.HttpServer httpServer2 = new org.apache.doris.httpv2.HttpServer(); - httpServer2.setPort(Config.http_port); - httpServer2.setMaxHttpPostSize(Config.jetty_server_max_http_post_size); - httpServer2.setAcceptors(Config.jetty_server_acceptors); - httpServer2.setSelectors(Config.jetty_server_selectors); - httpServer2.setWorkers(Config.jetty_server_workers); - httpServer2.start(dorisHomeDir); - } - + HttpServer httpServer = new HttpServer(); + httpServer.setPort(Config.http_port); + httpServer.setMaxHttpPostSize(Config.jetty_server_max_http_post_size); + httpServer.setAcceptors(Config.jetty_server_acceptors); + httpServer.setSelectors(Config.jetty_server_selectors); + httpServer.setWorkers(Config.jetty_server_workers); + httpServer.start(dorisHomeDir); + qeService.start(); ThreadPoolManager.registerAllThreadPoolMetric(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index 57a066680fe0e3..71ebf3c5bfef84 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -156,7 +156,7 @@ import org.apache.doris.ha.FrontendNodeType; import org.apache.doris.ha.HAProtocol; import org.apache.doris.ha.MasterInfo; -import org.apache.doris.http.meta.MetaBaseAction; +import org.apache.doris.httpv2.meta.MetaBaseAction; import org.apache.doris.journal.JournalCursor; import org.apache.doris.journal.JournalEntity; import org.apache.doris.journal.bdbje.Timestamp; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java index 040e8ca092b765..355477d9fcf963 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java @@ -327,22 +327,6 @@ public class Config extends ConfigBase { */ @ConfField public static int http_port = 8030; - /* - * Netty http param - */ - @ConfField public static int http_max_line_length = HttpServer.DEFAULT_MAX_LINE_LENGTH; - - @ConfField public static int http_max_header_size = HttpServer.DEFAULT_MAX_HEADER_SIZE; - - @ConfField public static int http_max_chunk_size = HttpServer.DEFAULT_MAX_CHUNK_SIZE; - - /** - * The backlog_num for netty http server - * When you enlarge this backlog_num, you should enlarge the value in - * the linux /proc/sys/net/core/somaxconn file at the same time - */ - @ConfField public static int http_backlog_num = 1024; - /** * Jetty container default configuration * Jetty's thread architecture model is very simple, divided into three thread pools: diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/ActionController.java b/fe/fe-core/src/main/java/org/apache/doris/http/ActionController.java deleted file mode 100644 index 8264f9169f7f6f..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/ActionController.java +++ /dev/null @@ -1,88 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - -import org.apache.doris.common.path.PathTrie; -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; - -public class ActionController { - private final PathTrie getHandlers = new PathTrie<>(WebUtils.REST_DECODER); - private final PathTrie postHandlers = new PathTrie<>(WebUtils.REST_DECODER); - private final PathTrie putHandlers = new PathTrie<>(WebUtils.REST_DECODER); - private final PathTrie deleteHandlers = new PathTrie<>(WebUtils.REST_DECODER); - private final PathTrie headHandlers = new PathTrie<>(WebUtils.REST_DECODER); - private final PathTrie optionsHandlers = new PathTrie<>(WebUtils.REST_DECODER); - - // Registers a rest handler to be execute when the provided method and path match the request. - public void registerHandler(HttpMethod method, String path, IAction handler) - throws IllegalArgException { - if (method.equals(HttpMethod.GET)) { - getHandlers.insert(path, handler); - } else if (method.equals(HttpMethod.POST)) { - postHandlers.insert(path, handler); - } else if (method.equals(HttpMethod.HEAD)) { - headHandlers.insert(path, handler); - } else if (method.equals(HttpMethod.DELETE)) { - deleteHandlers.insert(path, handler); - } else if (method.equals(HttpMethod.OPTIONS)) { - optionsHandlers.insert(path, handler); - } else if (method.equals(HttpMethod.PUT)) { - putHandlers.insert(path, handler); - } else { - throw new IllegalArgException( - "Can't handle [" + method + "] for path [" + path + "]"); - } - } - - public IAction getHandler(BaseRequest request) { - String path = getPath(request.getRequest().uri()); - HttpMethod method = request.getRequest().method(); - if (method.equals(HttpMethod.GET)) { - return getHandlers.retrieve(path, request.getParams()); - } else if (method.equals(HttpMethod.POST)) { - return postHandlers.retrieve(path, request.getParams()); - } else if (method.equals(HttpMethod.PUT)) { - return putHandlers.retrieve(path, request.getParams()); - } else if (method.equals(HttpMethod.DELETE)) { - return deleteHandlers.retrieve(path, request.getParams()); - } else if (method.equals(HttpMethod.HEAD)) { - return headHandlers.retrieve(path, request.getParams()); - } else if (method.equals(HttpMethod.OPTIONS)) { - return optionsHandlers.retrieve(path, request.getParams()); - } else { - return null; - } - } - - // e.g. - // in: /www/system?path=//jobs - // out: /www/system - private String getPath(String uri) { - if (Strings.isNullOrEmpty(uri)) { - return ""; - } - int pathEndIndex = uri.indexOf('?'); - if (pathEndIndex < 0) { - return uri; - } else { - return uri.substring(0, pathEndIndex); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/BaseAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/BaseAction.java deleted file mode 100644 index b3df5e3e338718..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/BaseAction.java +++ /dev/null @@ -1,375 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - -import org.apache.doris.analysis.UserIdentity; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.system.SystemInfoService; - -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.io.UnsupportedEncodingException; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelProgressiveFuture; -import io.netty.channel.ChannelProgressiveFutureListener; -import io.netty.channel.DefaultFileRegion; -import io.netty.handler.codec.base64.Base64; -import io.netty.handler.codec.http.DefaultFullHttpResponse; -import io.netty.handler.codec.http.DefaultHttpResponse; -import io.netty.handler.codec.http.FullHttpResponse; -import io.netty.handler.codec.http.HttpChunkedInput; -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpHeaderValues; -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponse; -import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.codec.http.HttpUtil; -import io.netty.handler.codec.http.HttpVersion; -import io.netty.handler.codec.http.LastHttpContent; -import io.netty.handler.codec.http.cookie.Cookie; -import io.netty.handler.codec.http.cookie.ServerCookieEncoder; -import io.netty.handler.ssl.SslHandler; -import io.netty.handler.stream.ChunkedFile; -import io.netty.handler.stream.ChunkedInput; -import io.netty.handler.stream.ChunkedStream; -import io.netty.util.CharsetUtil; - -public abstract class BaseAction implements IAction { - private static final Logger LOG = LogManager.getLogger(BaseAction.class); - - protected ActionController controller; - protected Catalog catalog; - - public BaseAction(ActionController controller) { - this.controller = controller; - // TODO(zc): remove this instance - this.catalog = Catalog.getCurrentCatalog(); - } - - @Override - public void handleRequest(BaseRequest request) throws Exception { - BaseResponse response = new BaseResponse(); - LOG.debug("receive http request. url={}", request.getRequest().uri()); - try { - execute(request, response); - } catch (Exception e) { - LOG.warn("fail to process url: {}", request.getRequest().uri(), e); - if (e instanceof UnauthorizedException) { - response.updateHeader(HttpHeaderNames.WWW_AUTHENTICATE.toString(), "Basic realm=\"\""); - writeResponse(request, response, HttpResponseStatus.UNAUTHORIZED); - } else { - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); - } - } - } - - public abstract void execute(BaseRequest request, BaseResponse response) throws DdlException; - - protected void writeResponse(BaseRequest request, BaseResponse response, HttpResponseStatus status) { - // if (HttpHeaders.is100ContinueExpected(request.getRequest())) { - // ctx.write(new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, - // HttpResponseStatus.CONTINUE)); - // } - - FullHttpResponse responseObj = null; - try { - responseObj = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, status, - Unpooled.wrappedBuffer(response.getContent().toString().getBytes("UTF-8"))); - } catch (UnsupportedEncodingException e) { - LOG.warn("get exception.", e); - responseObj = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, status, - Unpooled.wrappedBuffer(response.getContent().toString().getBytes())); - } - Preconditions.checkNotNull(responseObj); - HttpMethod method = request.getRequest().method(); - - checkDefaultContentTypeHeader(response, responseObj); - if (!method.equals(HttpMethod.HEAD)) { - response.updateHeader(HttpHeaderNames.CONTENT_LENGTH.toString(), - String.valueOf(responseObj.content().readableBytes())); - } - writeCustomHeaders(response, responseObj); - writeCookies(response, responseObj); - - boolean keepAlive = HttpUtil.isKeepAlive(request.getRequest()); - if (!keepAlive) { - request.getContext().write(responseObj).addListener(ChannelFutureListener.CLOSE); - } else { - responseObj.headers().set(HttpHeaderNames.CONNECTION.toString(), HttpHeaderValues.KEEP_ALIVE.toString()); - request.getContext().write(responseObj); - } - } - - // Object only support File or byte[] - protected void writeObjectResponse(BaseRequest request, BaseResponse response, HttpResponseStatus status, - Object obj, String fileName, boolean isOctStream) { - Preconditions.checkState((obj instanceof File) || (obj instanceof byte[])); - - HttpResponse responseObj = new DefaultHttpResponse(HttpVersion.HTTP_1_1, status); - - if (HttpUtil.isKeepAlive(request.getRequest())) { - response.updateHeader(HttpHeaderNames.CONNECTION.toString(), HttpHeaderValues.KEEP_ALIVE.toString()); - } - - if (isOctStream) { - response.updateHeader(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.APPLICATION_OCTET_STREAM.toString()); - response.updateHeader(HttpHeaderNames.CONTENT_DISPOSITION.toString(), - HttpHeaderValues.ATTACHMENT.toString() + "; " + HttpHeaderValues.FILENAME.toString() + "=" + fileName); - } - - ChannelFuture sendFileFuture; - ChannelFuture lastContentFuture; - - try { - Object writable = null; - long contentLen = 0; - boolean sslEnable = request.getContext().pipeline().get(SslHandler.class) != null; - if (obj instanceof File) { - RandomAccessFile rafFile = new RandomAccessFile((File) obj, "r"); - contentLen = rafFile.length(); - if (!sslEnable) { - // use zero-copy file transfer. - writable = new DefaultFileRegion(rafFile.getChannel(), 0, contentLen); - } else { - // cannot use zero-copy file transfer. - writable = new ChunkedFile(rafFile, 0, contentLen, 8192); - } - } else if (obj instanceof byte[]) { - contentLen = ((byte[]) obj).length; - if (!sslEnable) { - writable = Unpooled.wrappedBuffer((byte[]) obj); - } else { - writable = new ChunkedStream(new ByteArrayInputStream((byte[]) obj)); - } - } - - response.updateHeader(HttpHeaderNames.CONTENT_LENGTH.toString(), String.valueOf(contentLen)); - writeCookies(response, responseObj); - writeCustomHeaders(response, responseObj); - - // Write headers - request.getContext().write(responseObj); - - // Write object - if (!sslEnable) { - sendFileFuture = request.getContext().write(writable, request.getContext().newProgressivePromise()); - // Write the end marker. - lastContentFuture = request.getContext().writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT); - } else { - sendFileFuture = request.getContext().writeAndFlush( - new HttpChunkedInput((ChunkedInput) writable), - request.getContext().newProgressivePromise()); - // HttpChunkedInput will write the end marker (LastHttpContent) for us. - lastContentFuture = sendFileFuture; - } - } catch (FileNotFoundException ignore) { - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); - return; - } catch (IOException e1) { - writeResponse(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR); - return; - } - - sendFileFuture.addListener(new ChannelProgressiveFutureListener() { - @Override - public void operationProgressed(ChannelProgressiveFuture future, long progress, long total) { - if (total < 0) { // total unknown - LOG.debug("{} Transfer progress: {}", future.channel(), progress); - } else { - LOG.debug("{} Transfer progress: {} / {}", future.channel(), progress, total); - } - } - - @Override - public void operationComplete(ChannelProgressiveFuture future) { - LOG.debug("{} Transfer complete.", future.channel()); - if (!future.isSuccess()) { - Throwable cause = future.cause(); - LOG.error("something wrong. ", cause); - } - } - }); - - // Decide whether to close the connection or not. - boolean keepAlive = HttpUtil.isKeepAlive(request.getRequest()); - if (!keepAlive) { - // Close the connection when the whole content is written out. - lastContentFuture.addListener(ChannelFutureListener.CLOSE); - } - } - - // Set 'CONTENT_TYPE' header if it hasn't been set. - protected void checkDefaultContentTypeHeader(BaseResponse response, Object responseOj) { - if (!Strings.isNullOrEmpty(response.getContentType())) { - response.updateHeader(HttpHeaderNames.CONTENT_TYPE.toString(), response.getContentType()); - } else { - response.updateHeader(HttpHeaderNames.CONTENT_TYPE.toString(), "text/html"); - } - } - - protected void writeCustomHeaders(BaseResponse response, HttpResponse responseObj) { - for (Map.Entry> entry : response.getCustomHeaders().entrySet()) { - responseObj.headers().add(entry.getKey(), entry.getValue()); - } - } - - protected void writeCookies(BaseResponse response, HttpResponse responseObj) { - for (Cookie cookie : response.getCookies()) { - responseObj.headers().add(HttpHeaderNames.SET_COOKIE.toString(), ServerCookieEncoder.LAX.encode(cookie)); - } - } - - public static class ActionAuthorizationInfo { - public String fullUserName; - public String remoteIp; - public String password; - public String cluster; - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("user: ").append(fullUserName).append(", remote ip: ").append(remoteIp); - sb.append(", password: ").append(password).append(", cluster: ").append(cluster); - return sb.toString(); - } - } - - protected void checkGlobalAuth(UserIdentity currentUser, PrivPredicate predicate) throws UnauthorizedException { - if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(currentUser, predicate)) { - throw new UnauthorizedException("Access denied; you need (at least one of) the " - + predicate.getPrivs().toString() + " privilege(s) for this operation"); - } - } - - protected void checkDbAuth(UserIdentity currentUser, String db, PrivPredicate predicate) - throws UnauthorizedException { - if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(currentUser, db, predicate)) { - throw new UnauthorizedException("Access denied; you need (at least one of) the " - + predicate.getPrivs().toString() + " privilege(s) for this operation"); - } - } - - protected void checkTblAuth(UserIdentity currentUser, String db, String tbl, PrivPredicate predicate) - throws UnauthorizedException { - if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(currentUser, db, tbl, predicate)) { - throw new UnauthorizedException("Access denied; you need (at least one of) the " - + predicate.getPrivs().toString() + " privilege(s) for this operation"); - } - } - - // return currentUserIdentity from Doris auth - protected UserIdentity checkPassword(ActionAuthorizationInfo authInfo) - throws UnauthorizedException { - List currentUser = Lists.newArrayList(); - if (!Catalog.getCurrentCatalog().getAuth().checkPlainPassword(authInfo.fullUserName, - authInfo.remoteIp, authInfo.password, currentUser)) { - throw new UnauthorizedException("Access denied for " - + authInfo.fullUserName + "@" + authInfo.remoteIp); - } - Preconditions.checkState(currentUser.size() == 1); - return currentUser.get(0); - } - - public ActionAuthorizationInfo getAuthorizationInfo(BaseRequest request) - throws UnauthorizedException { - ActionAuthorizationInfo authInfo = new ActionAuthorizationInfo(); - if (!parseAuthInfo(request, authInfo)) { - LOG.info("parse auth info failed, Authorization header {}, url {}", - request.getAuthorizationHeader(), request.getRequest().uri()); - throw new UnauthorizedException("Need auth information."); - } - LOG.debug("get auth info: {}", authInfo); - return authInfo; - } - - private boolean parseAuthInfo(BaseRequest request, ActionAuthorizationInfo authInfo) { - String encodedAuthString = request.getAuthorizationHeader(); - if (Strings.isNullOrEmpty(encodedAuthString)) { - return false; - } - String[] parts = encodedAuthString.split(" "); - if (parts.length != 2) { - return false; - } - encodedAuthString = parts[1]; - ByteBuf buf = null; - ByteBuf decodeBuf = null; - try { - buf = Unpooled.copiedBuffer(ByteBuffer.wrap(encodedAuthString.getBytes())); - - // The authString is a string connecting user-name and password with - // a colon(':') - decodeBuf = Base64.decode(buf); - String authString = decodeBuf.toString(CharsetUtil.UTF_8); - // Note that password may contain colon, so can not simply use a - // colon to split. - int index = authString.indexOf(":"); - authInfo.fullUserName = authString.substring(0, index); - final String[] elements = authInfo.fullUserName.split("@"); - if (elements != null && elements.length < 2) { - authInfo.fullUserName = ClusterNamespace.getFullName(SystemInfoService.DEFAULT_CLUSTER, - authInfo.fullUserName); - authInfo.cluster = SystemInfoService.DEFAULT_CLUSTER; - } else if (elements != null && elements.length == 2) { - authInfo.fullUserName = ClusterNamespace.getFullName(elements[1], elements[0]); - authInfo.cluster = elements[1]; - } - authInfo.password = authString.substring(index + 1); - authInfo.remoteIp = request.getHostString(); - } finally { - // release the buf and decode buf after using Unpooled.copiedBuffer - // or it will get memory leak - if (buf != null) { - buf.release(); - } - - if (decodeBuf != null) { - decodeBuf.release(); - } - } - return true; - } - - protected int checkIntParam(String strParam) { - return Integer.parseInt(strParam); - } - - protected long checkLongParam(String strParam) { - return Long.parseLong(strParam); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/BaseRequest.java b/fe/fe-core/src/main/java/org/apache/doris/http/BaseRequest.java deleted file mode 100644 index 81633a5cdc587c..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/BaseRequest.java +++ /dev/null @@ -1,158 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - -import org.apache.doris.common.DdlException; - -import com.google.common.base.Strings; -import com.google.common.collect.Maps; - -import java.net.InetSocketAddress; -import java.nio.charset.Charset; -import java.util.List; -import java.util.Map; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.FullHttpRequest; -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.QueryStringDecoder; -import io.netty.handler.codec.http.cookie.ClientCookieDecoder; -import io.netty.handler.codec.http.cookie.Cookie; - -public class BaseRequest { - protected ChannelHandlerContext context; - protected HttpRequest request; - protected Map params = Maps.newHashMap(); - - private boolean isAuthorized = false; - private QueryStringDecoder decoder; - - public BaseRequest(ChannelHandlerContext ctx, HttpRequest request) { - this.context = ctx; - this.request = request; - } - - public ChannelHandlerContext getContext() { - return context; - } - - public void setContext(ChannelHandlerContext context) { - this.context = context; - } - - public HttpRequest getRequest() { - return request; - } - - public void setRequest(HttpRequest request) { - this.request = request; - } - - public Map getParams() { - return params; - } - - public void setParams(Map params) { - this.params = params; - } - - public boolean isAuthorized() { - return isAuthorized; - } - - public void setAuthorized(boolean isAuthorized) { - this.isAuthorized = isAuthorized; - } - - public Cookie getCookieByName(String cookieName) { - String cookieString = request.headers().get(HttpHeaderNames.COOKIE.toString()); - if (!Strings.isNullOrEmpty(cookieString)) { - Cookie cookie = ClientCookieDecoder.STRICT.decode(cookieString); - if (cookie.name().equalsIgnoreCase(cookieName)) { - return cookie; - } - } - return null; - } - - public String getCookieValue(String cookieName) { - Cookie cookie = getCookieByName(cookieName); - if (cookie != null) { - return cookie.value(); - } - return null; - } - - // get a single parameter. - // return null if key is not exist; return the first value if key is an array - public String getSingleParameter(String key) { - String uri = request.uri(); - if (decoder == null) { - decoder = new QueryStringDecoder(uri); - } - - List values = decoder.parameters().get(key); - if (values != null && values.size() > 0) { - return values.get(0); - } - - return params.get(key); - } - - public String getContent() throws DdlException { - if (request instanceof FullHttpRequest) { - FullHttpRequest fullHttpRequest = (FullHttpRequest) request; - return fullHttpRequest.content().toString(Charset.forName("UTF-8")); - } else { - throw new DdlException("Invalid request"); - } - } - - // get an array parameter. - // eg. ?a=1&a=2 - public List getArrayParameter(String key) { - String uri = request.uri(); - if (decoder == null) { - decoder = new QueryStringDecoder(uri); - } - - return decoder.parameters().get(key); - } - - public Map> getAllParameters() { - String uri = request.uri(); - if (decoder == null) { - decoder = new QueryStringDecoder(uri); - } - - return decoder.parameters(); - } - - public String getAuthorizationHeader() { - String authString = request.headers().get("Authorization"); - return authString; - } - - public String getHostString() { - // get client host - InetSocketAddress clientSocket = (InetSocketAddress) context.channel().remoteAddress(); - String clientIp = clientSocket.getHostString(); - return clientIp; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/BaseResponse.java b/fe/fe-core/src/main/java/org/apache/doris/http/BaseResponse.java deleted file mode 100644 index 297c8c7f8c9474..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/BaseResponse.java +++ /dev/null @@ -1,99 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; - -import java.util.List; -import java.util.Map; -import java.util.Set; - -import io.netty.handler.codec.http.cookie.Cookie; - -public class BaseResponse { - private String contentType; - protected StringBuilder content = new StringBuilder(); - - protected Map> customHeaders = Maps.newHashMap(); - private Set cookies = Sets.newHashSet(); - - public String getContentType() { - return contentType; - } - - public void setContentType(String contentType) { - this.contentType = contentType; - } - - public StringBuilder getContent() { - return content; - } - - public Set getCookies() { - return cookies; - } - - public Map> getCustomHeaders() { - return customHeaders; - } - - // update old key-value mapping of 'name' if Exist, or add new mapping if not exists. - // It will only change the mapping of 'name', other header will not be changed. - public void updateHeader(String name, String value) { - if (customHeaders == null) { - customHeaders = Maps.newHashMap(); - } - customHeaders.remove(name); - addHeader(name, value); - } - - // Add a custom header. - private void addHeader(String name, String value) { - if (customHeaders == null) { - customHeaders = Maps.newHashMap(); - } - List header = customHeaders.get(name); - if (header == null) { - header = Lists.newArrayList(); - customHeaders.put(name, header); - } - header.add(value); - } - - public void appendContent(String buffer) { - if (content == null) { - content = new StringBuilder(); - } - content.append(buffer); - } - - public void addCookie(Cookie cookie) { - cookies.add(cookie); - } - - public void updateCookieAge(BaseRequest request, String cookieName, long age) { - Cookie cookie = request.getCookieByName(cookieName); - if (cookie != null) { - cookies.remove(cookie); - cookie.setMaxAge(age); - cookies.add(cookie); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/HttpAuthManager.java b/fe/fe-core/src/main/java/org/apache/doris/http/HttpAuthManager.java deleted file mode 100644 index d0cd55ef80cff0..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/HttpAuthManager.java +++ /dev/null @@ -1,64 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - -import org.apache.doris.analysis.UserIdentity; - -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; - -import java.util.concurrent.TimeUnit; - -// We simulate a simplified session here: only store user-name of clients who already logged in, -// and we only have a default admin user for now. -public final class HttpAuthManager { - private static long SESSION_EXPIRE_TIME = 2; // hour - private static long SESSION_MAX_SIZE = 100; // avoid to store too many - - private static HttpAuthManager instance = new HttpAuthManager(); - - public static class SessionValue { - public UserIdentity currentUser; - } - - // session_id => session value - private Cache authSessions = CacheBuilder.newBuilder() - .maximumSize(SESSION_MAX_SIZE) - .expireAfterAccess(SESSION_EXPIRE_TIME, TimeUnit.HOURS) - .build(); - - private HttpAuthManager() { - // do nothing - } - - public static HttpAuthManager getInstance() { - return instance; - } - - public SessionValue getSessionValue(String sessionId) { - return authSessions.getIfPresent(sessionId); - } - - public void addSessionValue(String key, SessionValue value) { - authSessions.put(key, value); - } - - public Cache getAuthSessions() { - return authSessions; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/HttpServer.java b/fe/fe-core/src/main/java/org/apache/doris/http/HttpServer.java deleted file mode 100644 index 1b1df9cf2601c1..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/HttpServer.java +++ /dev/null @@ -1,279 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - -import org.apache.doris.common.Config; -import org.apache.doris.http.action.BackendAction; -import org.apache.doris.http.action.HaAction; -import org.apache.doris.http.action.HelpAction; -import org.apache.doris.http.action.IndexAction; -import org.apache.doris.http.action.LogAction; -import org.apache.doris.http.action.QueryAction; -import org.apache.doris.http.action.QueryProfileAction; -import org.apache.doris.http.action.SessionAction; -import org.apache.doris.http.action.StaticResourceAction; -import org.apache.doris.http.action.SystemAction; -import org.apache.doris.http.action.VariableAction; -import org.apache.doris.http.common.DorisHttpPostObjectAggregator; -import org.apache.doris.http.meta.ColocateMetaService; -import org.apache.doris.http.meta.MetaService.CheckAction; -import org.apache.doris.http.meta.MetaService.DumpAction; -import org.apache.doris.http.meta.MetaService.ImageAction; -import org.apache.doris.http.meta.MetaService.InfoAction; -import org.apache.doris.http.meta.MetaService.JournalIdAction; -import org.apache.doris.http.meta.MetaService.PutAction; -import org.apache.doris.http.meta.MetaService.RoleAction; -import org.apache.doris.http.meta.MetaService.VersionAction; -import org.apache.doris.http.rest.BootstrapFinishAction; -import org.apache.doris.http.rest.CancelStreamLoad; -import org.apache.doris.http.rest.CheckDecommissionAction; -import org.apache.doris.http.rest.ConnectionAction; -import org.apache.doris.http.rest.GetDdlStmtAction; -import org.apache.doris.http.rest.GetLoadInfoAction; -import org.apache.doris.http.rest.GetLogFileAction; -import org.apache.doris.http.rest.GetSmallFileAction; -import org.apache.doris.http.rest.GetStreamLoadState; -import org.apache.doris.http.rest.HealthAction; -import org.apache.doris.http.rest.LoadAction; -import org.apache.doris.http.rest.MetaReplayerCheckAction; -import org.apache.doris.http.rest.MetricsAction; -import org.apache.doris.http.rest.MigrationAction; -import org.apache.doris.http.rest.MultiAbort; -import org.apache.doris.http.rest.MultiCommit; -import org.apache.doris.http.rest.MultiDesc; -import org.apache.doris.http.rest.MultiList; -import org.apache.doris.http.rest.MultiStart; -import org.apache.doris.http.rest.MultiUnload; -import org.apache.doris.http.rest.ProfileAction; -import org.apache.doris.http.rest.QueryDetailAction; -import org.apache.doris.http.rest.RowCountAction; -import org.apache.doris.http.rest.SetConfigAction; -import org.apache.doris.http.rest.ShowDataAction; -import org.apache.doris.http.rest.ShowMetaInfoAction; -import org.apache.doris.http.rest.ShowProcAction; -import org.apache.doris.http.rest.ShowRuntimeInfoAction; -import org.apache.doris.http.rest.StorageTypeCheckAction; -import org.apache.doris.http.rest.TableQueryPlanAction; -import org.apache.doris.http.rest.TableRowCountAction; -import org.apache.doris.http.rest.TableSchemaAction; -import org.apache.doris.master.MetaHelper; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.File; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.Channel; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.handler.codec.http.HttpServerCodec; -import io.netty.handler.stream.ChunkedWriteHandler; - -public class HttpServer { - private static final Logger LOG = LogManager.getLogger(HttpServer.class); - - /** - * The default netty param, witch is the same as `HttpServerCodec`. - */ - public static final int DEFAULT_MAX_LINE_LENGTH = 4096; - public static final int DEFAULT_MAX_HEADER_SIZE = 8192; - public static final int DEFAULT_MAX_CHUNK_SIZE = 8192; - - private final int port; - private final int maxInitialLineLength; - private final int maxHeaderSize; - private final int maxChunkSize; - private ActionController controller; - - private Thread serverThread; - - private AtomicBoolean isStarted = new AtomicBoolean(false); - - public HttpServer(int port) { - this(port, DEFAULT_MAX_LINE_LENGTH, DEFAULT_MAX_HEADER_SIZE, DEFAULT_MAX_CHUNK_SIZE); - } - - public HttpServer(int port, int maxInitialLineLength, int maxHeaderSize, int maxChunkSize) { - this.port = port; - this.maxInitialLineLength = maxInitialLineLength; - this.maxHeaderSize = maxHeaderSize; - this.maxChunkSize = maxChunkSize; - controller = new ActionController(); - } - - public void setup() throws IllegalArgException { - registerActions(); - } - - private void registerActions() throws IllegalArgException { - // add rest action - LoadAction.registerAction(controller); - GetLoadInfoAction.registerAction(controller); - SetConfigAction.registerAction(controller); - GetDdlStmtAction.registerAction(controller); - MigrationAction.registerAction(controller); - StorageTypeCheckAction.registerAction(controller); - CancelStreamLoad.registerAction(controller); - GetStreamLoadState.registerAction(controller); - - // add web action - IndexAction.registerAction(controller); - SystemAction.registerAction(controller); - BackendAction.registerAction(controller); - LogAction.registerAction(controller); - QueryAction.registerAction(controller); - QueryProfileAction.registerAction(controller); - SessionAction.registerAction(controller); - VariableAction.registerAction(controller); - HelpAction.registerAction(controller); - StaticResourceAction.registerAction(controller); - HaAction.registerAction(controller); - - // Add multi action - MultiStart.registerAction(controller); - MultiDesc.registerAction(controller); - MultiCommit.registerAction(controller); - MultiUnload.registerAction(controller); - MultiAbort.registerAction(controller); - MultiList.registerAction(controller); - - // rest action - HealthAction.registerAction(controller); - MetricsAction.registerAction(controller); - ShowMetaInfoAction.registerAction(controller); - ShowProcAction.registerAction(controller); - ShowRuntimeInfoAction.registerAction(controller); - GetLogFileAction.registerAction(controller); - GetSmallFileAction.registerAction(controller); - RowCountAction.registerAction(controller); - CheckDecommissionAction.registerAction(controller); - MetaReplayerCheckAction.registerAction(controller); - ColocateMetaService.BucketSeqAction.registerAction(controller); - ColocateMetaService.ColocateMetaAction.registerAction(controller); - ColocateMetaService.MarkGroupStableAction.registerAction(controller); - ProfileAction.registerAction(controller); - QueryDetailAction.registerAction(controller); - ConnectionAction.registerAction(controller); - ShowDataAction.registerAction(controller); - - // meta service action - File imageDir = MetaHelper.getMasterImageDir(); - ImageAction.registerAction(controller, imageDir); - InfoAction.registerAction(controller, imageDir); - VersionAction.registerAction(controller, imageDir); - PutAction.registerAction(controller, imageDir); - JournalIdAction.registerAction(controller, imageDir); - CheckAction.registerAction(controller, imageDir); - DumpAction.registerAction(controller, imageDir); - RoleAction.registerAction(controller, imageDir); - - // external usage - TableRowCountAction.registerAction(controller); - TableSchemaAction.registerAction(controller); - TableQueryPlanAction.registerAction(controller); - - BootstrapFinishAction.registerAction(controller); - } - - public void start() { - serverThread = new Thread(new HttpServerThread(), "FE Http Server"); - serverThread.start(); - } - - protected class PaloHttpServerInitializer extends ChannelInitializer { - @Override - protected void initChannel(SocketChannel ch) throws Exception { - ch.pipeline().addLast(new HttpServerCodec(maxInitialLineLength, maxHeaderSize, maxChunkSize)); - ch.pipeline().addLast(new DorisHttpPostObjectAggregator(100 * 65536)); - ch.pipeline().addLast(new ChunkedWriteHandler()); - ch.pipeline().addLast(new HttpServerHandler(controller)); - } - } - - ServerBootstrap serverBootstrap; - - private class HttpServerThread implements Runnable { - @Override - public void run() { - // Configure the server. - EventLoopGroup bossGroup = new NioEventLoopGroup(); - EventLoopGroup workerGroup = new NioEventLoopGroup(); - try { - serverBootstrap = new ServerBootstrap(); - serverBootstrap.option(ChannelOption.SO_BACKLOG, Config.http_backlog_num); - // reused address and port to avoid bind already exception - serverBootstrap.option(ChannelOption.SO_REUSEADDR, true); - serverBootstrap.childOption(ChannelOption.SO_REUSEADDR, true); - serverBootstrap.group(bossGroup, workerGroup) - .channel(NioServerSocketChannel.class) - .childHandler(new PaloHttpServerInitializer()); - Channel ch = serverBootstrap.bind(port).sync().channel(); - - isStarted.set(true); - LOG.info("HttpServer started with port {}", port); - // block until server is closed - ch.closeFuture().sync(); - } catch (Exception e) { - LOG.error("Fail to start FE query http server[port: " + port + "] ", e); - System.exit(-1); - } finally { - bossGroup.shutdownGracefully(); - workerGroup.shutdownGracefully(); - } - } - } - - // used for test, release bound port - public void shutDown() { - if (serverBootstrap != null) { - Future future = serverBootstrap.config().group().shutdownGracefully(0, 1, TimeUnit.SECONDS).syncUninterruptibly(); - try { - future.get(); - isStarted.set(false); - LOG.info("HttpServer was closed completely"); - } catch (Throwable e) { - LOG.warn("Exception happened when close HttpServer", e); - } - serverBootstrap = null; - } - } - - public boolean isStarted() { - return isStarted.get(); - } - - public static void main(String[] args) throws Exception { - HttpServer httpServer = new HttpServer(8080); - httpServer.setup(); - System.out.println("before start http server."); - httpServer.start(); - System.out.println("after start http server."); - - while (true) { - Thread.sleep(2000); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/HttpServerHandler.java b/fe/fe-core/src/main/java/org/apache/doris/http/HttpServerHandler.java deleted file mode 100644 index eecbe60a6e49cb..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/HttpServerHandler.java +++ /dev/null @@ -1,117 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - -import org.apache.doris.http.action.IndexAction; -import org.apache.doris.http.action.NotFoundAction; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.net.URISyntaxException; - -import io.netty.buffer.Unpooled; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.handler.codec.http.DefaultFullHttpResponse; -import io.netty.handler.codec.http.FullHttpRequest; -import io.netty.handler.codec.http.FullHttpResponse; -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.codec.http.HttpVersion; -import io.netty.util.ReferenceCountUtil; - -public class HttpServerHandler extends ChannelInboundHandlerAdapter { - private static final Logger LOG = LogManager.getLogger(HttpServerHandler.class); - - private ActionController controller = null; - protected FullHttpRequest fullRequest = null; - protected HttpRequest request = null; - private BaseAction action = null; - - public HttpServerHandler(ActionController controller) { - super(); - this.controller = controller; - } - - @Override - public void channelReadComplete(ChannelHandlerContext ctx) { - ctx.flush(); - } - - @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - if (msg instanceof HttpRequest) { - this.request = (HttpRequest) msg; - LOG.debug("request: url:[{}]", request.uri()); - if (!isRequestValid(ctx, request)) { - writeResponse(ctx, HttpResponseStatus.BAD_REQUEST, "this is a bad request."); - return; - } - BaseRequest req = new BaseRequest(ctx, request); - - action = getAction(req); - if (action != null) { - LOG.debug("action: {} ", action.getClass().getName()); - action.handleRequest(req); - } - } else { - ReferenceCountUtil.release(msg); - } - } - - private boolean isRequestValid(ChannelHandlerContext ctx, HttpRequest request) throws URISyntaxException { - return true; - } - - private void writeResponse(ChannelHandlerContext context , HttpResponseStatus status, String content) { - FullHttpResponse responseObj = new DefaultFullHttpResponse( - HttpVersion.HTTP_1_1, - status, - Unpooled.wrappedBuffer(content.getBytes())); - responseObj.headers().set(HttpHeaderNames.CONTENT_TYPE.toString(), "text/html"); - responseObj.headers().set(HttpHeaderNames.CONTENT_LENGTH.toString(), responseObj.content().readableBytes()); - context.writeAndFlush(responseObj).addListener(ChannelFutureListener.CLOSE); - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - LOG.warn("Handle http from " + ctx.channel().remoteAddress() + " caught exception", cause); - ctx.close(); - } - - private BaseAction getAction(BaseRequest request) { - String uri = request.getRequest().uri(); - // ignore this request, which is a default request from client's browser. - if (uri.endsWith("/favicon.ico")) { - return NotFoundAction.getNotFoundAction(); - } else if (uri.equals("/")) { - return new IndexAction(controller); - } - - // Map params = Maps.newHashMap(); - BaseAction action = (BaseAction) controller.getHandler(request); - if (action == null) { - action = NotFoundAction.getNotFoundAction(); - } - - return action; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/IAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/IAction.java deleted file mode 100644 index a89d0ee3d035fa..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/IAction.java +++ /dev/null @@ -1,25 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - - -public interface IAction { - - public void handleRequest(BaseRequest request) throws Exception; - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/UnauthorizedException.java b/fe/fe-core/src/main/java/org/apache/doris/http/UnauthorizedException.java deleted file mode 100644 index 4802498ce09b98..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/UnauthorizedException.java +++ /dev/null @@ -1,26 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - -import org.apache.doris.common.DdlException; - -public class UnauthorizedException extends DdlException { - public UnauthorizedException(String msg) { - super(msg); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/WebUtils.java b/fe/fe-core/src/main/java/org/apache/doris/http/WebUtils.java deleted file mode 100644 index c9efe86ac0195e..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/WebUtils.java +++ /dev/null @@ -1,230 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http; - -import org.apache.doris.common.path.PathTrie; - -import com.google.common.base.Charsets; - -import java.nio.charset.Charset; -import java.util.Map; - -public class WebUtils { - public static final String HTTP_CORS_ALLOW_ORIGIN_SETTING = "http.cors.allow-origin"; - - public static final PathTrie.Decoder REST_DECODER = new PathTrie.Decoder() { - @Override - public String decode(String value) { - return WebUtils.decodeComponent(value); - } - }; - - public static boolean isBrowser(String userAgent) { - if (userAgent == null) { - return false; - } - // chrome, safari, firefox, ie - if (userAgent.startsWith("Mozilla")) { - return true; - } - return false; - } - - public static void decodeQueryString(String s, int fromIndex, Map params) { - if (fromIndex < 0) { - return; - } - if (fromIndex >= s.length()) { - return; - } - - String name = null; - int pos = fromIndex; // Beginning of the unprocessed region - int i; // End of the unprocessed region - char c = 0; // Current character - for (i = fromIndex; i < s.length(); i++) { - c = s.charAt(i); - if (c == '=' && name == null) { - if (pos != i) { - name = decodeComponent(s.substring(pos, i)); - } - pos = i + 1; - } else if (c == '&') { - if (name == null && pos != i) { - // We haven't seen an `=' so far but moved forward. - // Must be a param of the form '&a&' so add it with - // an empty value. - addParam(params, decodeComponent(s.substring(pos, i)), ""); - } else if (name != null) { - addParam(params, name, decodeComponent(s.substring(pos, i))); - name = null; - } - pos = i + 1; - } - } - - if (pos != i) { // Are there characters we haven't dealt with? - if (name == null) { // Yes and we haven't seen any `='. - addParam(params, decodeComponent(s.substring(pos, i)), ""); - } else { // Yes and this must be the last value. - addParam(params, name, decodeComponent(s.substring(pos, i))); - } - } else if (name != null) { // Have we seen a name without value? - addParam(params, name, ""); - } - } - - private static void addParam(Map params, String name, String value) { - params.put(name, value); - } - - /** - * Decodes a bit of an URL encoded by a browser. - *

- * This is equivalent to calling {@link #decodeComponent(String, Charset)} - * with the UTF-8 charset (recommended to comply with RFC 3986, Section 2). - * - * @param s The string to decode (can be empty). - * @return The decoded string, or {@code s} if there's nothing to decode. - * If the string to decode is {@code null}, returns an empty string. - * @throws IllegalArgumentException if the string contains a malformed - * escape sequence. - */ - public static String decodeComponent(final String s) { - return decodeComponent(s, Charsets.UTF_8); - } - - /** - * Decodes a bit of an URL encoded by a browser. - *

- * The string is expected to be encoded as per RFC 3986, Section 2. - * This is the encoding used by JavaScript functions {@code encodeURI} - * and {@code encodeURIComponent}, but not {@code escape}. For example - * in this encoding, é (in Unicode {@code U+00E9} or in UTF-8 - * {@code 0xC3 0xA9}) is encoded as {@code %C3%A9} or {@code %c3%a9}. - *

- * This is essentially equivalent to calling - * {@link java.net.URLDecoder URLDecoder}.{@link - * java.net.URLDecoder#decode(String, String)} - * except that it's over 2x faster and generates less garbage for the GC. - * Actually this function doesn't allocate any memory if there's nothing - * to decode, the argument itself is returned. - * - * @param s The string to decode (can be empty). - * @param charset The charset to use to decode the string (should really - * be {@link Charsets#UTF_8}. - * @return The decoded string, or {@code s} if there's nothing to decode. - * If the string to decode is {@code null}, returns an empty string. - * @throws IllegalArgumentException if the string contains a malformed - * escape sequence. - */ - @SuppressWarnings("fallthrough") - public static String decodeComponent(final String s, final Charset charset) { - if (s == null) { - return ""; - } - final int size = s.length(); - boolean modified = false; - for (int i = 0; i < size; i++) { - final char c = s.charAt(i); - switch (c) { - case '%': - i++; // We can skip at least one char, e.g. `%%'. - // Fall through. - case '+': - modified = true; - break; - } - } - if (!modified) { - return s; - } - final byte[] buf = new byte[size]; - int pos = 0; // position in `buf'. - for (int i = 0; i < size; i++) { - char c = s.charAt(i); - switch (c) { - case '+': - buf[pos++] = ' '; // "+" -> " " - break; - case '%': - if (i == size - 1) { - throw new IllegalArgumentException("unterminated escape" - + " sequence at end of string: " + s); - } - c = s.charAt(++i); - if (c == '%') { - buf[pos++] = '%'; // "%%" -> "%" - break; - } else if (i == size - 1) { - throw new IllegalArgumentException("partial escape" - + " sequence at end of string: " + s); - } - c = decodeHexNibble(c); - final char c2 = decodeHexNibble(s.charAt(++i)); - if (c == Character.MAX_VALUE || c2 == Character.MAX_VALUE) { - throw new IllegalArgumentException( - "invalid escape sequence `%" + s.charAt(i - 1) - + s.charAt(i) + "' at index " + (i - 2) - + " of: " + s); - } - c = (char) (c * 16 + c2); - // Fall through. - default: - buf[pos++] = (byte) c; - break; - } - } - return new String(buf, 0, pos, charset); - } - - /** - * Helper to decode half of a hexadecimal number from a string. - * - * @param c The ASCII character of the hexadecimal number to decode. - * Must be in the range {@code [0-9a-fA-F]}. - * @return The hexadecimal value represented in the ASCII character - * given, or {@link Character#MAX_VALUE} if the character is invalid. - */ - private static char decodeHexNibble(final char c) { - if ('0' <= c && c <= '9') { - return (char) (c - '0'); - } else if ('a' <= c && c <= 'f') { - return (char) (c - 'a' + 10); - } else if ('A' <= c && c <= 'F') { - return (char) (c - 'A' + 10); - } else { - return Character.MAX_VALUE; - } - } - - /** - * Determine if CORS setting is a regex - */ -// public static Pattern getCorsSettingRegex(Settings settings) { -// String corsSetting = settings.get(HTTP_CORS_ALLOW_ORIGIN_SETTING, "*"); -// int len = corsSetting.length(); -// boolean isRegex = len > 2 && corsSetting.startsWith("/") && corsSetting.endsWith("/"); -// -// if (isRegex) { -// return Pattern.compile(corsSetting.substring(1, corsSetting.length()-1)); -// } -// -// return null; -// } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/BackendAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/BackendAction.java deleted file mode 100644 index 73b44131189e49..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/BackendAction.java +++ /dev/null @@ -1,101 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.util.ListComparator; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.system.Backend; -import com.google.common.collect.ImmutableMap; - -import io.netty.handler.codec.http.HttpMethod; - -public class BackendAction extends WebBaseAction { - private static final Logger LOG = LogManager.getLogger(BackendAction.class); - - public BackendAction(ActionController controller) { - super(controller); - // TODO Auto-generated constructor stub - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/backend", new BackendAction(controller)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - - appendKnownBackendsInfo(response.getContent()); - - getPageFooter(response.getContent()); - writeResponse(request, response); - } - - private void appendKnownBackendsInfo(StringBuilder buffer) { - ImmutableMap backendMap = Catalog.getCurrentSystemInfo().getIdToBackend(); - - List> backendInfos = new ArrayList>(); - for (Backend backend : backendMap.values()) { - List backendInfo = new ArrayList(); - InetAddress address = null; - try { - address = InetAddress.getByName(backend.getHost()); - } catch (UnknownHostException e) { - LOG.warn("unknown host: " + backend.getHost(), e); - continue; - } - backendInfo.add(address.getHostName()); - backendInfo.add(backend.getId()); - backendInfo.add("heart_port: " + backend.getHeartbeatPort() - + ", be_port: " + backend.getBePort() - + ", http_port: " + backend.getHttpPort()); - - backendInfos.add(backendInfo); - } - - // sort by id - ListComparator> comparator = new ListComparator>(1); - Collections.sort(backendInfos, comparator); - - // set result - buffer.append("

Known Backends(" + backendMap.size() + ")

"); - - buffer.append("
");
-        for (List info : backendInfos) {
-            buffer.append(info.get(0));
-            buffer.append(" [id: " + info.get(1));
-            buffer.append(", " + info.get(2) + "]");
-            buffer.append(System.getProperty("line.separator"));
-        }
-        buffer.append("
"); - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/HaAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/HaAction.java deleted file mode 100644 index aa759b1da52d34..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/HaAction.java +++ /dev/null @@ -1,174 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.Config; -import org.apache.doris.ha.HAProtocol; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.persist.Storage; -import org.apache.doris.system.Frontend; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Date; -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; - -public class HaAction extends WebBaseAction { - - public HaAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/ha", new HaAction(controller)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - - appendRoleInfo(response.getContent()); - appendJournalInfo(response.getContent()); - appendCanReadInfo(response.getContent()); - appendNodesInfo(response.getContent()); - appendImageInfo(response.getContent()); - appendDbNames(response.getContent()); - appendFe(response.getContent()); - appendRemovedFe(response.getContent()); - - getPageFooter(response.getContent()); - writeResponse(request, response); - } - - private void appendRoleInfo(StringBuilder buffer) { - buffer.append("

Frontend Role

"); - buffer.append("
");
-        buffer.append("

" + Catalog.getCurrentCatalog().getFeType() + "

"); - buffer.append("
"); - } - - private void appendJournalInfo(StringBuilder buffer) { - buffer.append("

Current Journal Id

"); - buffer.append("
");
-        if (Catalog.getCurrentCatalog().isMaster()) {
-            buffer.append("

" + Catalog.getCurrentCatalog().getEditLog().getMaxJournalId() + "

"); - } else { - buffer.append("

" + Catalog.getCurrentCatalog().getReplayedJournalId() + "

"); - } - buffer.append("
"); - } - - private void appendNodesInfo(StringBuilder buffer) { - HAProtocol haProtocol = Catalog.getCurrentCatalog().getHaProtocol(); - if (haProtocol == null) { - return; - } - List electableNodes = haProtocol.getElectableNodes(true); - if (electableNodes.isEmpty()) { - return; - } - buffer.append("

Electable nodes

"); - buffer.append("
");
-        for (InetSocketAddress node : electableNodes) {
-            buffer.append("

" + node.getAddress() + "

"); - - } - buffer.append("
"); - - List observerNodes = haProtocol.getObserverNodes(); - if (observerNodes == null) { - return; - } - buffer.append("

Observer nodes

"); - buffer.append("
");
-        for (InetSocketAddress node : observerNodes) {
-            buffer.append("

" + node.getHostString() + "

"); - } - buffer.append("
"); - } - - private void appendCanReadInfo(StringBuilder buffer) { - buffer.append("

Can Read

"); - buffer.append("
");
-        buffer.append("

" + Catalog.getCurrentCatalog().canRead() + "

"); - - buffer.append("
"); - } - - private void appendImageInfo(StringBuilder buffer) { - try { - Storage storage = new Storage(Config.meta_dir + "/image"); - buffer.append("

Checkpoint Info

"); - buffer.append("
");
-            buffer.append("

last checkpoint version:" + storage.getImageSeq() + "

"); - long lastCheckpointTime = storage.getCurrentImageFile().lastModified(); - Date date = new Date(lastCheckpointTime); - buffer.append("

last checkpoint time: " + date + "

"); - buffer.append("
"); - } catch (IOException e) { - e.printStackTrace(); - } - } - - private void appendDbNames(StringBuilder buffer) { - List names = Catalog.getCurrentCatalog().getEditLog().getDatabaseNames(); - if (names == null) { - return; - } - - String msg = ""; - for (long name : names) { - msg += name + " "; - } - buffer.append("

Database names

"); - buffer.append("
");
-        buffer.append("

" + msg + "

"); - buffer.append("
"); - } - - private void appendFe(StringBuilder buffer) { - List fes = Catalog.getCurrentCatalog().getFrontends(null /* all */); - if (fes == null) { - return; - } - - buffer.append("

Allowed Frontends

"); - buffer.append("
");
-        for (Frontend fe : fes) {
-            buffer.append("

" + fe.toString() + "

"); - } - buffer.append("
"); - } - - private void appendRemovedFe(StringBuilder buffer) { - List feNames = Catalog.getCurrentCatalog().getRemovedFrontendNames(); - buffer.append("

Removed Frontends

"); - buffer.append("
");
-        for (String feName : feNames) {
-            buffer.append("

" + feName + "

"); - } - buffer.append("
"); - } - -} \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/HelpAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/HelpAction.java deleted file mode 100644 index 786426250b9811..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/HelpAction.java +++ /dev/null @@ -1,223 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.qe.HelpModule; -import org.apache.doris.qe.HelpTopic; - -import com.google.common.base.Strings; - -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; - -public class HelpAction extends WebBaseAction { - private static final String DIV_BACKGROUND_COLOR = "#FCFCFC"; - - String queryString = null; - - public HelpAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/help", new HelpAction(controller)); - } - - @Override - public boolean needAdmin() { - return false; - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - appendHelpStyle(response.getContent()); - - queryString = request.getSingleParameter("query"); - if (Strings.isNullOrEmpty(queryString)) { - // ATTN: according to Mysql protocol, the default query should be "contents" - // when you want to get server side help. - queryString = "contents"; - } else { - queryString = queryString.trim(); - } - appendHelpInfo(response.getContent()); - - getPageFooter(response.getContent()); - writeResponse(request, response); - } - - private void appendHelpInfo(StringBuilder buffer) { - buffer.append("

Help Info

"); - buffer.append("

This page lists the help info, " - + "like 'help contents' in Mysql client.

"); - - appendSearchButton(buffer); - appendExactMatchTopic(buffer); - appendFuzzyMatchTopic(buffer); - appendCategories(buffer); - } - - private void appendSearchButton(StringBuilder buffer) { - buffer.append("
" - + "
" - + "
" - + " " - + " " - + " " - + " " - + "
" - + "
" - + "Back To Home" - + "
"); - } - - private void appendExactMatchTopic(StringBuilder buffer) { - buffer.append("

Exact Matching Topic

"); - buffer.append("
"); - HelpModule module = HelpModule.getInstance(); - HelpTopic topic = module.getTopic(queryString); - if (topic == null) { - buffer.append("
No Exact Matching Topic.
"); - } else { - appendOneTopicInfo(buffer, topic); - } - buffer.append("
"); - } - - private void appendFuzzyMatchTopic(StringBuilder buffer) { - buffer.append("

Fuzzy Matching Topic(By Keyword)

"); - buffer.append("
"); - HelpModule module = HelpModule.getInstance(); - List topics = module.listTopicByKeyword(queryString); - if (topics.isEmpty()) { - buffer.append("
No Fuzzy Matching Topic.
"); - } else if (topics.size() == 1) { - buffer.append("

" - + "Find only one topic, show you the detail info below.

"); - appendOneTopicInfo(buffer, module.getTopic(topics.get(0))); - } else { - buffer.append("

Find " + topics.size() + " topics:

"); - appendNameList(buffer, topics, "Topics"); - } - buffer.append("
"); - } - - private void appendCategories(StringBuilder buffer) { - buffer.append("

Category Info

"); - buffer.append("
"); - HelpModule module = HelpModule.getInstance(); - List categories = module.listCategoryByName(queryString); - if (categories.isEmpty()) { - buffer.append("
No Matching Category.
"); - } else if (categories.size() == 1) { - buffer.append("

" - + "Find only one category, so show you the detail info below.

"); - List topics = module.listTopicByCategory(categories.get(0)); - if (topics.size() > 0) { - buffer.append("

Find " - + topics.size() - + " sub topics.

"); - appendNameList(buffer, topics, "Sub Topics"); - } - - List subCategories = module.listCategoryByCategory(categories.get(0)); - if (subCategories.size() > 0) { - buffer.append("

Find " - + subCategories.size() - + " sub categories.

"); - appendNameList(buffer, subCategories, "Sub Categories"); - } - } else { - buffer.append("

Find " + categories.size() + " category:

"); - appendNameList(buffer, categories, "Categories"); - } - buffer.append("
"); - } - - // The browser will combine continuous whitespace to one, we use
 tag to solve this issue.
-    private void appendOneTopicInfo(StringBuilder buffer, HelpTopic topic) {
-        buffer.append("
"); - buffer.append("

'" + escapeHtmlInPreTag(topic.getName()) + "'

"); - - buffer.append("Description"); - buffer.append("
" 
-                + escapeHtmlInPreTag(topic.getDescription())
-                + "
"); - - buffer.append("Example"); - buffer.append("
" 
-                + escapeHtmlInPreTag(topic.getExample()) 
-                + "
"); - - buffer.append("Keyword"); - buffer.append("
" 
-                + escapeHtmlInPreTag(topic.getKeywords().toString()) 
-                + "
"); - - buffer.append("Url"); - buffer.append("
" 
-                + escapeHtmlInPreTag(topic.getUrl()) 
-                + "
"); - buffer.append("
"); - } - - private void appendNameList(StringBuilder buffer, List names, String tableHeadName) { - buffer.append("
"); - buffer.append(""); - buffer.append(""); - - final String href = "?query="; - for (String name : names) { - buffer.append(""); - } - buffer.append("
" + tableHeadName + "
" + name + "
"); - buffer.append("
"); - } - - private void appendHelpStyle(StringBuilder buffer) { - buffer.append(""); - } -} - diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/IndexAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/IndexAction.java deleted file mode 100644 index af17b29db5f787..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/IndexAction.java +++ /dev/null @@ -1,275 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.common.Version; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; -import oshi.SystemInfo; -import oshi.hardware.CentralProcessor; -import oshi.hardware.GlobalMemory; -import oshi.hardware.HWDiskStore; -import oshi.hardware.HWPartition; -import oshi.hardware.HardwareAbstractionLayer; -import oshi.hardware.NetworkIF; -import oshi.hardware.VirtualMemory; -import oshi.software.os.FileSystem; -import oshi.software.os.NetworkParams; -import oshi.software.os.OSFileStore; -import oshi.software.os.OSProcess; -import oshi.software.os.OperatingSystem; -import oshi.util.FormatUtil; -import oshi.util.Util; - -public class IndexAction extends WebBaseAction { - - public IndexAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/index", new IndexAction(controller)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - appendVersionInfo(response.getContent()); - appendHardwareInfo(response.getContent()); - getPageFooter(response.getContent()); - writeResponse(request, response); - } - - private void appendVersionInfo(StringBuilder buffer) { - buffer.append("

Version

"); - buffer.append("
version info
"); - buffer.append("Version: " + Version.DORIS_BUILD_VERSION + "
"); - buffer.append("Git: " + Version.DORIS_BUILD_HASH + "
"); - buffer.append("Build Info: " + Version.DORIS_BUILD_INFO + "
"); - buffer.append("Build Time: " + Version.DORIS_BUILD_TIME + "
"); - buffer.append("
"); - } - - private void appendHardwareInfo(StringBuilder buffer) { - List hardwareInfo = new ArrayList<>(); - SystemInfo si = new SystemInfo(); - OperatingSystem os = si.getOperatingSystem(); - HardwareAbstractionLayer hal = si.getHardware(); - CentralProcessor processor = hal.getProcessor(); - GlobalMemory memory = hal.getMemory(); - buffer.append("

Hardware Info

"); - buffer.append("
");
-        buffer.append(String.join("
", getOperatingSystem(os)) + "
"); - buffer.append(String.join("
", getProcessor(processor)) + "
"); - buffer.append(String.join("
", getMemory(memory)) + "
"); - buffer.append(String.join("
", getProcesses(os, memory)) + "
"); - buffer.append(String.join("
", getDisks(hal.getDiskStores())) + "
"); - buffer.append(String.join("
", getFileSystem(os.getFileSystem())) + "
"); - buffer.append(String.join("
", getNetworkInterfaces(hal.getNetworkIFs())) + "
"); - buffer.append(String.join("
", getNetworkParameters(os.getNetworkParams())) + "
"); - buffer.append("
"); - } - private List getOperatingSystem(OperatingSystem os) { - List osInfo = new ArrayList<>(); - osInfo.add(String.valueOf(os)); - osInfo.add("Booted: " + Instant.ofEpochSecond(os.getSystemBootTime())); - osInfo.add("Uptime: " + FormatUtil.formatElapsedSecs(os.getSystemUptime())); - osInfo.add("Running with" + (os.isElevated() ? "" : "out") + " elevated permissions."); - return osInfo; - } - private List getProcessor(CentralProcessor processor) { - List processorInfo = new ArrayList<>(); - processorInfo.add(String.valueOf(processor)); - processorInfo.add(" " + processor.getPhysicalPackageCount() + " physical CPU package(s)"); - processorInfo.add(" " + processor.getPhysicalProcessorCount() + " physical CPU core(s)"); - processorInfo.add(" " + processor.getLogicalProcessorCount() + " logical CPU(s)"); - - processorInfo.add("Identifier: " + processor.getIdentifier()); - processorInfo.add("ProcessorID: " + processor.getProcessorID()); - processorInfo.add("Context Switches/Interrupts: " + processor.getContextSwitches() - + " / " + processor.getInterrupts() + "\n"); - - long[] prevTicks = processor.getSystemCpuLoadTicks(); - long[][] prevProcTicks = processor.getProcessorCpuLoadTicks(); - processorInfo.add("CPU, IOWait, and IRQ ticks @ 0 sec:" + Arrays.toString(prevTicks)); - // Wait a second... - Util.sleep(1000); - long[] ticks = processor.getSystemCpuLoadTicks(); - processorInfo.add("CPU, IOWait, and IRQ ticks @ 1 sec:" + Arrays.toString(ticks)); - long user = ticks[CentralProcessor.TickType.USER.getIndex()] - prevTicks[CentralProcessor.TickType.USER.getIndex()]; - long nice = ticks[CentralProcessor.TickType.NICE.getIndex()] - prevTicks[CentralProcessor.TickType.NICE.getIndex()]; - long sys = ticks[CentralProcessor.TickType.SYSTEM.getIndex()] - prevTicks[CentralProcessor.TickType.SYSTEM.getIndex()]; - long idle = ticks[CentralProcessor.TickType.IDLE.getIndex()] - prevTicks[CentralProcessor.TickType.IDLE.getIndex()]; - long iowait = ticks[CentralProcessor.TickType.IOWAIT.getIndex()] - prevTicks[CentralProcessor.TickType.IOWAIT.getIndex()]; - long irq = ticks[CentralProcessor.TickType.IRQ.getIndex()] - prevTicks[CentralProcessor.TickType.IRQ.getIndex()]; - long softirq = ticks[CentralProcessor.TickType.SOFTIRQ.getIndex()] - prevTicks[CentralProcessor.TickType.SOFTIRQ.getIndex()]; - long steal = ticks[CentralProcessor.TickType.STEAL.getIndex()] - prevTicks[CentralProcessor.TickType.STEAL.getIndex()]; - long totalCpu = user + nice + sys + idle + iowait + irq + softirq + steal; - - processorInfo.add(String.format( - "User: %.1f%% Nice: %.1f%% System: %.1f%% Idle: %.1f%% IOwait: %.1f%% IRQ: %.1f%% SoftIRQ: %.1f%% Steal: %.1f%%", - 100d * user / totalCpu, 100d * nice / totalCpu, 100d * sys / totalCpu, 100d * idle / totalCpu, - 100d * iowait / totalCpu, 100d * irq / totalCpu, 100d * softirq / totalCpu, 100d * steal / totalCpu)); - processorInfo.add(String.format("CPU load: %.1f%%", - processor.getSystemCpuLoadBetweenTicks(prevTicks) * 100)); - double[] loadAverage = processor.getSystemLoadAverage(3); - processorInfo.add("CPU load averages:" + (loadAverage[0] < 0 ? " N/A" : String.format(" %.2f", loadAverage[0])) - + (loadAverage[1] < 0 ? " N/A" : String.format(" %.2f", loadAverage[1])) - + (loadAverage[2] < 0 ? " N/A" : String.format(" %.2f", loadAverage[2]))); - // per core CPU - StringBuilder procCpu = new StringBuilder("CPU load per processor:"); - double[] load = processor.getProcessorCpuLoadBetweenTicks(prevProcTicks); - for (double avg : load) { - procCpu.append(String.format(" %.1f%%", avg * 100)); - } - processorInfo.add(procCpu.toString()); - long freq = processor.getVendorFreq(); - if (freq > 0) { - processorInfo.add("Vendor Frequency: " + FormatUtil.formatHertz(freq)); - } - freq = processor.getMaxFreq(); - if (freq > 0) { - processorInfo.add("Max Frequency: " + FormatUtil.formatHertz(freq)); - } - long[] freqs = processor.getCurrentFreq(); - if (freqs[0] > 0) { - StringBuilder sb = new StringBuilder("Current Frequencies: "); - for (int i = 0; i < freqs.length; i++) { - if (i > 0) { - sb.append(", "); - } - sb.append(FormatUtil.formatHertz(freqs[i])); - } - processorInfo.add(sb.toString()); - } - return processorInfo; - } - - private List getMemory(GlobalMemory memory) { - List memoryInfo = new ArrayList<>(); - memoryInfo.add("Memory: " + FormatUtil.formatBytes(memory.getAvailable()) + "/" - + FormatUtil.formatBytes(memory.getTotal())); - VirtualMemory vm = memory.getVirtualMemory(); - memoryInfo.add("Swap used: " + FormatUtil.formatBytes(vm.getSwapUsed()) + "/" - + FormatUtil.formatBytes(vm.getSwapTotal())); - return memoryInfo; - } - - private List getProcesses(OperatingSystem os, GlobalMemory memory) { - List processInfo = new ArrayList<>(); - processInfo.add("Processes: " + os.getProcessCount() + ", Threads: " + os.getThreadCount()); - // Sort by highest CPU - List procs = Arrays.asList(os.getProcesses(5, OperatingSystem.ProcessSort.CPU)); - - processInfo.add(" PID %CPU %MEM VSZ RSS Name"); - for (int i = 0; i < procs.size() && i < 5; i++) { - OSProcess p = procs.get(i); - processInfo.add(String.format(" %5d %5.1f %4.1f %9s %9s %s", p.getProcessID(), - 100d * (p.getKernelTime() + p.getUserTime()) / p.getUpTime(), - 100d * p.getResidentSetSize() / memory.getTotal(), FormatUtil.formatBytes(p.getVirtualSize()), - FormatUtil.formatBytes(p.getResidentSetSize()), p.getName())); - } - return processInfo; - } - private List getDisks(HWDiskStore[] diskStores) { - List diskInfo = new ArrayList<>(); - diskInfo.add("Disks:"); - for (HWDiskStore disk : diskStores) { - boolean readwrite = disk.getReads() > 0 || disk.getWrites() > 0; - diskInfo.add(String.format(" %s: (model: %s - S/N: %s) size: %s, reads: %s (%s), writes: %s (%s), xfer: %s ms", - disk.getName(), disk.getModel(), disk.getSerial(), - disk.getSize() > 0 ? FormatUtil.formatBytesDecimal(disk.getSize()) : "?", - readwrite ? disk.getReads() : "?", readwrite ? FormatUtil.formatBytes(disk.getReadBytes()) : "?", - readwrite ? disk.getWrites() : "?", readwrite ? FormatUtil.formatBytes(disk.getWriteBytes()) : "?", - readwrite ? disk.getTransferTime() : "?")); - HWPartition[] partitions = disk.getPartitions(); - for (HWPartition part : partitions) { - diskInfo.add(String.format(" |-- %s: %s (%s) Maj:Min=%d:%d, size: %s%s", part.getIdentification(), - part.getName(), part.getType(), part.getMajor(), part.getMinor(), - FormatUtil.formatBytesDecimal(part.getSize()), - part.getMountPoint().isEmpty() ? "" : " @ " + part.getMountPoint())); - } - } - return diskInfo; - } - - private List getFileSystem(FileSystem fileSystem) { - List fsInfo = new ArrayList<>(); - fsInfo.add("File System:"); - - fsInfo.add(String.format(" File Descriptors: %d/%d", fileSystem.getOpenFileDescriptors(), - fileSystem.getMaxFileDescriptors())); - - OSFileStore[] fsArray = fileSystem.getFileStores(); - for (OSFileStore fs : fsArray) { - long usable = fs.getUsableSpace(); - long total = fs.getTotalSpace(); - fsInfo.add(String.format(" %s (%s) [%s] %s of %s free (%.1f%%), %s of %s files free (%.1f%%) is %s " + - (fs.getLogicalVolume() != null && fs.getLogicalVolume().length() > 0 ? "[%s]" : "%s") + - " and is mounted at %s", - fs.getName(), fs.getDescription().isEmpty() ? "file system" : fs.getDescription(), fs.getType(), - FormatUtil.formatBytes(usable), FormatUtil.formatBytes(fs.getTotalSpace()), 100d * usable / total, - FormatUtil.formatValue(fs.getFreeInodes(), ""), FormatUtil.formatValue(fs.getTotalInodes(), ""), - 100d * fs.getFreeInodes() / fs.getTotalInodes(), fs.getVolume(), fs.getLogicalVolume(), - fs.getMount())); - } - return fsInfo; - } - - private List getNetworkInterfaces(NetworkIF[] networkIFs) { - List getNetwork = new ArrayList<>(); - getNetwork.add("Network interfaces:"); - for (NetworkIF net : networkIFs) { - getNetwork.add(String.format(" Name: %s (%s)", net.getName(), net.getDisplayName())); - getNetwork.add(String.format(" MAC Address: %s", net.getMacaddr())); - getNetwork.add(String.format(" MTU: %s, Speed: %s", net.getMTU(), FormatUtil.formatValue(net.getSpeed(), "bps"))); - getNetwork.add(String.format(" IPv4: %s", Arrays.toString(net.getIPv4addr()))); - getNetwork.add(String.format(" IPv6: %s", Arrays.toString(net.getIPv6addr()))); - boolean hasData = net.getBytesRecv() > 0 || net.getBytesSent() > 0 || net.getPacketsRecv() > 0 - || net.getPacketsSent() > 0; - getNetwork.add(String.format(" Traffic: received %s/%s%s; transmitted %s/%s%s", - hasData ? net.getPacketsRecv() + " packets" : "?", - hasData ? FormatUtil.formatBytes(net.getBytesRecv()) : "?", - hasData ? " (" + net.getInErrors() + " err)" : "", - hasData ? net.getPacketsSent() + " packets" : "?", - hasData ? FormatUtil.formatBytes(net.getBytesSent()) : "?", - hasData ? " (" + net.getOutErrors() + " err)" : "")); - } - return getNetwork; - } - - private List getNetworkParameters(NetworkParams networkParams) { - List networkParameterInfo = new ArrayList<>(); - networkParameterInfo.add("Network parameters:"); - networkParameterInfo.add(String.format(" Host name: %s", networkParams.getHostName())); - networkParameterInfo.add(String.format(" Domain name: %s", networkParams.getDomainName())); - networkParameterInfo.add(String.format(" DNS servers: %s", Arrays.toString(networkParams.getDnsServers()))); - networkParameterInfo.add(String.format(" IPv4 Gateway: %s", networkParams.getIpv4DefaultGateway())); - networkParameterInfo.add(String.format(" IPv6 Gateway: %s", networkParams.getIpv6DefaultGateway())); - return networkParameterInfo; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/LogAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/LogAction.java deleted file mode 100644 index 6185debfc7b108..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/LogAction.java +++ /dev/null @@ -1,168 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.common.Config; -import org.apache.doris.common.Log4jConfig; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import io.netty.handler.codec.http.HttpMethod; - -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.util.List; - -public class LogAction extends WebBaseAction { - private static final Logger LOG = LogManager.getLogger(LogAction.class); - private static long WEB_LOG_BYTES = 1024 * 1024; // 1MB - - private String addVerboseName; - private String delVerboseName; - - public LogAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/log", new LogAction(controller)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - - // get parameters - addVerboseName = request.getSingleParameter("add_verbose"); - delVerboseName = request.getSingleParameter("del_verbose"); - LOG.info("add verbose name: {}, del verbose name: {}", addVerboseName, delVerboseName); - - appendLogConf(response.getContent()); - appendLogInfo(response.getContent()); - - getPageFooter(response.getContent()); - writeResponse(request, response); - } - - private void appendLogConf(StringBuilder buffer) { - buffer.append("

Log Configuration

"); - try { - Log4jConfig.Tuple configs = Log4jConfig.updateLogging(null, null, null); - if (!Strings.isNullOrEmpty(addVerboseName)) { - addVerboseName = addVerboseName.trim(); - List verboseNames = Lists.newArrayList(configs.y); - if (!verboseNames.contains(addVerboseName)) { - verboseNames.add(addVerboseName); - configs = Log4jConfig.updateLogging(null, verboseNames.toArray(new String[verboseNames.size()]), - null); - } - } - if (!Strings.isNullOrEmpty(delVerboseName)) { - delVerboseName = delVerboseName.trim(); - List verboseNames = Lists.newArrayList(configs.y); - if (verboseNames.contains(delVerboseName)) { - verboseNames.remove(delVerboseName); - configs = Log4jConfig.updateLogging(null, verboseNames.toArray(new String[verboseNames.size()]), - null); - } - } - - buffer.append("Level: " + configs.x + "
"); - buffer.append("Verbose Names: " + StringUtils.join(configs.y, ",") + "
"); - buffer.append("Audit Names: " + StringUtils.join(configs.z, ",") + "
"); - appendUpdateVerboseButton(buffer, "add_verbose"); - appendUpdateVerboseButton(buffer, "del_verbose"); - } catch (IOException e) { - LOG.error(e); - e.printStackTrace(); - } - } - - private void appendUpdateVerboseButton(StringBuilder buffer, String type) { - String placeHolder = ""; - String buttonName = ""; - if (type.equals("add_verbose")) { - placeHolder = "new verbose name"; - buttonName = "Add"; - } else if (type.equals("del_verbose")) { - placeHolder = "del verbose name"; - buttonName = "Delete"; - } else { - return; - } - - buffer.append("
" - + "
" - + "
" - + " " - + " " - + " " - + " \n" - + "
\n" - + "
" - + "
"); - } - - private void appendLogInfo(StringBuilder buffer) { - buffer.append("

Log Contents

"); - - final String logPath = Config.sys_log_dir + "/fe.warn.log"; - buffer.append("Log path is: " + logPath + "
"); - - RandomAccessFile raf = null; - try { - raf = new RandomAccessFile(logPath, "r"); - long fileSize = raf.length(); - long startPos = fileSize < WEB_LOG_BYTES ? 0L : fileSize - WEB_LOG_BYTES; - long webContentLength = fileSize < WEB_LOG_BYTES ? fileSize : WEB_LOG_BYTES; - raf.seek(startPos); - buffer.append("

Showing last " + webContentLength + " bytes of log

"); - buffer.append("
");
-            String fileBuffer = null;
-            while ((fileBuffer = raf.readLine()) != null) {
-                buffer.append(fileBuffer).append("\n");
-            }
-            buffer.append("
"); - } catch (FileNotFoundException e) { - buffer.append("

Couldn't open log file: " - + logPath + "

"); - } catch (IOException e) { - buffer.append("

Failed to read log file: " - + logPath + "

"); - } finally { - try { - if (raf != null) { - raf.close(); - } - } catch (IOException e) { - LOG.warn("fail to close log file: " + logPath, e); - } - } - } -} - diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/NotFoundAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/NotFoundAction.java deleted file mode 100644 index 4cb6d28dd34a44..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/NotFoundAction.java +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -public class NotFoundAction extends WebBaseAction { - - public NotFoundAction(ActionController controller) { - super(controller); - } - - @Override - public boolean needAdmin() { - return false; - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - - response.appendContent("this is 404 page."); - - getPageFooter(response.getContent()); - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/404", new NotFoundAction(controller)); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/QueryAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/QueryAction.java deleted file mode 100644 index e3e4f6af1ac79a..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/QueryAction.java +++ /dev/null @@ -1,127 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.common.util.ProfileManager; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.List; - -public class QueryAction extends WebBaseAction { - private static final Logger LOG = LogManager.getLogger(QueryAction.class); - - public QueryAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/query", new QueryAction(controller)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - - addFinishedQueryInfo(response.getContent()); - - getPageFooter(response.getContent()); - - writeResponse(request, response); - } - - // Note: we do not show 'Query ID' column in web page - private void addFinishedQueryInfo(StringBuilder buffer) { - buffer.append("

Finished Queries

"); - buffer.append("

You need to set session variable by executing 'set is_report_success=true' before executing the SQL, to view the profile

"); - buffer.append("

This table lists the latest 100 queries

"); - - List> finishedQueries = ProfileManager.getInstance().getAllQueries(); - List columnHeaders = ProfileManager.PROFILE_HEADERS; - int queryIdIndex = 0; // the first column is 'Query ID' by default - for (int i = 0; i < columnHeaders.size(); ++i) { - if (columnHeaders.get(i).equals(ProfileManager.QUERY_ID)) { - queryIdIndex = i; - } - } - appendFinishedQueryTableHeader(buffer, columnHeaders, queryIdIndex); - appendFinishedQueryTableBody(buffer, finishedQueries, columnHeaders, queryIdIndex); - appendTableFooter(buffer); - } - - private void appendFinishedQueryTableHeader( - StringBuilder buffer, - final List columnHeaders, - int queryIdIndex) { - buffer.append(""); - for (int i = 0; i < columnHeaders.size(); ++i) { - if (i == queryIdIndex) { - continue; - } - buffer.append(""); - } - - buffer.append(""); - buffer.append(""); - } - - private void appendFinishedQueryTableBody( - StringBuilder buffer, - List> bodies, - List columnHeaders, - int queryIdIndex) { - for ( List row : bodies) { - buffer.append(""); - String queryId = row.get(queryIdIndex); - - for (int i = 0; i < row.size(); ++i) { - if (i == queryIdIndex) { - continue; - } - buffer.append(""); - } - - // add 'Profile' column - if (Strings.isNullOrEmpty(queryId)) { - LOG.warn("query id is null or empty, maybe we forget to push it " - + "into array when generate profile info."); - buffer.append(""); - } else { - buffer.append(""); - } - - buffer.append(""); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/QueryProfileAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/QueryProfileAction.java deleted file mode 100644 index 69ac168587e09d..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/QueryProfileAction.java +++ /dev/null @@ -1,68 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.common.util.ProfileManager; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -public class QueryProfileAction extends WebBaseAction { - - public QueryProfileAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/query_profile", new QueryProfileAction(controller)); - } - - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - - String queryId = request.getSingleParameter("query_id"); - if (Strings.isNullOrEmpty(queryId)) { - response.appendContent(""); - response.appendContent("

Must specify a query_id[]

"); - } - - String queryProfileStr = ProfileManager.getInstance().getProfile(queryId); - if (queryProfileStr != null) { - appendQueryProfile(response.getContent(), queryProfileStr); - getPageFooter(response.getContent()); - writeResponse(request, response); - } else { - appendQueryProfile(response.getContent(), "query id " + queryId + " not found."); - getPageFooter(response.getContent()); - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); - } - } - - private void appendQueryProfile(StringBuilder buffer, String queryProfileStr) { - buffer.append("
");
-        buffer.append(queryProfileStr);
-        buffer.append("
"); - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/SessionAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/SessionAction.java deleted file mode 100644 index 435cacfa481870..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/SessionAction.java +++ /dev/null @@ -1,85 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.service.ExecuteEnv; - -import com.google.common.collect.Lists; - -import java.util.ArrayList; -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; - -public class SessionAction extends WebBaseAction { - // we make - private static final ArrayList SESSION_TABLE_HEADER = Lists.newArrayList(); - - static { - SESSION_TABLE_HEADER.add("Id"); - SESSION_TABLE_HEADER.add("User"); - SESSION_TABLE_HEADER.add("Host"); - SESSION_TABLE_HEADER.add("Cluster"); - SESSION_TABLE_HEADER.add("Db"); - SESSION_TABLE_HEADER.add("Command"); - SESSION_TABLE_HEADER.add("Time"); - SESSION_TABLE_HEADER.add("State"); - SESSION_TABLE_HEADER.add("Info"); - } - - public SessionAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/session", new SessionAction(controller)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - appendSessionInfo(response.getContent()); - getPageFooter(response.getContent()); - writeResponse(request, response); - } - - private void appendSessionInfo(StringBuilder buffer) { - buffer.append("

Session Info

"); - - List threadInfos = ExecuteEnv.getInstance().getScheduler().listConnection("root"); - List> rowSet = Lists.newArrayList(); - long nowMs = System.currentTimeMillis(); - for (ConnectContext.ThreadInfo info : threadInfos) { - rowSet.add(info.toRow(nowMs)); - } - - buffer.append("

This page lists the session info, there are " - + rowSet.size() - + " active sessions.

"); - - appendTableHeader(buffer, SESSION_TABLE_HEADER); - appendTableBody(buffer, rowSet); - appendTableFooter(buffer); - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/StaticResourceAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/StaticResourceAction.java deleted file mode 100644 index cbd540d29a4827..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/StaticResourceAction.java +++ /dev/null @@ -1,242 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.PaloFe; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import com.google.common.base.Strings; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.File; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; -import java.util.GregorianCalendar; -import java.util.Locale; -import java.util.TimeZone; -import java.util.regex.Pattern; - -import javax.activation.MimetypesFileTypeMap; - -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -/** - * A simple handler that serves incoming HTTP requests to send their respective - * HTTP responses. It also implements {@code 'If-Modified-Since'} header to - * take advantage of browser cache, as described in - * RFC 2616. - * - *

How Browser Caching Works

- * - * Web browser caching works with HTTP headers as illustrated by the following - * sample: - *
    - *
  1. Request #1 returns the content of {@code /file1.txt}.
  2. - *
  3. Contents of {@code /file1.txt} is cached by the browser.
  4. - *
  5. Request #2 for {@code /file1.txt} does return the contents of the - * file again. Rather, a 304 Not Modified is returned. This tells the - * browser to use the contents stored in its cache.
  6. - *
  7. The server knows the file has not been modified because the - * {@code If-Modified-Since} date is the same as the file's last - * modified date.
  8. - *
- * - *
- * Request #1 Headers
- * ===================
- * GET /file1.txt HTTP/1.1
- *
- * Response #1 Headers
- * ===================
- * HTTP/1.1 200 OK
- * Date:               Tue, 01 Mar 2011 22:44:26 GMT
- * Last-Modified:      Wed, 30 Jun 2010 21:36:48 GMT
- * Expires:            Tue, 01 Mar 2012 22:44:26 GMT
- * Cache-Control:      private, max-age=31536000
- *
- * Request #2 Headers
- * ===================
- * GET /file1.txt HTTP/1.1
- * If-Modified-Since:  Wed, 30 Jun 2010 21:36:48 GMT
- *
- * Response #2 Headers
- * ===================
- * HTTP/1.1 304 Not Modified
- * Date:               Tue, 01 Mar 2011 22:44:28 GMT
- *
- * 
- * - * We use parameter named 'res' to specify the static resource path, it relative to the - * root path of http server. - */ -public class StaticResourceAction extends WebBaseAction { - private static final Logger LOG = LogManager.getLogger(StaticResourceAction.class); - - public static final String HTTP_DATE_FORMAT = "EEE, dd MMM yyyy HH:mm:ss zzz"; - public static final String HTTP_DATE_GMT_TIMEZONE = "GMT"; - public static final int HTTP_CACHE_SECONDS = 2592000; - - public MimetypesFileTypeMap mimeTypesMap; - public String rootDir; - - public StaticResourceAction(ActionController controller, String rootDir) { - super(controller); - mimeTypesMap = new MimetypesFileTypeMap(); - mimeTypesMap.addMimeTypes("text/html html htm"); - // According to RFC 4329(http://tools.ietf.org/html/rfc4329#section-7.2), the MIME type of - // javascript script is 'application/javascript' - mimeTypesMap.addMimeTypes("application/javascript js"); - // mimeTypesMap.addMimeTypes("text/javascript js"); - mimeTypesMap.addMimeTypes("text/css css"); - - this.rootDir = rootDir; - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - String httpDir = PaloFe.DORIS_HOME_DIR + "/webroot"; - StaticResourceAction action = new StaticResourceAction(controller, httpDir + "/static"); - controller.registerHandler(HttpMethod.GET, "/static/js", action); - controller.registerHandler(HttpMethod.GET, "/static/css", action); - controller.registerHandler(HttpMethod.GET, "/static", action); - controller.registerHandler(HttpMethod.GET, "/static/resource", action); - controller.registerHandler(HttpMethod.GET, "/static/images", action); - controller.registerHandler(HttpMethod.GET, "/static/Bootstrap-3.3.7/fonts/", action); - - StaticResourceAction action2 = new StaticResourceAction(controller, "webroot"); - controller.registerHandler(HttpMethod.GET, "/static_test", action2); - } - - @Override - public boolean needAdmin() { - return false; - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - String resourcePath = request.getSingleParameter("res"); - if (Strings.isNullOrEmpty(resourcePath)) { - LOG.error("Wrong request without 'res' parameter. url: {}", - request.getRequest().uri()); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - resourcePath = sanitizePath(resourcePath); - if (resourcePath == null) { - LOG.error("Close this request because of risk factor in 'res' parameter.url: {}", - request.getRequest().uri()); - writeResponse(request, response, HttpResponseStatus.FORBIDDEN); - return; - } - - String resourceAbsolutePath = rootDir + File.separator + resourcePath; - File resFile = new File(resourceAbsolutePath); - LOG.debug("resAbsolutePath: {}", resourceAbsolutePath); - - if (!resFile.exists() || resFile.isHidden() || resFile.isDirectory()) { - LOG.error("Request with wrong path. url: {}", request.getRequest().uri()); - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); - return; - } - if (!resFile.isFile()) { - LOG.error("Wrong request: not normal file. url: {}", request.getRequest().uri()); - writeResponse(request, response, HttpResponseStatus.FORBIDDEN); - return; - } - - // Cache validation - String ifModifiedSince = request.getRequest() - .headers().get(HttpHeaderNames.IF_MODIFIED_SINCE.toString()); - if (!Strings.isNullOrEmpty(ifModifiedSince)) { - SimpleDateFormat dateFormatter = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US); - Date ifModifiedSinceDate; - try { - ifModifiedSinceDate = dateFormatter.parse(ifModifiedSince); - // Only compare up to the second because the datetime format we send to the client - // does not have milliseconds - long ifModifiedSinceDateSeconds = ifModifiedSinceDate.getTime() / 1000; - long fileLastModifiedSeconds = resFile.lastModified() / 1000; - if (ifModifiedSinceDateSeconds == fileLastModifiedSeconds) { - writeResponse(request, response, HttpResponseStatus.NOT_MODIFIED); - return; - } - } catch (ParseException e) { - LOG.error("Fail to analyse IF_MODIFIED_SINCE header: ", e); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - } - - response.updateHeader(HttpHeaderNames.CONTENT_TYPE.toString(), getContentType(resourceAbsolutePath)); - setDateAndCacheHeaders(response, resFile); - - writeObjectResponse(request, response, HttpResponseStatus.OK, resFile, resFile.getName(), false); - } - - - - // Gets the content type header for the HTTP Response - private String getContentType(String filename) { - return mimeTypesMap.getContentType(filename); - } - - private void setDateAndCacheHeaders(BaseResponse response, File fileToCache) { - SimpleDateFormat dateFormatter = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US); - dateFormatter.setTimeZone(TimeZone.getTimeZone(HTTP_DATE_GMT_TIMEZONE)); - - // Date header - Calendar time = new GregorianCalendar(); - response.updateHeader(HttpHeaderNames.DATE.toString(), dateFormatter.format(time.getTime())); - - // Add cache headers - time.add(Calendar.SECOND, HTTP_CACHE_SECONDS); - response.updateHeader(HttpHeaderNames.EXPIRES.toString(), dateFormatter.format(time.getTime())); - response.updateHeader(HttpHeaderNames.CACHE_CONTROL.toString(), "private, max-age=" + HTTP_CACHE_SECONDS); - response.updateHeader(HttpHeaderNames.LAST_MODIFIED.toString(), - dateFormatter.format(new Date(fileToCache.lastModified()))); - } - - private static final Pattern INSECURE_URI = Pattern.compile(".*[<>&\"].*"); - - private String sanitizePath(String path) { - if (Strings.isNullOrEmpty(path)) { - return null; - } - // Convert file separators - String newPath = path.replace('/', File.separatorChar); - - // Simplistic dumb security check. - if (newPath.contains(File.separator + '.') - || newPath.contains('.' + File.separator) - || newPath.charAt(0) == '.' - || newPath.charAt(newPath.length() - 1) == '.' - || INSECURE_URI.matcher(newPath).matches()) { - return null; - } - - return newPath; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/SystemAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/SystemAction.java deleted file mode 100644 index 3c941d9c7f3cdc..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/SystemAction.java +++ /dev/null @@ -1,184 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import io.netty.handler.codec.http.HttpMethod; - -import org.apache.doris.analysis.RedirectStatus; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; -import org.apache.doris.common.proc.ProcDirInterface; -import org.apache.doris.common.proc.ProcNodeInterface; -import org.apache.doris.common.proc.ProcResult; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.MasterOpExecutor; -import org.apache.doris.qe.OriginStatement; -import org.apache.doris.qe.ShowResultSet; - -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; - -import org.apache.commons.validator.routines.UrlValidator; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.List; -import java.util.stream.Collectors; - -public class SystemAction extends WebBaseAction { - private static final Logger LOG = LogManager.getLogger(SystemAction.class); - - public SystemAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/system", new SystemAction(controller)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - - String currentPath = request.getSingleParameter("path"); - if (Strings.isNullOrEmpty(currentPath)) { - currentPath = "/"; - } - appendSystemInfo(response.getContent(), currentPath, currentPath); - - getPageFooter(response.getContent()); - writeResponse(request, response); - } - - private void appendSystemInfo(StringBuilder buffer, String procPath, String path) { - buffer.append("

System Info

"); - buffer.append("

This page lists the system info, like /proc in Linux.

"); - buffer.append("

Current path: " + path + "" - + "Parent Dir


"); - - ProcNodeInterface procNode = getProcNode(procPath); - if (procNode == null) { - buffer.append("

No such proc path[" + path + "]

"); - return; - } - boolean isDir = (procNode instanceof ProcDirInterface); - - List columnNames = null; - List> rows = null; - if (!Catalog.getCurrentCatalog().isMaster() && !Config.enable_bdbje_debug_mode) { - // forward to master - String showProcStmt = "SHOW PROC \"" + procPath + "\""; - MasterOpExecutor masterOpExecutor = new MasterOpExecutor(new OriginStatement(showProcStmt, 0), - ConnectContext.get(), RedirectStatus.FORWARD_NO_SYNC, true); - try { - masterOpExecutor.execute(); - } catch (Exception e) { - LOG.warn("Fail to forward. ", e); - buffer.append("

Failed to forward request to master

"); - return; - } - - ShowResultSet resultSet = masterOpExecutor.getProxyResultSet(); - if (resultSet == null) { - buffer.append("

Failed to get result from master

"); - return; - } - - columnNames = resultSet.getMetaData().getColumns().stream().map(c -> c.getName()).collect( - Collectors.toList()); - rows = resultSet.getResultRows(); - } else { - ProcResult result; - try { - result = procNode.fetchResult(); - } catch (AnalysisException e) { - buffer.append("

The result is null, " - + "maybe haven't be implemented completely[" + e.getMessage() + "], please check.

"); - buffer.append("

" - + "INFO: ProcNode type is [" + procNode.getClass().getName() - + "]

"); - return; - } - - columnNames = result.getColumnNames(); - rows = result.getRows(); - } - - Preconditions.checkNotNull(columnNames); - Preconditions.checkNotNull(rows); - - appendTableHeader(buffer, columnNames); - appendSystemTableBody(buffer, rows, isDir, path); - appendTableFooter(buffer); - } - - private void appendSystemTableBody(StringBuilder buffer, List> rows, boolean isDir, String path) { - UrlValidator validator = new UrlValidator(); - for ( List strList : rows) { - buffer.append("
"); - int columnIndex = 1; - for (String str : strList) { - buffer.append(""); - ++columnIndex; - } - buffer.append(""); - } - } - - // some example: - // '/' => '/' - // '///aaa' => '///' - // '/aaa/bbb///' => '/aaa' - // '/aaa/bbb/ccc' => '/aaa/bbb' - // ATTN: the root path's parent is itself. - private String getParentPath(String path) { - int lastSlashIndex = path.length() - 1; - while (lastSlashIndex > 0) { - int tempIndex = path.lastIndexOf('/', lastSlashIndex); - if (tempIndex > 0) { - if (tempIndex == lastSlashIndex) { - lastSlashIndex = tempIndex - 1; - continue; - } else if (tempIndex < lastSlashIndex) { // '//aaa/bbb' - lastSlashIndex = tempIndex; - return path.substring(0, lastSlashIndex); - } - } - } - return "/"; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/VariableAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/VariableAction.java deleted file mode 100644 index 0df3428f6f8e8b..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/VariableAction.java +++ /dev/null @@ -1,84 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.analysis.SetType; -import org.apache.doris.common.Config; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.qe.VariableMgr; - -import com.google.common.collect.Lists; - -import io.netty.handler.codec.http.HttpMethod; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; - -public class VariableAction extends WebBaseAction { - - public VariableAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/variable", new VariableAction(controller)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - getPageHeader(request, response.getContent()); - - appendConfigureInfo(response.getContent()); - appendVariableInfo(response.getContent()); - - getPageFooter(response.getContent()); - writeResponse(request, response); - } - - public void appendConfigureInfo(StringBuilder buffer) { - buffer.append("

Configure Info

"); - buffer.append("
");
-        HashMap confmap;
-        try {
-            confmap = Config.dump();
-            List keyList = Lists.newArrayList(confmap.keySet());
-            Collections.sort(keyList);
-            for (String key : keyList) {
-                buffer.append(key + "=" + confmap.get(key) + "\n");
-            }
-        } catch (Exception e) {
-            buffer.append("read conf exception" + e.toString());
-        }
-        buffer.append("
"); - } - - private void appendVariableInfo(StringBuilder buffer) { - buffer.append("

Variable Info

"); - buffer.append("
");
-        List> variableInfo = VariableMgr.dump(SetType.GLOBAL, null, null);
-        for (List list : variableInfo) {
-            buffer.append(list.get(0) + "=" + list.get(1) + "\n");
-        }
-        buffer.append("
"); - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/WebBaseAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/WebBaseAction.java deleted file mode 100644 index e710d68cfeaf6b..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/action/WebBaseAction.java +++ /dev/null @@ -1,374 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.action; - -import org.apache.doris.analysis.CompoundPredicate.Operator; -import org.apache.doris.analysis.UserIdentity; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; -import org.apache.doris.common.proc.ProcNodeInterface; -import org.apache.doris.common.proc.ProcService; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseAction; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.HttpAuthManager; -import org.apache.doris.http.HttpAuthManager.SessionValue; -import org.apache.doris.http.UnauthorizedException; -import org.apache.doris.http.rest.RestBaseResult; -import org.apache.doris.mysql.privilege.PaloPrivilege; -import org.apache.doris.mysql.privilege.PrivBitSet; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.system.SystemInfoService; - -import com.google.common.base.Strings; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.List; -import java.util.UUID; - -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.codec.http.cookie.DefaultCookie; - -public class WebBaseAction extends BaseAction { - private static final Logger LOG = LogManager.getLogger(WebBaseAction.class); - - protected static final String PALO_SESSION_ID = "PALO_SESSION_ID"; - private static final long PALO_SESSION_EXPIRED_TIME = 3600 * 24; // one day - - protected static final String PAGE_HEADER = "" - + "" - + "" - + " Apache Doris(Incubating)" - + " " - + " " - + " " - + " " - + " " - - + " " - + " " - + " " - - + " " - - + " " - + "" - + ""; - protected static final String PAGE_FOOTER = ""; - protected static final String NAVIGATION_BAR_PREFIX = - " " - + "
"; - - public WebBaseAction(ActionController controller) { - super(controller); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - if (!checkAuthWithCookie(request, response)) { - return; - } - - HttpMethod method = request.getRequest().method(); - if (method.equals(HttpMethod.GET)) { - executeGet(request, response); - } else if (method.equals(HttpMethod.POST)) { - executePost(request, response); - } else { - response.appendContent(new RestBaseResult("HTTP method is not allowed: " + method.name()).toJson()); - writeResponse(request, response, HttpResponseStatus.METHOD_NOT_ALLOWED); - } - } - - // Sub Action class should override this method - public void executeGet(BaseRequest request, BaseResponse response) { - response.appendContent(new RestBaseResult("Not implemented").toJson()); - writeResponse(request, response, HttpResponseStatus.NOT_IMPLEMENTED); - } - - // Sub Action class should override this method - public void executePost(BaseRequest request, BaseResponse response) { - response.appendContent(new RestBaseResult("Not implemented").toJson()); - writeResponse(request, response, HttpResponseStatus.NOT_IMPLEMENTED); - } - - // We first check cookie, if not admin, we check http's authority header - private boolean checkAuthWithCookie(BaseRequest request, BaseResponse response) { - if (!needPassword()) { - return true; - } - - if (checkCookie(request, response)) { - return true; - } - - // cookie is invalid. - ActionAuthorizationInfo authInfo; - try { - authInfo = getAuthorizationInfo(request); - UserIdentity currentUser = checkPassword(authInfo); - if (needAdmin()) { - checkGlobalAuth(currentUser, PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, - PaloPrivilege.NODE_PRIV), Operator.OR)); - } - request.setAuthorized(true); - SessionValue value = new SessionValue(); - value.currentUser = currentUser; - addSession(request, response, value); - - ConnectContext ctx = new ConnectContext(null); - ctx.setQualifiedUser(authInfo.fullUserName); - ctx.setRemoteIP(authInfo.remoteIp); - ctx.setCurrentUserIdentity(currentUser); - ctx.setCatalog(Catalog.getCurrentCatalog()); - ctx.setCluster(SystemInfoService.DEFAULT_CLUSTER); - ctx.setThreadLocalInfo(); - - return true; - } catch (UnauthorizedException e) { - response.appendContent("Authentication Failed.
" + e.getMessage()); - writeAuthResponse(request, response); - return false; - } - } - - private boolean checkCookie(BaseRequest request, BaseResponse response) { - String sessionId = request.getCookieValue(PALO_SESSION_ID); - HttpAuthManager authMgr = HttpAuthManager.getInstance(); - if (!Strings.isNullOrEmpty(sessionId)) { - SessionValue sessionValue = authMgr.getSessionValue(sessionId); - if (sessionValue == null) { - return false; - } - if (Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(sessionValue.currentUser, - PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, - PaloPrivilege.NODE_PRIV), - Operator.OR))) { - response.updateCookieAge(request, PALO_SESSION_ID, PALO_SESSION_EXPIRED_TIME); - request.setAuthorized(true); - - ConnectContext ctx = new ConnectContext(null); - ctx.setQualifiedUser(sessionValue.currentUser.getQualifiedUser()); - ctx.setRemoteIP(request.getHostString()); - ctx.setCurrentUserIdentity(sessionValue.currentUser); - ctx.setCatalog(Catalog.getCurrentCatalog()); - ctx.setCluster(SystemInfoService.DEFAULT_CLUSTER); - ctx.setThreadLocalInfo(); - return true; - } - } - return false; - } - - // return true if this Action need to check password. - // Currently, all sub actions need to check password except for MetaBaseAction. - // if needPassword() is false, then needAdmin() should also return false - public boolean needPassword() { - return true; - } - - // return true if this Action need Admin privilege. - public boolean needAdmin() { - return true; - } - - protected void writeAuthResponse(BaseRequest request, BaseResponse response) { - response.updateHeader(HttpHeaderNames.WWW_AUTHENTICATE.toString(), "Basic realm=\"\""); - writeResponse(request, response, HttpResponseStatus.UNAUTHORIZED); - } - - protected void writeResponse(BaseRequest request, BaseResponse response) { - writeResponse(request, response, HttpResponseStatus.OK); - } - - protected void addSession(BaseRequest request, BaseResponse response, SessionValue value) { - String key = UUID.randomUUID().toString(); - DefaultCookie cookie = new DefaultCookie(PALO_SESSION_ID, key); - cookie.setMaxAge(PALO_SESSION_EXPIRED_TIME); - response.addCookie(cookie); - HttpAuthManager.getInstance().addSessionValue(key, value); - } - - protected void getPageHeader(BaseRequest request, StringBuilder sb) { - String newPageHeaderString = PAGE_HEADER; - newPageHeaderString = newPageHeaderString.replaceAll("Apache Doris", - "" + Config.cluster_name + ""); - - sb.append(newPageHeaderString); - sb.append(NAVIGATION_BAR_PREFIX); - - if (request.isAuthorized()) { - sb.append("
  • ") - .append("system") - .append("
  • "); - sb.append("
  • ") - .append("backends") - .append("
  • "); - sb.append("
  • ") - .append("logs") - .append("
  • "); - sb.append("
  • ") - .append("queries") - .append("
  • "); - sb.append("
  • ") - .append("sessions") - .append("
  • "); - sb.append("
  • ") - .append("variables") - .append("
  • "); - sb.append("
  • ") - .append("ha") - .append("
  • "); - } - sb.append("
  • ") - .append("help") - .append("
  • "); - - sb.append(NAVIGATION_BAR_SUFFIX); - } - - protected void getPageFooter(StringBuilder sb) { - sb.append(PAGE_FOOTER); - } - - // Note: DO NOT omit '
    ', because it is useful in 'datatable' plugin. - protected void appendTableHeader(StringBuilder buffer, List columnHeaders) { - buffer.append("
    " + columnHeaders.get(i) + "Profile
    "); - buffer.append(row.get(i)); - buffer.append("Empty Query ID"); - buffer.append("Profile"); - buffer.append("
    "); - if (isDir && columnIndex == 1) { - String escapeStr = str.replace("%", "%25"); - buffer.append(""); - buffer.append(str); - buffer.append(""); - } else if (validator.isValid(str)) { - buffer.append(""); - buffer.append("URL"); - buffer.append(""); - } else { - buffer.append(str.replaceAll("\\n", "
    ")); - } - buffer.append("
    "); - buffer.append(" "); - for (String str : columnHeaders) { - buffer.append(""); - } - buffer.append(" "); - } - - protected void appendTableBody(StringBuilder buffer, List> bodies) { - buffer.append(""); - for ( List row : bodies) { - buffer.append(""); - for (String column : row) { - buffer.append(""); - } - buffer.append(""); - } - buffer.append(""); - } - - protected void appendTableFooter(StringBuilder buffer) { - buffer.append("
    " + str + "
    "); - buffer.append(column); - buffer.append("
    "); - } - - protected ProcNodeInterface getProcNode(String path) { - ProcService instance = ProcService.getInstance(); - ProcNodeInterface node; - try { - if (Strings.isNullOrEmpty(path)) { - node = instance.open("/"); - } else { - node = instance.open(path); - } - } catch (AnalysisException e) { - LOG.warn(e.getMessage()); - return null; - } - return node; - } - - // Because org.apache.commons.lang.StringEscapeUtils.escapeHtml() not only escape tags in html, - // but also escape Chinese character code, which may cause Chinese garbled in browser, so we - // define our own simplified escape method here. - // ATTN: we should make sure file-encoding of help files is utf-8 - protected String escapeHtmlInPreTag(String oriStr) { - if (oriStr == null) { - return ""; - } - - StringBuilder buff = new StringBuilder(); - char[] chars = oriStr.toCharArray(); - for (int i = 0; i < chars.length; ++i) { - switch (chars[i]) { - case '<': - buff.append("<"); - break; - case '>': - buff.append("<"); - break; - case '"': - buff.append("""); - break; - case '&': - buff.append("&"); - break; - default: - buff.append(chars[i]); - break; - } - } - - return buff.toString(); - } - - private static final NotFoundAction NOT_FOUND_ACTION = new NotFoundAction(null); - - public static NotFoundAction getNotFoundAction() { - return NOT_FOUND_ACTION; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/common/DorisHttpPostObjectAggregator.java b/fe/fe-core/src/main/java/org/apache/doris/http/common/DorisHttpPostObjectAggregator.java deleted file mode 100644 index d904c421a3da76..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/common/DorisHttpPostObjectAggregator.java +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.common; - -import io.netty.channel.ChannelPipeline; -import io.netty.handler.codec.http.HttpContent; -import io.netty.handler.codec.http.HttpMessage; -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpObject; -import io.netty.handler.codec.http.HttpObjectAggregator; -import io.netty.handler.codec.http.HttpRequest; - -/* - * only handle post request, avoid conflicting with {@link LoadAction} - * don't handle 100-continue header - */ -public class DorisHttpPostObjectAggregator extends HttpObjectAggregator { - private boolean startAggregated = false; - - public DorisHttpPostObjectAggregator(int maxContentLength) { - super(maxContentLength, false); - } - - @Override - protected boolean isStartMessage(HttpObject msg) throws Exception { - if (msg instanceof HttpMessage) { - HttpRequest request = (HttpRequest) msg; - if (request.method().equals(HttpMethod.POST)) { - startAggregated = true; - return true; - } - } - return false; - } - - @Override - protected boolean isContentMessage(HttpObject msg) throws Exception { - return msg instanceof HttpContent && startAggregated; - } - - // Doris FE needn't handle 100-continue header - @Override - protected Object newContinueResponse(HttpMessage start, int maxContentLength, ChannelPipeline pipeline) { - return null; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/meta/ColocateMetaService.java b/fe/fe-core/src/main/java/org/apache/doris/http/meta/ColocateMetaService.java deleted file mode 100644 index 42b713d13cfa67..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/meta/ColocateMetaService.java +++ /dev/null @@ -1,229 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.meta; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.ColocateGroupSchema; -import org.apache.doris.catalog.ColocateTableIndex; -import org.apache.doris.catalog.ColocateTableIndex.GroupId; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.http.rest.RestBaseAction; -import org.apache.doris.http.rest.RestBaseResult; -import org.apache.doris.http.rest.RestResult; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.lang.reflect.Type; -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -/* - * the colocate meta define in {@link ColocateTableIndex} - * The actions in ColocateMetaService is for modifying or showing colocate group info manually. - * - * ColocateMetaAction: - * get all information in ColocateTableIndex, as a json string - * eg: - * GET /api/colocate - * return: - * {"colocate_meta":{"groupName2Id":{...},"group2Tables":{}, ...},"status":"OK"} - * - * eg: - * POST /api/colocate/group_stable?db_id=123&group_id=456 (mark group[123.456] as unstable) - * DELETE /api/colocate/group_stable?db_id=123&group_id=456 (mark group[123.456] as stable) - * - * BucketSeqAction: - * change the backends per bucket sequence of a group - * eg: - * POST /api/colocate/bucketseq?db_id=123&group_id=456 - */ -public class ColocateMetaService { - private static final Logger LOG = LogManager.getLogger(ColocateMetaService.class); - private static final String GROUP_ID = "group_id"; - private static final String TABLE_ID = "table_id"; - private static final String DB_ID = "db_id"; - - private static ColocateTableIndex colocateIndex = Catalog.getCurrentColocateIndex(); - - private static GroupId checkAndGetGroupId(BaseRequest request) throws DdlException { - long grpId = Long.valueOf(request.getSingleParameter(GROUP_ID).trim()); - long dbId = Long.valueOf(request.getSingleParameter(DB_ID).trim()); - GroupId groupId = new GroupId(dbId, grpId); - - if (!colocateIndex.isGroupExist(groupId)) { - throw new DdlException("the group " + groupId + "isn't exist"); - } - return groupId; - } - - public static class ColocateMetaBaseAction extends RestBaseAction { - ColocateMetaBaseAction(ActionController controller) { - super(controller); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - if (redirectToMaster(request, response)) { - return; - } - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); - executeInMasterWithAdmin(request, response); - } - - // implement in derived classes - protected void executeInMasterWithAdmin(BaseRequest request, BaseResponse response) - throws DdlException { - throw new DdlException("Not implemented"); - } - } - - // get all colocate meta - public static class ColocateMetaAction extends ColocateMetaBaseAction { - ColocateMetaAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - ColocateMetaAction action = new ColocateMetaAction(controller); - controller.registerHandler(HttpMethod.GET, "/api/colocate", action); - } - - @Override - public void executeInMasterWithAdmin(BaseRequest request, BaseResponse response) - throws DdlException { - response.setContentType("application/json"); - RestResult result = new RestResult(); - result.addResultEntry("colocate_meta", Catalog.getCurrentColocateIndex()); - sendResult(request, response, result); - } - } - - // mark a colocate group as stable or unstable - public static class MarkGroupStableAction extends ColocateMetaBaseAction { - MarkGroupStableAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - MarkGroupStableAction action = new MarkGroupStableAction(controller); - controller.registerHandler(HttpMethod.POST, "/api/colocate/group_stable", action); - controller.registerHandler(HttpMethod.DELETE, "/api/colocate/group_stable", action); - } - - @Override - public void executeInMasterWithAdmin(BaseRequest request, BaseResponse response) - throws DdlException { - GroupId groupId = checkAndGetGroupId(request); - - HttpMethod method = request.getRequest().method(); - if (method.equals(HttpMethod.POST)) { - colocateIndex.markGroupUnstable(groupId, "mark unstable via http api", true); - } else if (method.equals(HttpMethod.DELETE)) { - colocateIndex.markGroupStable(groupId, true); - } else { - response.appendContent(new RestBaseResult("HTTP method is not allowed.").toJson()); - writeResponse(request, response, HttpResponseStatus.METHOD_NOT_ALLOWED); - } - - sendResult(request, response); - } - } - - // update a backendsPerBucketSeq meta for a colocate group - public static class BucketSeqAction extends ColocateMetaBaseAction { - private static final Logger LOG = LogManager.getLogger(BucketSeqAction.class); - - BucketSeqAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - BucketSeqAction action = new BucketSeqAction(controller); - controller.registerHandler(HttpMethod.POST, "/api/colocate/bucketseq", action); - } - - @Override - public void executeInMasterWithAdmin(BaseRequest request, BaseResponse response) - throws DdlException { - final String clusterName = ConnectContext.get().getClusterName(); - if (Strings.isNullOrEmpty(clusterName)) { - throw new DdlException("No cluster selected."); - } - GroupId groupId = checkAndGetGroupId(request); - - String meta = request.getContent(); - Type type = new TypeToken>>() {}.getType(); - List> backendsPerBucketSeq = new Gson().fromJson(meta, type); - LOG.info("get buckets sequence: {}", backendsPerBucketSeq); - - ColocateGroupSchema groupSchema = Catalog.getCurrentColocateIndex().getGroupSchema(groupId); - if (backendsPerBucketSeq.size() != groupSchema.getBucketsNum()) { - throw new DdlException("Invalid bucket num. expected: " + groupSchema.getBucketsNum() + ", actual: " - + backendsPerBucketSeq.size()); - } - - List clusterBackendIds = Catalog.getCurrentSystemInfo().getClusterBackendIds(clusterName, true); - //check the Backend id - for (List backendIds : backendsPerBucketSeq) { - if (backendIds.size() != groupSchema.getReplicaAlloc().getTotalReplicaNum()) { - throw new DdlException("Invalid backend num per bucket. expected: " - + groupSchema.getReplicaAlloc().getTotalReplicaNum() + ", actual: " + backendIds.size()); - } - for (Long beId : backendIds) { - if (!clusterBackendIds.contains(beId)) { - throw new DdlException("The backend " + beId + " does not exist or not available"); - } - } - } - - int bucketsNum = colocateIndex.getBackendsPerBucketSeq(groupId).size(); - Preconditions.checkState(backendsPerBucketSeq.size() == bucketsNum, - backendsPerBucketSeq.size() + " vs. " + bucketsNum); - updateBackendPerBucketSeq(groupId, backendsPerBucketSeq); - LOG.info("the group {} backendsPerBucketSeq meta has been changed to {}", groupId, backendsPerBucketSeq); - - sendResult(request, response); - } - - private void updateBackendPerBucketSeq(GroupId groupId, List> backendsPerBucketSeq) - throws DdlException { - throw new DdlException("Currently not support"); - /* - colocateIndex.addBackendsPerBucketSeq(groupId, backendsPerBucketSeq); - ColocatePersistInfo info2 = ColocatePersistInfo.createForBackendsPerBucketSeq(groupId, backendsPerBucketSeq); - Catalog.getCurrentCatalog().getEditLog().logColocateBackendsPerBucketSeq(info2); - */ - } - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/meta/InvalidClientException.java b/fe/fe-core/src/main/java/org/apache/doris/http/meta/InvalidClientException.java deleted file mode 100644 index 6c5d9dbcf00826..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/meta/InvalidClientException.java +++ /dev/null @@ -1,26 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.meta; - -import org.apache.doris.common.DdlException; - -public class InvalidClientException extends DdlException { - public InvalidClientException(String msg) { - super(msg); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaBaseAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaBaseAction.java deleted file mode 100644 index b8d8843420e147..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaBaseAction.java +++ /dev/null @@ -1,108 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.meta; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.action.WebBaseAction; -import org.apache.doris.master.MetaHelper; -import org.apache.doris.system.Frontend; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.File; - -import io.netty.handler.codec.http.HttpResponseStatus; - -public class MetaBaseAction extends WebBaseAction { - private static final Logger LOG = LogManager.getLogger(MetaBaseAction.class); - private static String CONTENT_DISPOSITION = "Content-disposition"; - - public static final String CLUSTER_ID = "cluster_id"; - public static final String TOKEN = "token"; - - protected File imageDir; - - public MetaBaseAction(ActionController controller, File imageDir) { - super(controller); - this.imageDir = imageDir; - } - - @Override - public boolean needAdmin() { - return false; - } - - @Override - public boolean needPassword() { - return false; - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - if (needCheckClientIsFe()) { - try { - checkFromValidFe(request, response); - } catch (InvalidClientException e) { - response.appendContent("invalid client host."); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - } - super.execute(request, response); - } - - protected boolean needCheckClientIsFe() { - return true; - } - - protected void writeFileResponse(BaseRequest request, BaseResponse response, File file) { - if (file == null || !file.exists()) { - response.appendContent("File does not exist."); - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); - return; - } - - // add custom header - response.updateHeader(CONTENT_DISPOSITION, "attachment; filename=" + file.getName()); - response.updateHeader(MetaHelper.X_IMAGE_SIZE, String.valueOf(file.length())); - - writeObjectResponse(request, response, HttpResponseStatus.OK, file, file.getName(), true); - return; - } - - private boolean isFromValidFe(BaseRequest request) { - String clientHost = request.getHostString(); - Frontend fe = Catalog.getCurrentCatalog().getFeByHost(clientHost); - if (fe == null) { - LOG.warn("request is not from valid FE. client: {}", clientHost); - return false; - } - return true; - } - - private void checkFromValidFe(BaseRequest request, BaseResponse response) - throws InvalidClientException { - if (!isFromValidFe(request)) { - throw new InvalidClientException("invalid client host"); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaService.java b/fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaService.java deleted file mode 100644 index a7079000351a9d..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaService.java +++ /dev/null @@ -1,356 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.meta; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.Config; -import org.apache.doris.ha.FrontendNodeType; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.master.MetaHelper; -import org.apache.doris.persist.MetaCleaner; -import org.apache.doris.persist.Storage; -import org.apache.doris.persist.StorageInfo; -import org.apache.doris.system.Frontend; - -import com.google.common.base.Strings; -import com.google.gson.Gson; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.OutputStream; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -public class MetaService { - private static final int TIMEOUT_SECOND = 10; - - public static class ImageAction extends MetaBaseAction { - private static final String VERSION = "version"; - - public ImageAction(ActionController controller, File imageDir) { - super(controller, imageDir); - } - - public static void registerAction(ActionController controller, File imageDir) - throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/image", new ImageAction(controller, imageDir)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - String versionStr = request.getSingleParameter(VERSION); - if (Strings.isNullOrEmpty(versionStr)) { - response.appendContent("Miss version parameter"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - - long version = checkLongParam(versionStr); - if (version < 0) { - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - - File imageFile = Storage.getImageFile(imageDir, version); - if (!imageFile.exists()) { - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); - return; - } - - writeFileResponse(request, response, imageFile); - } - } - - public static class InfoAction extends MetaBaseAction { - private static final Logger LOG = LogManager.getLogger(InfoAction.class); - - public InfoAction(ActionController controller, File imageDir) { - super(controller, imageDir); - } - - public static void registerAction (ActionController controller, File imageDir) - throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/info", new InfoAction(controller, imageDir)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - try { - Storage currentStorageInfo = new Storage(imageDir.getAbsolutePath()); - StorageInfo storageInfo = new StorageInfo(currentStorageInfo.getClusterID(), - currentStorageInfo.getImageSeq(), currentStorageInfo.getEditsSeq()); - - response.setContentType("application/json"); - Gson gson = new Gson(); - response.appendContent(gson.toJson(storageInfo)); - writeResponse(request, response); - return; - } catch (IOException e) { - LOG.warn("IO error.", e); - response.appendContent("failed to get master info."); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - } - } - - public static class VersionAction extends MetaBaseAction { - public VersionAction(ActionController controller, File imageDir) { - super(controller, imageDir); - } - - public static void registerAction (ActionController controller, File imageDir) - throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/version", new VersionAction(controller, imageDir)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - File versionFile = new File(imageDir, Storage.VERSION_FILE); - writeFileResponse(request, response, versionFile); - } - } - - public static class PutAction extends MetaBaseAction { - private static final Logger LOG = LogManager.getLogger(PutAction.class); - - private static final String VERSION = "version"; - private static final String PORT = "port"; - - public PutAction(ActionController controller, File imageDir) { - super(controller, imageDir); - } - - public static void registerAction (ActionController controller, File imageDir) - throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/put", new PutAction(controller, imageDir)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - String machine = request.getHostString(); - String portStr = request.getSingleParameter(PORT); - // check port to avoid SSRF(Server-Side Request Forgery) - if (Strings.isNullOrEmpty(portStr)) { - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - { - int port = Integer.parseInt(portStr); - if (port < 0 || port > 65535) { - LOG.warn("port is invalid. port={}", port); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - } - - String versionStr = request.getSingleParameter(VERSION); - if (Strings.isNullOrEmpty(versionStr)) { - response.appendContent("Miss version parameter"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - - long version = checkLongParam(versionStr); - - // for master node, reject image put - if (Catalog.getCurrentCatalog().isMaster()) { - response.appendContent("this node is master, reject image put"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - LOG.error("this node is master, but receive image put from host {}, reject it", machine); - return; - } - - // do not accept image whose version is bigger than max journalId - // if accepted, newly added log will not be replayed when restart - long maxJournalId = Catalog.getCurrentCatalog().getMaxJournalId(); - if (version > maxJournalId) { - response.appendContent("image version [" + version + "] is bigger than local max journal id [" - + maxJournalId + "], reject image put"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - LOG.error("receive image whose version [{}] is bigger than local max journal id [{}], reject it", - version, maxJournalId); - return; - } - - String url = "http://" + machine + ":" + portStr - + "/image?version=" + versionStr; - String filename = Storage.IMAGE + "." + versionStr; - - File dir = new File(Catalog.getCurrentCatalog().getImageDir()); - try { - OutputStream out = MetaHelper.getOutputStream(filename, dir); - MetaHelper.getRemoteFile(url, TIMEOUT_SECOND * 1000, out); - MetaHelper.complete(filename, dir); - writeResponse(request, response); - } catch (FileNotFoundException e) { - LOG.warn("file not found. file: {}", filename, e); - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); - return; - } catch (IOException e) { - LOG.warn("failed to get remote file. url: {}", url, e); - writeResponse(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR); - return; - } - - // Delete old image files - MetaCleaner cleaner = new MetaCleaner(Config.meta_dir + "/image"); - try { - cleaner.clean(); - } catch (IOException e) { - LOG.error("Follower/Observer delete old image file fail.", e); - } - } - } - - public static class JournalIdAction extends MetaBaseAction { - public JournalIdAction(ActionController controller, File imageDir) { - super(controller, imageDir); - } - - public static void registerAction (ActionController controller, File imageDir) - throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/journal_id", new JournalIdAction(controller, imageDir)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - long id = Catalog.getCurrentCatalog().getReplayedJournalId(); - response.updateHeader("id", Long.toString(id)); - writeResponse(request, response); - } - } - - public static class RoleAction extends MetaBaseAction { - private static final String HOST = "host"; - private static final String PORT = "port"; - - public RoleAction(ActionController controller, File imageDir) { - super(controller, imageDir); - } - - public static void registerAction (ActionController controller, File imageDir) - throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/role", new RoleAction(controller, imageDir)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - String host = request.getSingleParameter(HOST); - String portString = request.getSingleParameter(PORT); - - if (!Strings.isNullOrEmpty(host) && !Strings.isNullOrEmpty(portString)) { - int port = Integer.parseInt(portString); - Frontend fe = Catalog.getCurrentCatalog().checkFeExist(host, port); - if (fe == null) { - response.updateHeader("role", FrontendNodeType.UNKNOWN.name()); - } else { - response.updateHeader("role", fe.getRole().name()); - response.updateHeader("name", fe.getNodeName()); - } - writeResponse(request, response); - } else { - response.appendContent("Miss parameter"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - } - } - - /* - * This action is used to get the electable_nodes config and the cluster id of - * the fe with the given ip and port. When one frontend start, it should check - * the local electable_nodes config and local cluster id with other frontends. - * If there is any difference, local fe will exit. This is designed to protect - * the consistency of the cluster. - */ - public static class CheckAction extends MetaBaseAction { - private static final Logger LOG = LogManager.getLogger(CheckAction.class); - - public CheckAction(ActionController controller, File imageDir) { - super(controller, imageDir); - } - - public static void registerAction(ActionController controller, File imageDir) - throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/check", - new CheckAction(controller, imageDir)); - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - try { - Storage storage = new Storage(imageDir.getAbsolutePath()); - response.updateHeader(MetaBaseAction.CLUSTER_ID, Integer.toString(storage.getClusterID())); - response.updateHeader(MetaBaseAction.TOKEN, storage.getToken()); - } catch (IOException e) { - LOG.error(e); - } - writeResponse(request, response); - } - } - - public static class DumpAction extends MetaBaseAction { - private static final Logger LOG = LogManager.getLogger(CheckAction.class); - - public DumpAction(ActionController controller, File imageDir) { - super(controller, imageDir); - } - - public static void registerAction (ActionController controller, File imageDir) - throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/dump", new DumpAction(controller, imageDir)); - } - - @Override - public boolean needAdmin() { - return true; - } - - @Override - protected boolean needCheckClientIsFe() { - return false; - } - - @Override - public void executeGet(BaseRequest request, BaseResponse response) { - /* - * Before dump, we acquired the catalog read lock and all databases' read lock and all - * the jobs' read lock. This will guarantee the consistency of database and job queues. - * But Backend may still inconsistent. - */ - - // TODO: Still need to lock ClusterInfoService to prevent add or drop Backends - String dumpFilePath = Catalog.getCurrentCatalog().dumpImage(); - if (dumpFilePath == null) { - response.appendContent("dump failed. " + dumpFilePath); - } - - response.appendContent("dump finished. " + dumpFilePath); - writeResponse(request, response); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/BootstrapFinishAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/BootstrapFinishAction.java deleted file mode 100644 index 7af15429eec000..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/BootstrapFinishAction.java +++ /dev/null @@ -1,162 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.Config; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import com.google.common.base.Strings; -import com.google.gson.Gson; - -import io.netty.handler.codec.http.HttpMethod; -import org.apache.doris.common.Version; - -/* - * fe_host:fe_http_port/api/bootstrap - * return: - * {"status":"OK","msg":"Success","replayedJournal"=123456, "queryPort"=9000, "rpcPort"=9001} - * {"status":"FAILED","msg":"err info..."} - */ -public class BootstrapFinishAction extends RestBaseAction { - private static final String CLUSTER_ID = "cluster_id"; - private static final String TOKEN = "token"; - - public static final String REPLAYED_JOURNAL_ID = "replayedJournalId"; - public static final String QUERY_PORT = "queryPort"; - public static final String RPC_PORT = "rpcPort"; - public static final String VERSION = "version"; - - public BootstrapFinishAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/bootstrap", new BootstrapFinishAction(controller)); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) throws DdlException { - boolean isReady = Catalog.getCurrentCatalog().isReady(); - - // to json response - BootstrapResult result = null; - if (isReady) { - result = new BootstrapResult(); - String clusterIdStr = request.getSingleParameter(CLUSTER_ID); - String token = request.getSingleParameter(TOKEN); - if (!Strings.isNullOrEmpty(clusterIdStr) && !Strings.isNullOrEmpty(token)) { - // cluster id or token is provided, return more info - int clusterId = 0; - try { - clusterId = Integer.valueOf(clusterIdStr); - } catch (NumberFormatException e) { - result.status = ActionStatus.FAILED; - result.msg = "invalid cluster id format: " + clusterIdStr; - } - - if (result.status == ActionStatus.OK) { - if (clusterId != Catalog.getCurrentCatalog().getClusterId()) { - result.status = ActionStatus.FAILED; - result.msg = "invalid cluster id: " + Catalog.getCurrentCatalog().getClusterId(); - } - } - - if (result.status == ActionStatus.OK) { - if (!token.equals(Catalog.getCurrentCatalog().getToken())) { - result.status = ActionStatus.FAILED; - result.msg = "invalid token: " + Catalog.getCurrentCatalog().getToken(); - } - } - - if (result.status == ActionStatus.OK) { - // cluster id and token are valid, return replayed journal id - long replayedJournalId = Catalog.getCurrentCatalog().getReplayedJournalId(); - result.setMaxReplayedJournal(replayedJournalId); - result.setQueryPort(Config.query_port); - result.setRpcPort(Config.rpc_port); - result.setVersion(Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH); - } - } - } else { - result = new BootstrapResult("not ready"); - } - - // send result - response.setContentType("application/json"); - response.getContent().append(result.toJson()); - sendResult(request, response); - } - - public static class BootstrapResult extends RestBaseResult { - private long replayedJournalId = 0; - private int queryPort = 0; - private int rpcPort = 0; - private String version = ""; - - public BootstrapResult() { - super(); - } - - public BootstrapResult(String msg) { - super(msg); - } - - public void setMaxReplayedJournal(long replayedJournalId) { - this.replayedJournalId = replayedJournalId; - } - - public long getMaxReplayedJournal() { - return replayedJournalId; - } - - public void setQueryPort(int queryPort) { - this.queryPort = queryPort; - } - - public int getQueryPort() { - return queryPort; - } - - public void setRpcPort(int rpcPort) { - this.rpcPort = rpcPort; - } - - public int getRpcPort() { - return rpcPort; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - - @Override - public String toJson() { - Gson gson = new Gson(); - return gson.toJson(this); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/CancelStreamLoad.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/CancelStreamLoad.java deleted file mode 100644 index 23f12f441a196f..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/CancelStreamLoad.java +++ /dev/null @@ -1,83 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.UserException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; - -public class CancelStreamLoad extends RestBaseAction { - public CancelStreamLoad(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) - throws IllegalArgException { - CancelStreamLoad action = new CancelStreamLoad(controller); - controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_cancel", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException { - - if (redirectToMaster(request, response)) { - return; - } - - final String clusterName = ConnectContext.get().getClusterName(); - if (Strings.isNullOrEmpty(clusterName)) { - throw new DdlException("No cluster selected."); - } - - String dbName = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(dbName)) { - throw new DdlException("No database selected."); - } - - String fullDbName = ClusterNamespace.getFullName(clusterName, dbName); - - String label = request.getSingleParameter(LABEL_KEY); - if (Strings.isNullOrEmpty(label)) { - throw new DdlException("No label selected."); - } - - // FIXME(cmy) - // checkWritePriv(authInfo.fullUserName, fullDbName); - - Database db = Catalog.getCurrentCatalog().getDbOrDdlException(fullDbName); - - try { - Catalog.getCurrentGlobalTransactionMgr().abortTransaction(db.getId(), label, "user cancel"); - } catch (UserException e) { - throw new DdlException(e.getMessage()); - } - - sendResult(request, response, new RestBaseResult()); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/CheckDecommissionAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/CheckDecommissionAction.java deleted file mode 100644 index 10a555c222c2d4..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/CheckDecommissionAction.java +++ /dev/null @@ -1,93 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.alter.SystemHandler; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.Pair; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.system.SystemInfoService; - -import com.google.common.base.Strings; -import com.google.common.collect.Lists; - -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; - -/* - * calc row count from replica to table - * fe_host:fe_http_port/api/check_decommission?host_ports=host:port,host2:port2... - * return: - * {"status":"OK","msg":"Success"} - * {"status":"FAILED","msg":"err info..."} - */ -public class CheckDecommissionAction extends RestBaseAction { - public static final String HOST_PORTS = "host_ports"; - - public CheckDecommissionAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/check_decommission", new CheckDecommissionAction(controller)); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.OPERATOR); - - String hostPorts = request.getSingleParameter(HOST_PORTS); - if (Strings.isNullOrEmpty(hostPorts)) { - throw new DdlException("No host:port specified."); - } - - String[] hostPortArr = hostPorts.split(","); - if (hostPortArr.length == 0) { - throw new DdlException("No host:port specified."); - } - - List> hostPortPairs = Lists.newArrayList(); - for (String hostPort : hostPortArr) { - Pair pair; - try { - pair = SystemInfoService.validateHostAndPort(hostPort); - } catch (AnalysisException e) { - throw new DdlException(e.getMessage()); - } - hostPortPairs.add(pair); - } - - SystemHandler.checkDecommission(hostPortPairs); - - // to json response - RestBaseResult result = new RestBaseResult(); - - // send result - response.setContentType("application/json"); - response.getContent().append(result.toJson()); - sendResult(request, response); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ConnectionAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/ConnectionAction.java deleted file mode 100644 index 3ac8f288be7fb5..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ConnectionAction.java +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; -import org.apache.doris.common.util.DebugUtil; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.service.ExecuteEnv; - -// This class is used to get current query_id of connection_id. -// Every connection holds at most one query at every point. -// Some we can get query_id firstly, and get query by query_id. -public class ConnectionAction extends RestBaseAction { - public ConnectionAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/connection", new ConnectionAction(controller)); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - String connStr = request.getSingleParameter("connection_id"); - if (connStr == null) { - response.getContent().append("not valid parameter"); - sendResult(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - int connectionId = Integer.valueOf(connStr.trim()); - ConnectContext context = ExecuteEnv.getInstance().getScheduler().getContext(connectionId); - if (context == null || context.queryId() == null) { - response.getContent().append("connection id " + connectionId + " not found."); - sendResult(request, response, HttpResponseStatus.NOT_FOUND); - return; - } - String queryId = DebugUtil.printId(context.queryId()); - response.setContentType("application/json"); - response.getContent().append("{\"query_id\" : \"" + queryId + "\"}"); - sendResult(request, response); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetDdlStmtAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetDdlStmtAction.java deleted file mode 100644 index e9c23429fa2451..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetDdlStmtAction.java +++ /dev/null @@ -1,108 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.Table; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.codehaus.jackson.map.ObjectMapper; - -import java.util.List; -import java.util.Map; - -import io.netty.handler.codec.http.HttpMethod; - -/* - * used to get a table's ddl stmt - * eg: - * fe_host:http_port/api/_get_ddl?db=xxx&tbl=yyy - */ -public class GetDdlStmtAction extends RestBaseAction { - private static final Logger LOG = LogManager.getLogger(GetDdlStmtAction.class); - private static final String DB_PARAM = "db"; - private static final String TABLE_PARAM = "tbl"; - - public GetDdlStmtAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - GetDdlStmtAction action = new GetDdlStmtAction(controller); - controller.registerHandler(HttpMethod.GET, "/api/_get_ddl", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); - - String dbName = request.getSingleParameter(DB_PARAM); - String tableName = request.getSingleParameter(TABLE_PARAM); - - if (Strings.isNullOrEmpty(dbName) || Strings.isNullOrEmpty(tableName)) { - throw new DdlException("Missing params. Need database name and Table name"); - } - - Database db = Catalog.getCurrentCatalog().getDbOrDdlException(dbName); - Table table = db.getTableOrDdlException(tableName); - - List createTableStmt = Lists.newArrayList(); - List addPartitionStmt = Lists.newArrayList(); - List createRollupStmt = Lists.newArrayList(); - - table.readLock(); - try { - Catalog.getDdlStmt(table, createTableStmt, addPartitionStmt, createRollupStmt, true, false /* show password */); - } finally { - table.readUnlock(); - } - - Map> results = Maps.newHashMap(); - results.put("TABLE", createTableStmt); - results.put("PARTITION", addPartitionStmt); - results.put("ROLLUP", createRollupStmt); - - // to json response - String result = ""; - ObjectMapper mapper = new ObjectMapper(); - try { - result = mapper.writeValueAsString(results); - } catch (Exception e) { - // do nothing - } - - // send result - response.setContentType("application/json"); - response.getContent().append(result); - sendResult(request, response); - } -} \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLoadInfoAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLoadInfoAction.java deleted file mode 100644 index b215702817fbb4..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLoadInfoAction.java +++ /dev/null @@ -1,88 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.common.DdlException; -import org.apache.doris.common.MetaNotFoundException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.load.Load; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; - -// Get load information of one load job -public class GetLoadInfoAction extends RestBaseAction { - public GetLoadInfoAction(ActionController controller, boolean isStreamLoad) { - super(controller); - } - - public static void registerAction(ActionController controller) - throws IllegalArgException { - GetLoadInfoAction action = new GetLoadInfoAction(controller, false); - controller.registerHandler(HttpMethod.GET, "/api/{" + DB_KEY + "}/_load_info", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - Load.JobInfo info = new Load.JobInfo(request.getSingleParameter(DB_KEY), - request.getSingleParameter(LABEL_KEY), - ConnectContext.get().getClusterName()); - if (Strings.isNullOrEmpty(info.dbName)) { - throw new DdlException("No database selected"); - } - if (Strings.isNullOrEmpty(info.label)) { - throw new DdlException("No label selected"); - } - if (Strings.isNullOrEmpty(info.clusterName)) { - throw new DdlException("No cluster name selected"); - } - - if (redirectToMaster(request, response)) { - return; - } - - try { - catalog.getLoadInstance().getJobInfo(info); - if (info.tblNames.isEmpty()) { - checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), info.dbName, PrivPredicate.LOAD); - } else { - for (String tblName : info.tblNames) { - checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), info.dbName, tblName, - PrivPredicate.LOAD); - } - } - } catch (DdlException | MetaNotFoundException e) { - catalog.getLoadManager().getLoadJobInfo(info); - } - sendResult(request, response, new Result(info)); - } - - private static class Result extends RestBaseResult { - private Load.JobInfo jobInfo; - public Result(Load.JobInfo info) { - jobInfo = info; - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLogFileAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLogFileAction.java deleted file mode 100644 index 9e80219e647a1f..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLogFileAction.java +++ /dev/null @@ -1,131 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.common.Config; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import com.google.common.base.Strings; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; - -import org.codehaus.jackson.map.ObjectMapper; - -import java.io.File; -import java.util.Map; -import java.util.Set; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -/* - * get log file infos: - * curl -I http://fe_host:http_port/api/get_log_file?type=fe.audit.log - * return: - * HTTP/1.1 200 OK - * file_infos: {"fe.audit.log":24759,"fe.audit.log.20190528.1":132934} - * content-type: text/html - * connection: keep-alive - * - * get log file: - * curl -X GET http://fe_host:http_port/api/get_log_file?type=fe.audit.log&file=fe.audit.log.20190528.1 - */ -public class GetLogFileAction extends RestBaseAction { - private final Set logFileTypes = Sets.newHashSet("fe.audit.log"); - - public GetLogFileAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/get_log_file", new GetLogFileAction(controller)); - controller.registerHandler(HttpMethod.HEAD, "/api/get_log_file", new GetLogFileAction(controller)); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) { - String logType = request.getSingleParameter("type"); - String logFile = request.getSingleParameter("file"); - - // check param empty - if (Strings.isNullOrEmpty(logType)) { - response.appendContent("Miss type parameter"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - - // check type valid or not - if (!logFileTypes.contains(logType)) { - response.appendContent("log type: " + logType + " is invalid!"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - - HttpMethod method = request.getRequest().method(); - if (method.equals(HttpMethod.HEAD)) { - String fileInfos = getFileInfos(logType); - response.updateHeader("file_infos", fileInfos); - writeResponse(request, response, HttpResponseStatus.OK); - return; - } else if (method.equals(HttpMethod.GET)) { - File log = getLogFile(logType, logFile); - if (!log.exists() || !log.isFile()) { - response.appendContent("Log file not exist: " + log.getName()); - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); - return; - } - writeObjectResponse(request, response, HttpResponseStatus.OK, log, log.getName(), true); - } else { - response.appendContent(new RestBaseResult("HTTP method is not allowed.").toJson()); - writeResponse(request, response, HttpResponseStatus.METHOD_NOT_ALLOWED); - } - } - - private String getFileInfos(String logType) { - Map fileInfos = Maps.newTreeMap(); - if (logType.equals("fe.audit.log")) { - File logDir = new File(Config.audit_log_dir); - File[] files = logDir.listFiles(); - for (int i = 0; i < files.length; i++) { - if (files[i].isFile() && files[i].getName().startsWith("fe.audit.log")) { - fileInfos.put(files[i].getName(), files[i].length()); - } - } - } - - String result = ""; - ObjectMapper mapper = new ObjectMapper(); - try { - result = mapper.writeValueAsString(fileInfos); - } catch (Exception e) { - // do nothing - } - return result; - } - - private File getLogFile(String logType, String logFile) { - String logPath = ""; - if ("fe.audit.log".equals(logType)) { - logPath = Config.audit_log_dir + "/" + logFile; - } - return new File(logPath); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetSmallFileAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetSmallFileAction.java deleted file mode 100644 index c678b8a10adeb3..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetSmallFileAction.java +++ /dev/null @@ -1,92 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.util.SmallFileMgr; -import org.apache.doris.common.util.SmallFileMgr.SmallFile; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import com.google.common.base.Strings; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -public class GetSmallFileAction extends RestBaseAction { - private static final Logger LOG = LogManager.getLogger(GetSmallFileAction.class); - - public GetSmallFileAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/get_small_file", new GetSmallFileAction(controller)); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - String token = request.getSingleParameter("token"); - String fileIdStr = request.getSingleParameter("file_id"); - - // check param empty - if (Strings.isNullOrEmpty(token) || Strings.isNullOrEmpty(fileIdStr)) { - response.appendContent("Missing parameter"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - - // check token - if (!token.equals(Catalog.getCurrentCatalog().getToken())) { - response.appendContent("Invalid token"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - - long fileId = -1; - try { - fileId = Long.valueOf(fileIdStr); - } catch (NumberFormatException e) { - response.appendContent("Invalid file id format: " + fileIdStr); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - - SmallFileMgr fileMgr = Catalog.getCurrentCatalog().getSmallFileMgr(); - SmallFile smallFile = fileMgr.getSmallFile(fileId); - if (smallFile == null || !smallFile.isContent) { - response.appendContent("File not found or is not content"); - writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - - HttpMethod method = request.getRequest().method(); - if (method.equals(HttpMethod.GET)) { - writeObjectResponse(request, response, HttpResponseStatus.OK, smallFile.getContentBytes(), - smallFile.name, true); - } else { - response.appendContent(new RestBaseResult("HTTP method is not allowed.").toJson()); - writeResponse(request, response, HttpResponseStatus.METHOD_NOT_ALLOWED); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetStreamLoadState.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetStreamLoadState.java deleted file mode 100644 index 6be12368eb263c..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetStreamLoadState.java +++ /dev/null @@ -1,86 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; - -public class GetStreamLoadState extends RestBaseAction { - public GetStreamLoadState(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) - throws IllegalArgException { - GetStreamLoadState action = new GetStreamLoadState(controller); - controller.registerHandler(HttpMethod.GET, "/api/{" + DB_KEY + "}/get_load_state", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - - if (redirectToMaster(request, response)) { - return; - } - - final String clusterName = ConnectContext.get().getClusterName(); - if (Strings.isNullOrEmpty(clusterName)) { - throw new DdlException("No cluster selected."); - } - - String dbName = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(dbName)) { - throw new DdlException("No database selected."); - } - - String fullDbName = ClusterNamespace.getFullName(clusterName, dbName); - - String label = request.getSingleParameter(LABEL_KEY); - if (Strings.isNullOrEmpty(label)) { - throw new DdlException("No label selected."); - } - - // FIXME(cmy) - // checkReadPriv(authInfo.fullUserName, fullDbName); - - Database db = Catalog.getCurrentCatalog().getDbOrDdlException(fullDbName); - - String state = Catalog.getCurrentGlobalTransactionMgr().getLabelState(db.getId(), label).toString(); - - sendResult(request, response, new Result(state)); - } - - private static class Result extends RestBaseResult { - private String state; - public Result(String state) { - this.state = state; - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/HealthAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/HealthAction.java deleted file mode 100644 index 78322a68ad563f..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/HealthAction.java +++ /dev/null @@ -1,47 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import io.netty.handler.codec.http.HttpMethod; - -public class HealthAction extends RestBaseAction { - public HealthAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) - throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/health", new HealthAction(controller)); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - response.setContentType("application/json"); - - RestResult result = new RestResult(); - result.addResultEntry("total_backend_num", Catalog.getCurrentSystemInfo().getBackendIds(false).size()); - result.addResultEntry("online_backend_num", Catalog.getCurrentSystemInfo().getBackendIds(true).size()); - sendResult(request, response, result); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/LoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/LoadAction.java deleted file mode 100644 index f0d51065eb89be..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/LoadAction.java +++ /dev/null @@ -1,141 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.service.ExecuteEnv; -import org.apache.doris.system.Backend; -import org.apache.doris.system.SystemInfoService; -import org.apache.doris.thrift.TNetworkAddress; - -import com.google.common.base.Strings; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.List; - -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpMethod; - -public class LoadAction extends RestBaseAction { - private static final Logger LOG = LogManager.getLogger(LoadAction.class); - - public static final String SUB_LABEL_NAME_PARAM = "sub_label"; - - private ExecuteEnv execEnv; - private boolean isStreamLoad = false; - - public LoadAction(ActionController controller, ExecuteEnv execEnv) { - this(controller, execEnv, false); - } - - public LoadAction(ActionController controller, ExecuteEnv execEnv, boolean isStreamLoad) { - super(controller); - this.execEnv = execEnv; - this.isStreamLoad = isStreamLoad; - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - ExecuteEnv execEnv = ExecuteEnv.getInstance(); - LoadAction action = new LoadAction(controller, execEnv); - controller.registerHandler(HttpMethod.PUT, - "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_load", action); - - controller.registerHandler(HttpMethod.PUT, - "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_stream_load", - new LoadAction(controller, execEnv, true)); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException { - - // A 'Load' request must have 100-continue header - if (!request.getRequest().headers().contains(HttpHeaders.Names.EXPECT)) { - throw new DdlException("There is no 100-continue header"); - } - - final String clusterName = ConnectContext.get().getClusterName(); - if (Strings.isNullOrEmpty(clusterName)) { - throw new DdlException("No cluster selected."); - } - - String dbName = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(dbName)) { - throw new DdlException("No database selected."); - } - - String tableName = request.getSingleParameter(TABLE_KEY); - if (Strings.isNullOrEmpty(tableName)) { - throw new DdlException("No table selected."); - } - - String fullDbName = ClusterNamespace.getFullName(clusterName, dbName); - - String label = request.getSingleParameter(LABEL_KEY); - if (!isStreamLoad) { - if (Strings.isNullOrEmpty(label)) { - throw new DdlException("No label selected."); - } - } else { - label = request.getRequest().headers().get(LABEL_KEY); - } - - // check auth - checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tableName, PrivPredicate.LOAD); - - TNetworkAddress redirectAddr; - if (!isStreamLoad && !Strings.isNullOrEmpty(request.getSingleParameter(SUB_LABEL_NAME_PARAM))) { - // only multi mini load need to redirect to Master, because only Master has the info of table to - // the Backend which the file exists. - if (redirectToMaster(request, response)) { - return; - } - redirectAddr = execEnv.getMultiLoadMgr().redirectAddr(fullDbName, label); - } else { - // Choose a backend sequentially. - SystemInfoService.BeAvailablePredicate beAvailablePredicate = - new SystemInfoService.BeAvailablePredicate(false, false, true); - List backendIds = Catalog.getCurrentSystemInfo().seqChooseBackendIdsByStorageMediumAndTag( - 1, beAvailablePredicate, false, clusterName, null, null); - if (backendIds == null) { - throw new DdlException(SystemInfoService.NO_BACKEND_LOAD_AVAILABLE_MSG); - } - - Backend backend = Catalog.getCurrentSystemInfo().getBackend(backendIds.get(0)); - if (backend == null) { - throw new DdlException(SystemInfoService.NO_BACKEND_LOAD_AVAILABLE_MSG); - } - - redirectAddr = new TNetworkAddress(backend.getHost(), backend.getHttpPort()); - } - - LOG.info("redirect load action to destination={}, stream: {}, db: {}, tbl: {}, label: {}", - redirectAddr.toString(), isStreamLoad, dbName, tableName, label); - redirectTo(request, response, redirectAddr); - } -} - diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MetaReplayerCheckAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MetaReplayerCheckAction.java deleted file mode 100644 index f874ecf9bd716a..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MetaReplayerCheckAction.java +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import org.codehaus.jackson.map.ObjectMapper; - -import java.util.Map; - -import io.netty.handler.codec.http.HttpMethod; - -/* - * used to get meta replay info - * eg: - * fe_host:http_port/api/_meta_replay_state - */ -public class MetaReplayerCheckAction extends RestBaseAction { - public MetaReplayerCheckAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - MetaReplayerCheckAction action = new MetaReplayerCheckAction(controller); - controller.registerHandler(HttpMethod.GET, "/api/_meta_replay_state", action); - } - - @Override - protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException { - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); - - Map resultMap = Catalog.getCurrentCatalog().getMetaReplayState().getInfo(); - - // to json response - String result = ""; - ObjectMapper mapper = new ObjectMapper(); - try { - result = mapper.writeValueAsString(resultMap); - } catch (Exception e) { - // do nothing - } - - // send result - response.setContentType("application/json"); - response.getContent().append(result); - sendResult(request, response); - } -} \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MetricsAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MetricsAction.java deleted file mode 100644 index 34bc7854e77be0..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MetricsAction.java +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.metric.MetricRepo; -import org.apache.doris.metric.MetricVisitor; -import org.apache.doris.metric.JsonMetricVisitor; -import org.apache.doris.metric.PrometheusMetricVisitor; -import org.apache.doris.metric.SimpleCoreMetricVisitor; - -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; - -//fehost:port/metrics -//fehost:port/metrics?type=core -//fehost:port/metrics?type=json -public class MetricsAction extends RestBaseAction { - - private static final String TYPE_PARAM = "type"; - - public MetricsAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/metrics", new MetricsAction(controller)); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - String type = request.getSingleParameter(TYPE_PARAM); - MetricVisitor visitor = null; - if (!Strings.isNullOrEmpty(type) && type.equalsIgnoreCase("core")) { - visitor = new SimpleCoreMetricVisitor("doris_fe"); - } else if (!Strings.isNullOrEmpty(type) && type.equalsIgnoreCase("json")) { - visitor = new JsonMetricVisitor("doris_fe"); - } else { - visitor = new PrometheusMetricVisitor("doris_fe"); - } - - response.setContentType("text/plain"); - response.getContent().append(MetricRepo.getMetric(visitor)); - sendResult(request, response); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MigrationAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MigrationAction.java deleted file mode 100644 index 6f36c5736fe9c3..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MigrationAction.java +++ /dev/null @@ -1,164 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.MaterializedIndex; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.Replica; -import org.apache.doris.catalog.Table; -import org.apache.doris.catalog.Table.TableType; -import org.apache.doris.catalog.Tablet; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.util.ListComparator; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.base.Strings; -import com.google.common.collect.Lists; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.codehaus.jackson.map.ObjectMapper; - -import java.util.Collections; -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; - -/* - * used to get table's sorted tablet info - * eg: - * fe_host:http_port/api/_migration?db=xxx&tbl=yyy - */ -public class MigrationAction extends RestBaseAction { - private static final Logger LOG = LogManager.getLogger(MigrationAction.class); - private static final String DB_PARAM = "db"; - private static final String TABLE_PARAM = "tbl"; - - public MigrationAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - MigrationAction action = new MigrationAction(controller); - controller.registerHandler(HttpMethod.GET, "/api/_migration", action); - } - - @Override - protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException { - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); - - String dbName = request.getSingleParameter(DB_PARAM); - String tableName = request.getSingleParameter(TABLE_PARAM); - - if (Strings.isNullOrEmpty(dbName)) { - throw new DdlException("Missing params. Need database name"); - } - - Database db = Catalog.getCurrentCatalog().getDbOrDdlException(dbName); - - List> rows = Lists.newArrayList(); - - - - if (!Strings.isNullOrEmpty(tableName)) { - OlapTable olapTable = db.getOlapTableOrDdlException(tableName); - olapTable.readLock(); - try { - for (Partition partition : olapTable.getPartitions()) { - String partitionName = partition.getName(); - MaterializedIndex baseIndex = partition.getBaseIndex(); - for (Tablet tablet : baseIndex.getTablets()) { - List row = Lists.newArrayList(); - row.add(tableName); - row.add(partitionName); - row.add(tablet.getId()); - row.add(olapTable.getSchemaHashByIndexId(baseIndex.getId())); - for (Replica replica : tablet.getReplicas()) { - row.add(replica.getBackendId()); - break; - } - rows.add(row); - } - } - } finally { - olapTable.readUnlock(); - } - } else { - List tableList = db.getTables(); - - // get all olap table - for (Table table : tableList) { - if (table.getType() != TableType.OLAP) { - continue; - } - - OlapTable olapTable = (OlapTable) table; - table.readLock(); - try { - tableName = table.getName(); - for (Partition partition : olapTable.getPartitions()) { - String partitionName = partition.getName(); - MaterializedIndex baseIndex = partition.getBaseIndex(); - for (Tablet tablet : baseIndex.getTablets()) { - List row = Lists.newArrayList(); - row.add(tableName); - row.add(partitionName); - row.add(tablet.getId()); - row.add(olapTable.getSchemaHashByIndexId(baseIndex.getId())); - for (Replica replica : tablet.getReplicas()) { - row.add(replica.getBackendId()); - break; - } - rows.add(row); - } - } - } finally { - table.readUnlock(); - } - } - } - ListComparator> comparator = new ListComparator>(0, 1, 2); - Collections.sort(rows, comparator); - - // to json response - String result = ""; - ObjectMapper mapper = new ObjectMapper(); - try { - result = mapper.writeValueAsString(rows); - } catch (Exception e) { - // do nothing - } - - // send result - response.setContentType("application/json"); - response.getContent().append(result); - sendResult(request, response); - } - - public static void print(String msg) { - System.out.println(System.currentTimeMillis() + " " + msg); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiAbort.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiAbort.java deleted file mode 100644 index 893a47b92b3420..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiAbort.java +++ /dev/null @@ -1,72 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.service.ExecuteEnv; - -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; - -public class MultiAbort extends RestBaseAction { - private ExecuteEnv execEnv; - - public MultiAbort(ActionController controller, ExecuteEnv execEnv) { - super(controller); - this.execEnv = execEnv; - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - ExecuteEnv executeEnv = ExecuteEnv.getInstance(); - MultiAbort action = new MultiAbort(controller, executeEnv); - controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_abort", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - String db = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(db)) { - throw new DdlException("No database selected"); - } - String label = request.getSingleParameter(LABEL_KEY); - if (Strings.isNullOrEmpty(label)) { - throw new DdlException("No label selected"); - } - - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db); - checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); - - // only Master has these load info - if (redirectToMaster(request, response)) { - return; - } - - execEnv.getMultiLoadMgr().abort(fullDbName, label); - sendResult(request, response, RestBaseResult.getOk()); - } -} - diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiCommit.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiCommit.java deleted file mode 100644 index 956c73dc3957f0..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiCommit.java +++ /dev/null @@ -1,77 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.service.ExecuteEnv; - -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; - -public class MultiCommit extends RestBaseAction { - private ExecuteEnv execEnv; - - public MultiCommit(ActionController controller, ExecuteEnv execEnv) { - super(controller); - this.execEnv = execEnv; - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - ExecuteEnv executeEnv = ExecuteEnv.getInstance(); - MultiCommit action = new MultiCommit(controller, executeEnv); - controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_commit", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - String db = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(db)) { - throw new DdlException("No database selected"); - } - String label = request.getSingleParameter(LABEL_KEY); - if (Strings.isNullOrEmpty(label)) { - throw new DdlException("No label selected"); - } - - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db); - checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); - - // only Master has these load info - if (redirectToMaster(request, response)) { - return; - } - RestBaseResult result = new RestBaseResult(); - try { - execEnv.getMultiLoadMgr().commit(fullDbName, label); - } catch (Exception e) { - result.msg = e.getMessage(); - result.status = ActionStatus.FAILED; - } - sendResult(request, response, result); - } -} - diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiDesc.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiDesc.java deleted file mode 100644 index 5f023c7a42aabe..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiDesc.java +++ /dev/null @@ -1,85 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.service.ExecuteEnv; - -import com.google.common.base.Strings; -import com.google.common.collect.Lists; - -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; - -// List all labels of one multi-load -public class MultiDesc extends RestBaseAction { - private ExecuteEnv execEnv; - - public MultiDesc(ActionController controller, ExecuteEnv execEnv) { - super(controller); - this.execEnv = execEnv; - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - ExecuteEnv executeEnv = ExecuteEnv.getInstance(); - MultiDesc action = new MultiDesc(controller, executeEnv); - controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_desc", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - String db = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(db)) { - throw new DdlException("No database selected"); - } - String label = request.getSingleParameter(LABEL_KEY); - if (Strings.isNullOrEmpty(label)) { - throw new DdlException("No label selected"); - } - - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db); - checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); - - // only Master has these load info - if (redirectToMaster(request, response)) { - return; - } - - final List labels = Lists.newArrayList(); - execEnv.getMultiLoadMgr().desc(fullDbName, label, labels); - sendResult(request, response, new Result(labels)); - } - - private static class Result extends RestBaseResult { - private List labels; - - public Result(List labels) { - this.labels = labels; - } - } -} - diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiList.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiList.java deleted file mode 100644 index 407edca9c20fca..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiList.java +++ /dev/null @@ -1,82 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.service.ExecuteEnv; - -import com.google.common.base.Strings; -import com.google.common.collect.Lists; - -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; - -// list all multi load before commit -public class MultiList extends RestBaseAction { - - private ExecuteEnv execEnv; - - public MultiList(ActionController controller, ExecuteEnv execEnv) { - super(controller); - this.execEnv = execEnv; - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - ExecuteEnv executeEnv = ExecuteEnv.getInstance(); - MultiList action = new MultiList(controller, executeEnv); - controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_list", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - String db = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(db)) { - throw new DdlException("No database selected"); - } - - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db); - checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); - - // only Master has these load info - if (redirectToMaster(request, response)) { - return; - } - - final List labels = Lists.newArrayList(); - execEnv.getMultiLoadMgr().list(fullDbName, labels); - sendResult(request, response, new Result(labels)); - } - - private static class Result extends RestBaseResult { - private List labels; - - public Result(List labels) { - this.labels = labels; - } - } -} - diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiStart.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiStart.java deleted file mode 100644 index da8581dbd0de1c..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiStart.java +++ /dev/null @@ -1,92 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.analysis.LoadStmt; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.service.ExecuteEnv; - -import com.google.common.base.Strings; -import com.google.common.collect.Maps; - -import java.util.Map; - -import io.netty.handler.codec.http.HttpMethod; - -// Start multi action -public class MultiStart extends RestBaseAction { - private ExecuteEnv execEnv; - - public MultiStart(ActionController controller, ExecuteEnv execEnv) { - super(controller); - this.execEnv = execEnv; - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - ExecuteEnv executeEnv = ExecuteEnv.getInstance(); - MultiStart action = new MultiStart(controller, executeEnv); - controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_start", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - String db = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(db)) { - throw new DdlException("No database selected"); - } - String label = request.getSingleParameter(LABEL_KEY); - if (Strings.isNullOrEmpty(label)) { - throw new DdlException("No label selected"); - } - - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db); - checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); - - // Multi start request must redirect to master, because all following sub requests will be handled - // on Master - if (redirectToMaster(request, response)) { - return; - } - - Map properties = Maps.newHashMap(); - String[] keys = {LoadStmt.TIMEOUT_PROPERTY, LoadStmt.MAX_FILTER_RATIO_PROPERTY}; - for (String key : keys) { - String value = request.getSingleParameter(key); - if (!Strings.isNullOrEmpty(value)) { - properties.put(key, value); - } - } - for (String key : keys) { - String value = request.getRequest().headers().get(key); - if (!Strings.isNullOrEmpty(value)) { - properties.put(key, value); - } - } - execEnv.getMultiLoadMgr().startMulti(fullDbName, label, properties); - sendResult(request, response, RestBaseResult.getOk()); - } -} - diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiUnload.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiUnload.java deleted file mode 100644 index 8e5f89ad5cb088..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiUnload.java +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.service.ExecuteEnv; - -import com.google.common.base.Strings; - -import io.netty.handler.codec.http.HttpMethod; - -public class MultiUnload extends RestBaseAction { - private static final String SUB_LABEL_KEY = "sub_label"; - - private ExecuteEnv execEnv; - - public MultiUnload(ActionController controller, ExecuteEnv execEnv) { - super(controller); - this.execEnv = execEnv; - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - ExecuteEnv executeEnv = ExecuteEnv.getInstance(); - MultiUnload action = new MultiUnload(controller, executeEnv); - controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_unload", action); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - String db = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(db)) { - throw new DdlException("No database selected"); - } - String label = request.getSingleParameter(LABEL_KEY); - if (Strings.isNullOrEmpty(label)) { - throw new DdlException("No label selected"); - } - String subLabel = request.getSingleParameter(SUB_LABEL_KEY); - if (Strings.isNullOrEmpty(subLabel)) { - throw new DdlException("No sub_label selected"); - } - - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db); - checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); - - if (redirectToMaster(request, response)) { - return; - } - - execEnv.getMultiLoadMgr().unload(fullDbName, label, subLabel); - sendResult(request, response, RestBaseResult.getOk()); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ProfileAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/ProfileAction.java deleted file mode 100644 index 0bb35ed7fbfa8c..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ProfileAction.java +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.common.util.ProfileManager; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -// This class is a RESTFUL interface to get query profile. -// It will be used in query monitor to collect profiles. -// Usage: -// wget http://fe_host:fe_http_port/api/profile?query_id=123456 -public class ProfileAction extends RestBaseAction { - - public ProfileAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/profile", new ProfileAction(controller)); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - String queryId = request.getSingleParameter("query_id"); - if (queryId == null) { - response.getContent().append("not valid parameter"); - sendResult(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - String queryProfileStr = ProfileManager.getInstance().getProfile(queryId); - if (queryProfileStr != null) { - response.getContent().append(queryProfileStr); - sendResult(request, response); - } else { - response.getContent().append("query id " + queryId + " not found."); - sendResult(request, response, HttpResponseStatus.NOT_FOUND); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/QueryDetailAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/QueryDetailAction.java deleted file mode 100644 index c12a640e4a16e9..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/QueryDetailAction.java +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import com.google.gson.Gson; -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; -import java.util.List; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.qe.QueryDetail; -import org.apache.doris.qe.QueryDetailQueue; - -public class QueryDetailAction extends RestBaseAction { - - public QueryDetailAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/query_detail", new QueryDetailAction(controller)); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - String eventTimeStr = request.getSingleParameter("event_time"); - if (eventTimeStr == null) { - response.getContent().append("not valid parameter"); - sendResult(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - long eventTime = Long.valueOf(eventTimeStr.trim()); - List queryDetails = QueryDetailQueue.getQueryDetails(eventTime); - Gson gson = new Gson(); - String json_string = gson.toJson(queryDetails); - response.getContent().append(json_string); - sendResult(request, response); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseAction.java deleted file mode 100644 index 8fea6046b24e8f..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseAction.java +++ /dev/null @@ -1,127 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.analysis.UserIdentity; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseAction; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.UnauthorizedException; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.thrift.TNetworkAddress; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.net.URI; -import java.net.URISyntaxException; - -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpResponseStatus; - -public class RestBaseAction extends BaseAction { - protected static final String DB_KEY = "db"; - protected static final String TABLE_KEY = "table"; - protected static final String LABEL_KEY = "label"; - private static final Logger LOG = LogManager.getLogger(RestBaseAction.class); - - public RestBaseAction(ActionController controller) { - super(controller); - } - - @Override - public void handleRequest(BaseRequest request) throws Exception { - LOG.debug("receive http request. url={}", request.getRequest().uri()); - BaseResponse response = new BaseResponse(); - try { - execute(request, response); - } catch (DdlException e) { - if (e instanceof UnauthorizedException) { - response.appendContent(e.getMessage()); - response.updateHeader(HttpHeaderNames.WWW_AUTHENTICATE.toString(), "Basic realm=\"\""); - writeResponse(request, response, HttpResponseStatus.UNAUTHORIZED); - } else { - sendResult(request, response, new RestBaseResult(e.getMessage())); - } - } - } - - @Override - public void execute(BaseRequest request, BaseResponse response) throws DdlException { - ActionAuthorizationInfo authInfo = getAuthorizationInfo(request); - // check password - UserIdentity currentUser = checkPassword(authInfo); - ConnectContext ctx = new ConnectContext(null); - ctx.setCatalog(Catalog.getCurrentCatalog()); - ctx.setQualifiedUser(authInfo.fullUserName); - ctx.setRemoteIP(authInfo.remoteIp); - ctx.setCurrentUserIdentity(currentUser); - ctx.setCluster(authInfo.cluster); - ctx.setThreadLocalInfo(); - executeWithoutPassword(request, response); - } - - // If user password should be checked, the derived class should implement this method, NOT 'execute()', - // otherwise, override 'execute()' directly - protected void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - throw new DdlException("Not implemented"); - } - - public void sendResult(BaseRequest request, BaseResponse response, RestBaseResult result) { - response.appendContent(result.toJson()); - writeResponse(request, response, HttpResponseStatus.OK); - } - - public void sendResult(BaseRequest request, BaseResponse response, HttpResponseStatus status) { - writeResponse(request, response, status); - } - - public void sendResult(BaseRequest request, BaseResponse response) { - writeResponse(request, response, HttpResponseStatus.OK); - } - - public void redirectTo(BaseRequest request, BaseResponse response, TNetworkAddress addr) - throws DdlException { - String urlStr = request.getRequest().uri(); - URI urlObj = null; - URI resultUriObj = null; - try { - urlObj = new URI(urlStr); - resultUriObj = new URI("http", null, addr.getHostname(), - addr.getPort(), urlObj.getPath(), urlObj.getQuery(), null); - } catch (URISyntaxException e) { - LOG.warn(e.getMessage()); - throw new DdlException(e.getMessage()); - } - response.updateHeader(HttpHeaderNames.LOCATION.toString(), resultUriObj.toString()); - writeResponse(request, response, HttpResponseStatus.TEMPORARY_REDIRECT); - } - - public boolean redirectToMaster(BaseRequest request, BaseResponse response) throws DdlException { - Catalog catalog = Catalog.getCurrentCatalog(); - if (catalog.isMaster()) { - return false; - } - redirectTo(request, response, new TNetworkAddress(catalog.getMasterIp(), catalog.getMasterHttpPort())); - return true; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RowCountAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/RowCountAction.java deleted file mode 100644 index c343607148d557..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RowCountAction.java +++ /dev/null @@ -1,115 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.MaterializedIndex; -import org.apache.doris.catalog.MaterializedIndex.IndexExtState; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.Replica; -import org.apache.doris.catalog.Tablet; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.base.Strings; -import com.google.common.collect.Maps; - -import org.codehaus.jackson.map.ObjectMapper; - -import java.util.Map; - -import io.netty.handler.codec.http.HttpMethod; - -/* - * calc row count from replica to table - * fe_host:fe_http_port/api/rowcount?db=dbname&table=tablename - */ -public class RowCountAction extends RestBaseAction { - - public RowCountAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/rowcount", new RowCountAction(controller)); - } - - @Override - protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException { - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); - - String dbName = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(dbName)) { - throw new DdlException("No database selected."); - } - - String tableName = request.getSingleParameter(TABLE_KEY); - if (Strings.isNullOrEmpty(tableName)) { - throw new DdlException("No table selected."); - } - - Map indexRowCountMap = Maps.newHashMap(); - Catalog catalog = Catalog.getCurrentCatalog(); - Database db = catalog.getDbOrDdlException(dbName); - OlapTable olapTable = db.getOlapTableOrDdlException(tableName); - olapTable.writeLockOrDdlException(); - try { - for (Partition partition : olapTable.getAllPartitions()) { - long version = partition.getVisibleVersion(); - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { - long indexRowCount = 0L; - for (Tablet tablet : index.getTablets()) { - long tabletRowCount = 0L; - for (Replica replica : tablet.getReplicas()) { - if (replica.checkVersionCatchUp(version, false) - && replica.getRowCount() > tabletRowCount) { - tabletRowCount = replica.getRowCount(); - } - } - indexRowCount += tabletRowCount; - } // end for tablets - index.setRowCount(indexRowCount); - indexRowCountMap.put(olapTable.getIndexNameById(index.getId()), indexRowCount); - } // end for indices - } // end for partitions - } finally { - olapTable.writeUnlock(); - } - - // to json response - String result = ""; - ObjectMapper mapper = new ObjectMapper(); - try { - result = mapper.writeValueAsString(indexRowCountMap); - } catch (Exception e) { - // do nothing - } - - // send result - response.setContentType("application/json"); - response.getContent().append(result); - sendResult(request, response); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java deleted file mode 100644 index f096b10bf7da93..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java +++ /dev/null @@ -1,136 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import io.netty.handler.codec.http.HttpMethod; - -import org.apache.doris.common.ConfigBase; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.collect.Maps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.codehaus.jackson.map.ObjectMapper; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -/* - * used to set fe config - * eg: - * fe_host:http_port/api/_set_config?config_key1=config_value1&config_key2=config_value2&... - */ -public class SetConfigAction extends RestBaseAction { - private static final Logger LOG = LogManager.getLogger(SetConfigAction.class); - - private static final String PERSIST_PARAM = "persist"; - private static final String RESET_PERSIST = "reset_persist"; - - public SetConfigAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - SetConfigAction action = new SetConfigAction(controller); - controller.registerHandler(HttpMethod.GET, "/api/_set_config", action); - } - - @Override - protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException { - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); - - boolean needPersist = false; - boolean resetPersist = true; - Map> configs = request.getAllParameters(); - if (configs.containsKey(PERSIST_PARAM)) { - List val = configs.remove(PERSIST_PARAM); - if (val.size() == 1 && val.get(0).equals("true")) { - needPersist = true; - } - } - if (configs.containsKey(RESET_PERSIST)) { - List val = configs.remove(RESET_PERSIST); - if (val.size() == 1 && val.get(0).equals("false")) { - resetPersist = false; - } - } - - Map setConfigs = Maps.newHashMap(); - Map errConfigs = Maps.newHashMap(); - - LOG.debug("get config from url: {}, need persist: {}", configs, needPersist); - - for (Map.Entry> config : configs.entrySet()) { - String confKey = config.getKey(); - List confValue = config.getValue(); - try { - if (confValue != null && confValue.size() == 1) { - ConfigBase.setMutableConfig(confKey, confValue.get(0)); - setConfigs.put(confKey, confValue.get(0)); - } else { - throw new DdlException("conf value size != 1"); - } - } catch (DdlException e) { - LOG.warn("failed to set config {}:{}", confKey, confValue, e); - errConfigs.put(confKey, String.valueOf(confValue)); - } - } - - String persistMsg = ""; - if (needPersist) { - try { - ConfigBase.persistConfig(setConfigs, resetPersist); - persistMsg = "ok"; - } catch (IOException e) { - LOG.warn("failed to persist config", e); - persistMsg = e.getMessage(); - } - } - - Map resultMap = Maps.newHashMap(); - resultMap.put("set", setConfigs); - resultMap.put("err", errConfigs); - resultMap.put("persist", persistMsg); - - // to json response - String result = ""; - ObjectMapper mapper = new ObjectMapper(); - try { - result = mapper.writeValueAsString(resultMap); - } catch (Exception e) { - // do nothing - } - - // send result - response.setContentType("application/json"); - response.getContent().append(result); - sendResult(request, response); - } - - public static void print(String msg) { - System.out.println(System.currentTimeMillis() + " " + msg); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowDataAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowDataAction.java deleted file mode 100644 index af8036eb8b8c38..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowDataAction.java +++ /dev/null @@ -1,91 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Table; -import org.apache.doris.catalog.Table.TableType; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; - -import io.netty.handler.codec.http.HttpMethod; -import java.util.concurrent.ConcurrentHashMap; -import java.util.List; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class ShowDataAction extends RestBaseAction { - private static final Logger LOG = LogManager.getLogger(ShowDataAction.class); - - public ShowDataAction(ActionController controller) { - super(controller); - } - - public static void registerAction (ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/show_data", new ShowDataAction(controller)); - } - - public long getDataSizeOfDatabase(Database db) { - long totalSize = 0; - long tableSize = 0; - // sort by table name - List
    tables = db.getTables(); - for (Table table : tables) { - if (table.getType() != TableType.OLAP) { - continue; - } - table.readLock(); - try { - tableSize = ((OlapTable)table).getDataSize(); - } finally { - table.readUnlock(); - } - totalSize += tableSize; - } // end for tables - return totalSize; - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - String dbName = request.getSingleParameter("db"); - ConcurrentHashMap fullNameToDb = Catalog.getCurrentCatalog().getFullNameToDb(); - long totalSize = 0; - if (dbName != null) { - Database db = fullNameToDb.get("default_cluster:"+dbName); - if (db == null) { - response.getContent().append("database " + dbName + " not found."); - sendResult(request, response, HttpResponseStatus.NOT_FOUND); - return; - } - totalSize = getDataSizeOfDatabase(db); - } else { - for (Database db : fullNameToDb.values()) { - LOG.info("database name: {}", db.getFullName()); - totalSize += getDataSizeOfDatabase(db); - } - } - response.getContent().append(String.valueOf(totalSize)); - sendResult(request, response); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowMetaInfoAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowMetaInfoAction.java deleted file mode 100644 index e3bbef3c405724..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowMetaInfoAction.java +++ /dev/null @@ -1,203 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.MaterializedIndex; -import org.apache.doris.catalog.MaterializedIndex.IndexExtState; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.Replica; -import org.apache.doris.catalog.Replica.ReplicaState; -import org.apache.doris.catalog.Table; -import org.apache.doris.catalog.Table.TableType; -import org.apache.doris.catalog.Tablet; -import org.apache.doris.common.Config; -import org.apache.doris.ha.HAProtocol; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.persist.Storage; - -import com.google.gson.Gson; - -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import io.netty.handler.codec.http.HttpMethod; - -public class ShowMetaInfoAction extends RestBaseAction { - private enum Action { - SHOW_DB_SIZE, - SHOW_HA, - INVALID; - - public static Action getAction(String str) { - try { - return valueOf(str); - } catch (Exception ex) { - return INVALID; - } - } - } - - private static final Logger LOG = LogManager.getLogger(ShowMetaInfoAction.class); - - public ShowMetaInfoAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/show_meta_info", - new ShowMetaInfoAction(controller)); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - String action = request.getSingleParameter("action"); - Gson gson = new Gson(); - response.setContentType("application/json"); - - switch (Action.getAction(action.toUpperCase())) { - case SHOW_DB_SIZE: - response.getContent().append(gson.toJson(getDataSize())); - break; - case SHOW_HA: - response.getContent().append(gson.toJson(getHaInfo())); - break; - default: - break; - } - sendResult(request, response); - } - - public Map getHaInfo() { - HashMap feInfo = new HashMap(); - feInfo.put("role", Catalog.getCurrentCatalog().getFeType().toString()); - if (Catalog.getCurrentCatalog().isMaster()) { - feInfo.put("current_journal_id", - String.valueOf(Catalog.getCurrentCatalog().getEditLog().getMaxJournalId())); - } else { - feInfo.put("current_journal_id", - String.valueOf(Catalog.getCurrentCatalog().getReplayedJournalId())); - } - - HAProtocol haProtocol = Catalog.getCurrentCatalog().getHaProtocol(); - if (haProtocol != null) { - - InetSocketAddress master = null; - try { - master = haProtocol.getLeader(); - } catch (Exception e) { - // this may happen when majority of FOLLOWERS are down and no MASTER right now. - LOG.warn("failed to get leader: {}", e.getMessage()); - } - if (master != null) { - feInfo.put("master", master.getHostString()); - } else { - feInfo.put("master", "unknown"); - } - - List electableNodes = haProtocol.getElectableNodes(false); - ArrayList electableNodeNames = new ArrayList(); - if (!electableNodes.isEmpty()) { - for (InetSocketAddress node : electableNodes) { - electableNodeNames.add(node.getHostString()); - } - feInfo.put("electable_nodes", StringUtils.join(electableNodeNames.toArray(), ",")); - } - - List observerNodes = haProtocol.getObserverNodes(); - ArrayList observerNodeNames = new ArrayList(); - if (observerNodes != null) { - for (InetSocketAddress node : observerNodes) { - observerNodeNames.add(node.getHostString()); - } - feInfo.put("observer_nodes", StringUtils.join(observerNodeNames.toArray(), ",")); - } - } - - feInfo.put("can_read", String.valueOf(Catalog.getCurrentCatalog().canRead())); - feInfo.put("is_ready", String.valueOf(Catalog.getCurrentCatalog().isReady())); - try { - Storage storage = new Storage(Config.meta_dir + "/image"); - feInfo.put("last_checkpoint_version", String.valueOf(storage.getImageSeq())); - long lastCheckpointTime = storage.getCurrentImageFile().lastModified(); - feInfo.put("last_checkpoint_time", String.valueOf(lastCheckpointTime)); - } catch (IOException e) { - LOG.warn(e.getMessage()); - } - return feInfo; - } - - public Map getDataSize() { - Map result = new HashMap(); - List dbNames = Catalog.getCurrentCatalog().getDbNames(); - - for (String dbName : dbNames) { - Database db = Catalog.getCurrentCatalog().getDbNullable(dbName); - if (db == null) { - continue; - } - - long totalSize = 0; - List
    tables = db.getTables(); - for (Table table : tables) { - if (table.getType() != TableType.OLAP) { - continue; - } - - OlapTable olapTable = (OlapTable) table; - long tableSize = 0; - for (Partition partition : olapTable.getAllPartitions()) { - long partitionSize = 0; - for (MaterializedIndex mIndex : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { - long indexSize = 0; - for (Tablet tablet : mIndex.getTablets()) { - long maxReplicaSize = 0; - for (Replica replica : tablet.getReplicas()) { - if (replica.getState() == ReplicaState.NORMAL - || replica.getState() == ReplicaState.SCHEMA_CHANGE) { - if (replica.getDataSize() > maxReplicaSize) { - maxReplicaSize = replica.getDataSize(); - } - } - } // end for replicas - indexSize += maxReplicaSize; - } // end for tablets - partitionSize += indexSize; - } // end for tables - tableSize += partitionSize; - } // end for partitions - totalSize += tableSize; - } // end for tables - result.put(dbName, totalSize); - } // end for dbs - return result; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowProcAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowProcAction.java deleted file mode 100644 index bb53d9c7e7b3b5..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowProcAction.java +++ /dev/null @@ -1,136 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.analysis.RedirectStatus; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.proc.ProcNodeInterface; -import org.apache.doris.common.proc.ProcResult; -import org.apache.doris.common.proc.ProcService; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.MasterOpExecutor; -import org.apache.doris.qe.OriginStatement; -import org.apache.doris.qe.ShowResultSet; -import org.apache.doris.system.SystemInfoService; - -import com.google.common.base.Strings; -import com.google.gson.Gson; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.List; - -import io.netty.handler.codec.http.HttpMethod; - -// Format: -// http://username:password@192.168.1.1:8030/api/show_proc?path=/ -public class ShowProcAction extends RestBaseAction { - private static final Logger LOG = LogManager.getLogger(ShowProcAction.class); - - public ShowProcAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/show_proc", new ShowProcAction(controller)); - } - - @Override - public void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException { - // check authority - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); - - String path = request.getSingleParameter("path"); - String forward = request.getSingleParameter("forward"); - boolean isForward = false; - if (!Strings.isNullOrEmpty(forward) && forward.equals("true")) { - isForward = true; - } - - // forward to master if necessary - if (!Catalog.getCurrentCatalog().isMaster() && isForward) { - String showProcStmt = "SHOW PROC \"" + path + "\""; - ConnectContext context = new ConnectContext(null); - context.setCatalog(Catalog.getCurrentCatalog()); - context.setCluster(SystemInfoService.DEFAULT_CLUSTER); - context.setQualifiedUser(ConnectContext.get().getQualifiedUser()); - context.setRemoteIP(ConnectContext.get().getRemoteIP()); - MasterOpExecutor masterOpExecutor = new MasterOpExecutor(new OriginStatement(showProcStmt, 0), context, - RedirectStatus.FORWARD_NO_SYNC, true); - LOG.debug("need to transfer to Master. stmt: {}", context.getStmtId()); - - try { - masterOpExecutor.execute(); - } catch (Exception e) { - response.appendContent("Failed to forward stmt: " + e.getMessage()); - sendResult(request, response); - return; - } - - ShowResultSet resultSet = masterOpExecutor.getProxyResultSet(); - if (resultSet == null) { - response.appendContent("Failed to get result set"); - sendResult(request, response); - return; - } - - Gson gson = new Gson(); - response.setContentType("application/json"); - response.getContent().append(gson.toJson(resultSet.getResultRows())); - - } else { - ProcNodeInterface procNode = null; - ProcService instance = ProcService.getInstance(); - try { - if (Strings.isNullOrEmpty(path)) { - procNode = instance.open("/"); - } else { - procNode = instance.open(path); - } - } catch (AnalysisException e) { - LOG.warn(e.getMessage()); - response.getContent().append("[]"); - } - - if (procNode != null) { - ProcResult result; - try { - result = procNode.fetchResult(); - List> rows = result.getRows(); - - Gson gson = new Gson(); - response.setContentType("application/json"); - response.getContent().append(gson.toJson(rows)); - } catch (AnalysisException e) { - LOG.warn(e.getMessage()); - response.getContent().append("[]"); - } - } - } - - sendResult(request, response); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowRuntimeInfoAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowRuntimeInfoAction.java deleted file mode 100644 index 66ec5c4c3500da..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowRuntimeInfoAction.java +++ /dev/null @@ -1,69 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import com.google.gson.Gson; - -import io.netty.handler.codec.http.HttpMethod; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.HashMap; - -public class ShowRuntimeInfoAction extends RestBaseAction { - private static final Logger LOG = LogManager.getLogger(ShowRuntimeInfoAction.class); - - public ShowRuntimeInfoAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, "/api/show_runtime_info", - new ShowRuntimeInfoAction(controller)); - } - - @Override - public void execute(BaseRequest request, BaseResponse response) { - HashMap feInfo = new HashMap(); - - // Get memory info - Runtime r = Runtime.getRuntime(); - feInfo.put("free_mem", String.valueOf(r.freeMemory())); - feInfo.put("total_mem", String.valueOf(r.totalMemory())); - feInfo.put("max_mem", String.valueOf(r.maxMemory())); - - // Get thread count - ThreadGroup parentThread; - for (parentThread = Thread.currentThread().getThreadGroup(); - parentThread.getParent() != null; - parentThread = parentThread.getParent()) { - }; - feInfo.put("thread_cnt", String.valueOf(parentThread.activeCount())); - - Gson gson = new Gson(); - response.setContentType("application/json"); - response.getContent().append(gson.toJson(feInfo)); - - sendResult(request, response); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/StorageTypeCheckAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/StorageTypeCheckAction.java deleted file mode 100644 index bf6d271ebe2443..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/StorageTypeCheckAction.java +++ /dev/null @@ -1,98 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.MaterializedIndexMeta; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Table; -import org.apache.doris.catalog.Table.TableType; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.thrift.TStorageType; - -import com.google.common.base.Strings; - -import org.json.simple.JSONObject; - -import java.util.List; -import java.util.Map; - -import io.netty.handler.codec.http.HttpMethod; - -public class StorageTypeCheckAction extends RestBaseAction { - public StorageTypeCheckAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - StorageTypeCheckAction action = new StorageTypeCheckAction(controller); - controller.registerHandler(HttpMethod.GET, "/api/_check_storagetype", action); - } - - @Override - protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException { - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); - - String dbName = request.getSingleParameter(DB_KEY); - if (Strings.isNullOrEmpty(dbName)) { - throw new DdlException("Parameter db is missing"); - } - - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), dbName); - Database db = catalog.getDbOrDdlException(fullDbName); - - JSONObject root = new JSONObject(); - List
    tableList = db.getTables(); - - for (Table tbl : tableList) { - if (tbl.getType() != TableType.OLAP) { - continue; - } - - OlapTable olapTbl = (OlapTable) tbl; - olapTbl.readLock(); - try { - JSONObject indexObj = new JSONObject(); - for (Map.Entry entry : olapTbl.getIndexIdToMeta().entrySet()) { - MaterializedIndexMeta indexMeta = entry.getValue(); - if (indexMeta.getStorageType() == TStorageType.ROW) { - indexObj.put(olapTbl.getIndexNameById(entry.getKey()), indexMeta.getStorageType().name()); - } - } - root.put(tbl.getName(), indexObj); - } finally { - olapTbl.readUnlock(); - } - } - - // to json response - String result = root.toString(); - - // send result - response.setContentType("application/json"); - response.getContent().append(result); - sendResult(request, response); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java deleted file mode 100644 index d252ba36dbcbfe..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java +++ /dev/null @@ -1,306 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.analysis.InlineViewRef; -import org.apache.doris.analysis.SelectStmt; -import org.apache.doris.analysis.StatementBase; -import org.apache.doris.analysis.TableName; -import org.apache.doris.analysis.TableRef; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.Table; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.DorisHttpException; -import org.apache.doris.common.MetaNotFoundException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.planner.PlanFragment; -import org.apache.doris.planner.Planner; -import org.apache.doris.planner.ScanNode; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.OriginStatement; -import org.apache.doris.qe.StmtExecutor; -import org.apache.doris.thrift.TDataSink; -import org.apache.doris.thrift.TDataSinkType; -import org.apache.doris.thrift.TMemoryScratchSink; -import org.apache.doris.thrift.TNetworkAddress; -import org.apache.doris.thrift.TPaloScanRange; -import org.apache.doris.thrift.TPlanFragment; -import org.apache.doris.thrift.TQueryOptions; -import org.apache.doris.thrift.TQueryPlanInfo; -import org.apache.doris.thrift.TScanRangeLocations; -import org.apache.doris.thrift.TTabletVersionInfo; -import org.apache.doris.thrift.TUniqueId; - -import com.google.common.base.Strings; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.thrift.TException; -import org.apache.thrift.TSerializer; -import org.codehaus.jackson.map.ObjectMapper; -import org.json.simple.JSONObject; -import org.json.simple.JSONValue; - -import java.util.ArrayList; -import java.util.Base64; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -/** - * This class responsible for parse the sql and generate the query plan fragment for a (only one) table{@see OlapTable} - * the related tablet maybe pruned by query planer according the `where` predicate. - */ -public class TableQueryPlanAction extends RestBaseAction { - - public static final Logger LOG = LogManager.getLogger(TableQueryPlanAction.class); - - public TableQueryPlanAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.POST, - "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_query_plan", - new TableQueryPlanAction(controller)); - controller.registerHandler(HttpMethod.GET, - "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_query_plan", - new TableQueryPlanAction(controller)); - } - - @Override - protected void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - // just allocate 2 slot for top holder map - Map resultMap = new HashMap<>(4); - String dbName = request.getSingleParameter(DB_KEY); - String tableName = request.getSingleParameter(TABLE_KEY); - String postContent = request.getContent(); - try { - // may be these common validate logic should be moved to one base class - if (Strings.isNullOrEmpty(dbName) - || Strings.isNullOrEmpty(tableName)) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "{database}/{table} must be selected"); - } - if (Strings.isNullOrEmpty(postContent)) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "POST body must contains [sql] root object"); - } - JSONObject jsonObject = (JSONObject) JSONValue.parse(postContent); - if (jsonObject == null) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "malformed json [ " + postContent + " ]"); - } - String sql = (String) jsonObject.get("sql"); - if (Strings.isNullOrEmpty(sql)) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "POST body must contains [sql] root object"); - } - LOG.info("receive SQL statement [{}] from external service [ user [{}]] for database [{}] table [{}]", - sql, ConnectContext.get().getCurrentUserIdentity(), dbName, tableName); - - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), dbName); - // check privilege for select, otherwise return HTTP 401 - checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tableName, PrivPredicate.SELECT); - Table table; - try { - Database db = Catalog.getCurrentCatalog().getDbOrMetaException(fullDbName); - table = db.getTableOrMetaException(tableName, Table.TableType.OLAP); - } catch (MetaNotFoundException e) { - throw new DorisHttpException(HttpResponseStatus.FORBIDDEN, e.getMessage()); - } - - // may be should acquire writeLock - table.readLock(); - try { - // parse/analysis/plan the sql and acquire tablet distributions - handleQuery(ConnectContext.get(), fullDbName, tableName, sql, resultMap); - } finally { - table.readUnlock(); - } - } catch (DorisHttpException e) { - // status code should conforms to HTTP semantic - resultMap.put("status", e.getCode().code()); - resultMap.put("exception", e.getMessage()); - } - ObjectMapper mapper = new ObjectMapper(); - try { - String result = mapper.writeValueAsString(resultMap); - // send result with extra information - response.setContentType("application/json"); - response.getContent().append(result); - sendResult(request, response, HttpResponseStatus.valueOf(Integer.parseInt(String.valueOf(resultMap.get("status"))))); - } catch (Exception e) { - // may be this never happen - response.getContent().append(e.getMessage()); - sendResult(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR); - } - } - - - /** - * process the sql syntax and return the resolved pruned tablet - * - * @param context context for analyzer - * @param sql the single table select statement - * @param result the acquired results - * @return - * @throws DorisHttpException - */ - private void handleQuery(ConnectContext context, String requestDb, String requestTable, String sql, - Map result) throws DorisHttpException { - // use SE to resolve sql - StmtExecutor stmtExecutor = new StmtExecutor(context, new OriginStatement(sql, 0), false); - try { - TQueryOptions tQueryOptions = context.getSessionVariable().toThrift(); - // Conduct Planner create SingleNodePlan#createPlanFragments - tQueryOptions.num_nodes = 1; - // analyze sql - stmtExecutor.analyze(tQueryOptions); - } catch (Exception e) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, e.getMessage()); - } - // the parsed logical statement - StatementBase query = stmtExecutor.getParsedStmt(); - // only process select semantic - if (!(query instanceof SelectStmt)) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "Select statement needed, but found [" + sql + " ]"); - } - SelectStmt stmt = (SelectStmt) query; - // just only process sql like `select * from table where `, only support executing scan semantic - if (stmt.hasAggInfo() || stmt.hasAnalyticInfo() - || stmt.hasOrderByClause() || stmt.hasOffset() || stmt.hasLimit() || stmt.isExplain()) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "only support single table filter-prune-scan, but found [ " + sql + "]"); - } - // process only one table by one http query - List fromTables = stmt.getTableRefs(); - if (fromTables.size() != 1) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "Select statement must have only one table"); - } - - TableRef fromTable = fromTables.get(0); - if (fromTable instanceof InlineViewRef) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "Select statement must not embed another statement"); - } - // check consistent http requested resource with sql referenced - // if consistent in this way, can avoid check privilege - TableName tableAndDb = fromTables.get(0).getName(); - if (!(tableAndDb.getDb().equals(requestDb) && tableAndDb.getTbl().equals(requestTable))) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "requested database and table must consistent with sql: request [ " - + requestDb + "." + requestTable + "]" + "and sql [" + tableAndDb.toString() + "]"); - } - - // acquired Planner to get PlanNode and fragment templates - Planner planner = stmtExecutor.planner(); - // acquire ScanNode to obtain pruned tablet - // in this way, just retrieve only one scannode - List scanNodes = planner.getScanNodes(); - if (scanNodes.size() != 1) { - throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Planner should plan just only one ScanNode but found [ " + scanNodes.size() + "]"); - } - List scanRangeLocations = scanNodes.get(0).getScanRangeLocations(0); - // acquire the PlanFragment which the executable template - List fragments = planner.getFragments(); - if (fragments.size() != 1) { - throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Planner should plan just only one PlanFragment but found [ " + fragments.size() + "]"); - } - - TQueryPlanInfo tQueryPlanInfo = new TQueryPlanInfo(); - - - // acquire TPlanFragment - TPlanFragment tPlanFragment = fragments.get(0).toThrift(); - // set up TMemoryScratchSink - TDataSink tDataSink = new TDataSink(); - tDataSink.type = TDataSinkType.MEMORY_SCRATCH_SINK; - tDataSink.memory_scratch_sink = new TMemoryScratchSink(); - tPlanFragment.output_sink = tDataSink; - - tQueryPlanInfo.plan_fragment = tPlanFragment; - tQueryPlanInfo.desc_tbl = query.getAnalyzer().getDescTbl().toThrift(); - // set query_id - UUID uuid = UUID.randomUUID(); - tQueryPlanInfo.query_id = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()); - - Map tablet_info = new HashMap<>(); - // acquire resolved tablet distribution - Map tabletRoutings = assemblePrunedPartitions(scanRangeLocations); - tabletRoutings.forEach((tabletId, node) -> { - long tablet = Long.parseLong(tabletId); - tablet_info.put(tablet, new TTabletVersionInfo(tablet, node.version, 0l /*versionHash*/, node.schemaHash)); - }); - tQueryPlanInfo.tablet_info = tablet_info; - - // serialize TQueryPlanInfo and encode plan with Base64 to string in order to translate by json format - TSerializer serializer = new TSerializer(); - String opaqued_query_plan; - try { - byte[] query_plan_stream = serializer.serialize(tQueryPlanInfo); - opaqued_query_plan = Base64.getEncoder().encodeToString(query_plan_stream); - } catch (TException e) { - throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "TSerializer failed to serialize PlanFragment, reason [ " + e.getMessage() + " ]"); - } - result.put("partitions", tabletRoutings); - result.put("opaqued_query_plan", opaqued_query_plan); - result.put("status", 200); - } - - /** - * acquire all involved (already pruned) tablet routing - * - * @param scanRangeLocationsList - * @return - */ - private Map assemblePrunedPartitions(List scanRangeLocationsList) { - Map result = new HashMap<>(); - for (TScanRangeLocations scanRangeLocations : scanRangeLocationsList) { - // only process palo(doris) scan range - TPaloScanRange scanRange = scanRangeLocations.scan_range.palo_scan_range; - Node tabletRouting = new Node(Long.parseLong(scanRange.version), Integer.parseInt(scanRange.schema_hash)); - for (TNetworkAddress address : scanRange.hosts) { - tabletRouting.addRouting(address.hostname + ":" + address.port); - } - result.put(String.valueOf(scanRange.tablet_id), tabletRouting); - } - return result; - } - - // helper class for json transformation - final class Node { - // ["host1:port1", "host2:port2", "host3:port3"] - public List routings = new ArrayList<>(); - public long version; - public int schemaHash; - - public Node(long version, int schemaHash) { - this.version = version; - this.schemaHash = schemaHash; - } - - private void addRouting(String routing) { - routings.add(routing); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableRowCountAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableRowCountAction.java deleted file mode 100644 index 1b4a8edece5329..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableRowCountAction.java +++ /dev/null @@ -1,113 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Table; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.DorisHttpException; -import org.apache.doris.common.MetaNotFoundException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.base.Strings; - -import org.codehaus.jackson.map.ObjectMapper; - -import java.util.HashMap; -import java.util.Map; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -/** - * This class is responsible for fetch the approximate row count of the specified table from cluster-meta data, - * the approximate row maybe used for some computing system to decide use which compute-algorithm can be used - * such as shuffle join or broadcast join. - *

    - * This API is not intended to compute the exact row count of the specified table, if you need the exact row count, - * please consider using the sql syntax `select count(*) from {table}` - */ -public class TableRowCountAction extends RestBaseAction { - public TableRowCountAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.GET, - "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_count", - new TableRowCountAction(controller)); - } - - @Override - protected void executeWithoutPassword(BaseRequest request, BaseResponse response) - throws DdlException { - // just allocate 2 slot for top holder map - Map resultMap = new HashMap<>(4); - String dbName = request.getSingleParameter(DB_KEY); - String tableName = request.getSingleParameter(TABLE_KEY); - try { - if (Strings.isNullOrEmpty(dbName) - || Strings.isNullOrEmpty(tableName)) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "{database}/{table} must be selected"); - } - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), dbName); - // check privilege for select, otherwise return HTTP 401 - checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tableName, PrivPredicate.SELECT); - Table table; - try { - Database db = Catalog.getCurrentCatalog().getDbOrMetaException(fullDbName); - table = db.getTableOrMetaException(tableName, Table.TableType.OLAP); - } catch (MetaNotFoundException e) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, e.getMessage()); - } - - table.readLock(); - try { - OlapTable olapTable = (OlapTable) table; - resultMap.put("status", 200); - resultMap.put("size", olapTable.proximateRowCount()); - } finally { - table.readUnlock(); - } - } catch (DorisHttpException e) { - // status code should conforms to HTTP semantic - resultMap.put("status", e.getCode().code()); - resultMap.put("exception", e.getMessage()); - } - ObjectMapper mapper = new ObjectMapper(); - try { - String result = mapper.writeValueAsString(resultMap); - // send result with extra information - response.setContentType("application/json"); - response.getContent().append(result); - sendResult(request, response, HttpResponseStatus.valueOf(Integer.parseInt(String.valueOf(resultMap.get("status"))))); - } catch (Exception e) { - // may be this never happen - response.getContent().append(e.getMessage()); - sendResult(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableSchemaAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableSchemaAction.java deleted file mode 100644 index 9000ae4890f9c4..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableSchemaAction.java +++ /dev/null @@ -1,135 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.http.rest; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.PrimitiveType; -import org.apache.doris.catalog.ScalarType; -import org.apache.doris.catalog.Table; -import org.apache.doris.catalog.Type; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.DorisHttpException; -import org.apache.doris.common.MetaNotFoundException; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.BaseRequest; -import org.apache.doris.http.BaseResponse; -import org.apache.doris.http.IllegalArgException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.base.Strings; - -import org.codehaus.jackson.map.ObjectMapper; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -/** - * Get table schema for specified cluster.database.table with privilege checking - */ -public class TableSchemaAction extends RestBaseAction { - - public TableSchemaAction(ActionController controller) { - super(controller); - } - - public static void registerAction(ActionController controller) throws IllegalArgException { - // the extra `/api` path is so disgusting - controller.registerHandler(HttpMethod.GET, - "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_schema", new TableSchemaAction - (controller)); - } - - @Override - protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException { - // just allocate 2 slot for top holder map - Map resultMap = new HashMap<>(2); - String dbName = request.getSingleParameter(DB_KEY); - String tableName = request.getSingleParameter(TABLE_KEY); - try { - if (Strings.isNullOrEmpty(dbName) - || Strings.isNullOrEmpty(tableName)) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "No database or table selected."); - } - String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), dbName); - // check privilege for select, otherwise return 401 HTTP status - checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tableName, PrivPredicate.SELECT); - Table table; - try { - Database db = Catalog.getCurrentCatalog().getDbOrMetaException(fullDbName); - table = db.getTableOrMetaException(tableName, Table.TableType.OLAP); - } catch (MetaNotFoundException e) { - throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, e.getMessage()); - } - - table.readLock(); - try { - List columns = table.getBaseSchema(); - List> propList = new ArrayList(columns.size()); - for (Column column : columns) { - Map baseInfo = new HashMap<>(2); - Type colType = column.getOriginType(); - PrimitiveType primitiveType = colType.getPrimitiveType(); - if (primitiveType == PrimitiveType.DECIMALV2) { - ScalarType scalarType = (ScalarType) colType; - baseInfo.put("precision", scalarType.getPrecision() + ""); - baseInfo.put("scale", scalarType.getScalarScale() + ""); - } - baseInfo.put("type", primitiveType.toString()); - baseInfo.put("comment", column.getComment()); - baseInfo.put("name", column.getDisplayName()); - Optional aggregationType = Optional.ofNullable(column.getAggregationType()); - baseInfo.put("aggregation_type", aggregationType.isPresent() ? column.getAggregationType().toSql() : ""); - propList.add(baseInfo); - } - resultMap.put("status", 200); - resultMap.put("properties", propList); - } catch (Exception e) { - // Transform the general Exception to custom DorisHttpException - throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, e.getMessage() == null ? "Null Pointer Exception" : e.getMessage()); - } finally { - table.readUnlock(); - } - } catch (DorisHttpException e) { - // status code should conforms to HTTP semantic - resultMap.put("status", e.getCode().code()); - resultMap.put("exception", e.getMessage()); - } - ObjectMapper mapper = new ObjectMapper(); - try { - String result = mapper.writeValueAsString(resultMap); - // send result with extra information - response.setContentType("application/json"); - response.getContent().append(result); - sendResult(request, response, HttpResponseStatus.valueOf(Integer.parseInt(String.valueOf(resultMap.get("status"))))); - } catch (Exception e) { - // may be this never happen - response.getContent().append(e.getMessage()); - sendResult(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/IllegalArgException.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/IllegalArgException.java similarity index 72% rename from fe/fe-core/src/main/java/org/apache/doris/http/IllegalArgException.java rename to fe/fe-core/src/main/java/org/apache/doris/httpv2/IllegalArgException.java index 9e82202d8344a5..b77cf0844940c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/http/IllegalArgException.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/IllegalArgException.java @@ -14,21 +14,20 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - -package org.apache.doris.http; +package org.apache.doris.httpv2; public class IllegalArgException extends Exception { - private static final long serialVersionUID = 3344697787301861667L; + private static final long serialVersionUID = 3344697787301861667L; - public IllegalArgException() { - super(""); - } + public IllegalArgException() { + super(""); + } - public IllegalArgException(String msg) { - super(msg); - } + public IllegalArgException(String msg) { + super(msg); + } - public IllegalArgException(String msg, Throwable cause) { - super(msg, cause); - } -} + public IllegalArgException(String msg, Throwable cause) { + super(msg, cause); + } +} \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ActionStatus.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ActionStatus.java similarity index 95% rename from fe/fe-core/src/main/java/org/apache/doris/http/rest/ActionStatus.java rename to fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ActionStatus.java index a72b94cd946adb..23cd6f7fbca1d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ActionStatus.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ActionStatus.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.http.rest; +package org.apache.doris.httpv2.entity; // Status of RESTful action public enum ActionStatus { diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseResult.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestBaseResult.java similarity index 97% rename from fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseResult.java rename to fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestBaseResult.java index 63cd07cc02f6e6..5d27d9e6915c68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestBaseResult.java @@ -14,14 +14,14 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - -package org.apache.doris.http.rest; +package org.apache.doris.httpv2.entity; import com.google.gson.Gson; import com.google.gson.GsonBuilder; // Base restful result public class RestBaseResult { + private static final RestBaseResult OK = new RestBaseResult(); public ActionStatus status; public String msg; diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestResult.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestResult.java similarity index 94% rename from fe/fe-core/src/main/java/org/apache/doris/http/rest/RestResult.java rename to fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestResult.java index d8819848c151ae..d501bfdc59ae77 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestResult.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.http.rest; +package org.apache.doris.httpv2.entity; import java.util.Map; @@ -23,6 +23,7 @@ import com.google.gson.Gson; public class RestResult extends RestBaseResult { + private Map resultMap; public RestResult() { @@ -38,7 +39,7 @@ public String toJson() { Gson gson = new Gson(); addResultEntry("status", status); if (status != ActionStatus.OK) { - addResultEntry("msg", msg); + addResultEntry("msg", msg); } return gson.toJson(resultMap); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java index d79cd5b9976214..2213034cfe00df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java @@ -20,7 +20,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; -import org.apache.doris.http.rest.RestBaseResult; +import org.apache.doris.httpv2.entity.RestBaseResult; import org.apache.doris.load.Load; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java index 8cacb0832a8ea4..d5b2b71e589637 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java @@ -20,7 +20,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.DdlException; -import org.apache.doris.http.rest.RestBaseResult; +import org.apache.doris.httpv2.entity.RestBaseResult; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; import org.apache.doris.service.ExecuteEnv; diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java index 943aa26bac3c54..86983901d8df76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java @@ -19,7 +19,7 @@ import org.apache.doris.analysis.LoadStmt; import org.apache.doris.common.DdlException; -import org.apache.doris.http.rest.RestBaseResult; +import org.apache.doris.httpv2.entity.RestBaseResult; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; import org.apache.doris.service.ExecuteEnv; diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java index 1dc7e77be780c2..35533433d4d615 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java @@ -20,8 +20,8 @@ import org.apache.doris.common.Config; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.ThreadPoolManager; -import org.apache.doris.http.HttpServer; -import org.apache.doris.http.IllegalArgException; +import org.apache.doris.httpv2.HttpServer; +import org.apache.doris.httpv2.IllegalArgException; import org.apache.doris.journal.JournalEntity; import org.apache.doris.meta.MetaContext; import org.apache.doris.qe.QeService; @@ -86,20 +86,10 @@ public void startDebugMode(String dorisHomeDir) { // Only start MySQL and HttpServer private void startService(String dorisHomeDir) throws IllegalArgException, IOException { // HTTP server - if (!Config.enable_http_server_v2) { - HttpServer httpServer = new HttpServer( - Config.http_port, - Config.http_max_line_length, - Config.http_max_header_size, - Config.http_max_chunk_size - ); - httpServer.setup(); - httpServer.start(); - } else { - org.apache.doris.httpv2.HttpServer httpServer2 = new org.apache.doris.httpv2.HttpServer(); - httpServer2.setPort(Config.http_port); - httpServer2.start(dorisHomeDir); - } + + HttpServer httpServer = new HttpServer(); + httpServer.setPort(Config.http_port); + httpServer.start(dorisHomeDir); // MySQl server QeService qeService = new QeService(Config.query_port, Config.mysql_service_nio_enabled, ExecuteEnv.getInstance().getScheduler()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/path/PathTrieTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/path/PathTrieTest.java index 66531d6ed3e413..afeef0db1fc297 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/path/PathTrieTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/path/PathTrieTest.java @@ -19,11 +19,6 @@ import static com.google.common.collect.Maps.newHashMap; -import org.apache.doris.http.ActionController; -import org.apache.doris.http.rest.MultiStart; -import org.apache.doris.http.rest.TableQueryPlanAction; -import org.apache.doris.http.rest.TableRowCountAction; -import org.apache.doris.http.rest.TableSchemaAction; import org.apache.doris.service.ExecuteEnv; import com.google.common.collect.Maps; @@ -165,18 +160,4 @@ public void testNamedWildcardAndLookupWithWildcard() { Assert.assertEquals(params.get("test"), "*"); } - @Test - public void testInsert(@Injectable ActionController controller, - @Injectable ExecuteEnv execEnv) { - PathTrie pathTrie = new PathTrie(); - pathTrie.insert("/api/{db}/_multi_start", new MultiStart(controller, execEnv)); - pathTrie.insert("/api/{db}/{table}/_count", new TableRowCountAction(controller)); - pathTrie.insert("/api/{db}/{table}/_schema", new TableSchemaAction(controller)); - pathTrie.insert("/api/{db}/{table}/_query_plan", new TableQueryPlanAction(controller)); - Map params = Maps.newHashMap(); - pathTrie.retrieve("/api/test/_multi_start", params); - Assert.assertEquals("test", params.get("db")); - - - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java b/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java index 8f2a1c444ce627..71f4adaf247277 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java +++ b/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java @@ -38,9 +38,12 @@ import org.apache.doris.catalog.TabletInvertedIndex; import org.apache.doris.catalog.TabletMeta; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.ExceptionChecker.ThrowingRunnable; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.httpv2.HttpServer; +import org.apache.doris.httpv2.IllegalArgException; import org.apache.doris.load.Load; import org.apache.doris.mysql.privilege.PaloAuth; import org.apache.doris.persist.EditLog; @@ -110,6 +113,8 @@ abstract public class DorisHttpTestCase { protected String rootAuth = Credentials.basic("root", ""); + public static final String DORIS_HOME_DIR = System.getenv("DORIS_HOME"); + @Mocked private static EditLog editLog; @@ -291,13 +296,13 @@ public static void initHttpServer() throws IllegalArgException, InterruptedExcep } } - httpServer = new HttpServer(HTTP_PORT); - httpServer.setup(); - httpServer.start(); - // must ensure the http server started before any unit test - while (!httpServer.isStarted()) { - Thread.sleep(500); - } + httpServer = new HttpServer(); + httpServer.setPort(Config.http_port); + httpServer.setMaxHttpPostSize(Config.jetty_server_max_http_post_size); + httpServer.setAcceptors(Config.jetty_server_acceptors); + httpServer.setSelectors(Config.jetty_server_selectors); + httpServer.setWorkers(Config.jetty_server_workers); + httpServer.start(DORIS_HOME_DIR); } @@ -337,10 +342,6 @@ TabletInvertedIndex getCurrentInvertedIndex() { public void tearDown() { } - @AfterClass - public static void closeHttpServer() { - httpServer.shutDown(); - } public void doSetUp() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/http/HttpAuthManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/http/HttpAuthManagerTest.java index a6ed325d2ba5d7..2cd23dc34d0b37 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/http/HttpAuthManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/http/HttpAuthManagerTest.java @@ -17,9 +17,12 @@ package org.apache.doris.http; +import java.util.ArrayList; +import java.util.List; import org.apache.doris.analysis.UserIdentity; -import org.apache.doris.http.HttpAuthManager.SessionValue; +import org.apache.doris.httpv2.HttpAuthManager.SessionValue; +import org.apache.doris.httpv2.HttpAuthManager; import org.junit.Assert; import org.junit.Test; @@ -31,15 +34,20 @@ public void testNormal() { HttpAuthManager authMgr = HttpAuthManager.getInstance(); String sessionId = "test_session_id"; String username = "test-user"; - SessionValue sessionValue = new SessionValue(); + HttpAuthManager.SessionValue sessionValue = new HttpAuthManager.SessionValue(); sessionValue.currentUser = UserIdentity.createAnalyzedUserIdentWithIp(username, "%"); authMgr.addSessionValue(sessionId, sessionValue); Assert.assertEquals(1, authMgr.getAuthSessions().size()); - System.out.println("username in test: " + authMgr.getSessionValue(sessionId).currentUser); - Assert.assertEquals(username, authMgr.getSessionValue(sessionId).currentUser.getQualifiedUser()); + List sessionIds = new ArrayList<>(); + sessionIds.add (sessionId); + System.out.println("username in test: " + authMgr.getSessionValue(sessionIds).currentUser); + Assert.assertEquals(username, authMgr.getSessionValue(sessionIds).currentUser.getQualifiedUser()); + String noExistSession = "no-exist-session-id"; - Assert.assertNull(authMgr.getSessionValue(noExistSession)); + sessionIds.clear(); + sessionIds.add (noExistSession); + Assert.assertNull(authMgr.getSessionValue(sessionIds)); Assert.assertEquals(1, authMgr.getAuthSessions().size()); } }