From ec1913ae5c5bc2622ea0534b2d8def5a4a86e67e Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 16 Sep 2020 16:43:53 +0800 Subject: [PATCH 01/12] [UI Part 5] Enable HTTP Server 2 by FE config Add a new FE config `enable_http_server_v2` to enable new HTTP Server implementation. The default value is false. --- .gitignore | 2 + build.sh | 38 +++++++++++ .../en/administrator-guide/colocation-join.md | 67 ++++--------------- .../administrator-guide/colocation-join.md | 67 ++++--------------- .../main/java/org/apache/doris/PaloFe.java | 27 +++++--- .../java/org/apache/doris/common/Config.java | 8 +++ .../org/apache/doris/system/HeartbeatMgr.java | 40 ++++++++--- 7 files changed, 125 insertions(+), 124 deletions(-) diff --git a/.gitignore b/.gitignore index e2c576843adfe9..8fb5733edbb179 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,8 @@ dependency-reduced-pom.xml fe_plugins/**/.classpath fe_plugins/**/.factorypath samples/**/.classpath +fe/fe-core/src/main/resources/static/ +nohup.out #ignore eclipse project file & idea project file diff --git a/build.sh b/build.sh index 20f74c503f51da..89b61079a1042a 100755 --- a/build.sh +++ b/build.sh @@ -50,6 +50,7 @@ Usage: $0 Optional options: --be build Backend --fe build Frontend and Spark Dpp application + --ui build Frontend web ui with npm --spark-dpp build Spark DPP application --clean clean and build target @@ -59,6 +60,7 @@ Usage: $0 $0 --fe --clean clean and build Frontend and Spark Dpp application $0 --fe --be --clean clean and build Frontend, Spark Dpp application and Backend $0 --spark-dpp build Spark DPP application alone + $0 --fe --ui build Frontend web ui with npm " exit 1 } @@ -69,6 +71,7 @@ OPTS=$(getopt \ -o 'h' \ -l 'be' \ -l 'fe' \ + -l 'ui' \ -l 'spark-dpp' \ -l 'clean' \ -l 'help' \ @@ -82,6 +85,7 @@ eval set -- "$OPTS" BUILD_BE= BUILD_FE= +BUILD_UI= BUILD_SPARK_DPP= CLEAN= RUN_UT= @@ -90,12 +94,14 @@ if [ $# == 1 ] ; then # default BUILD_BE=1 BUILD_FE=1 + BUILD_UI=1 BUILD_SPARK_DPP=1 CLEAN=0 RUN_UT=0 else BUILD_BE=0 BUILD_FE=0 + BUILD_UI=0 BUILD_SPARK_DPP=0 CLEAN=0 RUN_UT=0 @@ -103,6 +109,7 @@ else case "$1" in --be) BUILD_BE=1 ; shift ;; --fe) BUILD_FE=1 ; shift ;; + --ui) BUILD_UI=1 ; shift ;; --spark-dpp) BUILD_SPARK_DPP=1 ; shift ;; --clean) CLEAN=1 ; shift ;; --ut) RUN_UT=1 ; shift ;; @@ -134,6 +141,7 @@ fi echo "Get params: BUILD_BE -- $BUILD_BE BUILD_FE -- $BUILD_FE + BUILD_UI -- $BUILD_UI BUILD_SPARK_DPP -- $BUILD_SPARK_DPP CLEAN -- $CLEAN RUN_UT -- $RUN_UT @@ -186,6 +194,36 @@ if [ ${BUILD_FE} -eq 1 -o ${BUILD_SPARK_DPP} -eq 1 ]; then fi fi + +function build_ui() { + # check NPM env here, not in env.sh. + # Because UI should be considered a non-essential component at runtime. + # Only when the compilation is required, check the relevant compilation environment. + NPM=npm + if ! ${NPM} --version; then + echo "Error: npm is not found" + exit 1 + fi + if [[ ! -z ${CUSTOM_NPM_REGISTRY} ]]; then + ${NPM} config set registry ${CUSTOM_NPM_REGISTRY} + npm_reg=`${NPM} get registry` + echo "NPM registry: $npm_reg" + fi + + echo "Build Frontend UI" + cd ${DORIS_HOME}/ui + ${NPM} install + ${NPM} run build + rm -rf ${DORIS_HOME}/fe/fe-core/src/main/resources/static/ + mkdir -p ${DORIS_HOME}/fe/fe-core/src/main/resources/static + cp -r ${DORIS_HOME}/ui/dist/* ${DORIS_HOME}/fe/fe-core/src/main/resources/static +} + +# FE UI must be built before building FE +if [ ${BUILD_UI} -eq 1 ] ; then + build_ui +fi + # Clean and build Frontend if [ ${FE_MODULES}x != ""x ]; then echo "Build Frontend Modules: $FE_MODULES" diff --git a/docs/en/administrator-guide/colocation-join.md b/docs/en/administrator-guide/colocation-join.md index ea2e5118c3b8cd..e899e8c9fd6ac9 100644 --- a/docs/en/administrator-guide/colocation-join.md +++ b/docs/en/administrator-guide/colocation-join.md @@ -357,60 +357,21 @@ The API is implemented on the FE side and accessed using `fe_host: fe_http_port` GET /api/colocate Return the internal Colocation info in JSON format: - + { - "colocate_meta": { - "groupName2Id": { - "g1": { - "dbId": 10005, - "grpId": 10008 - } - }, - "group2Tables": {}, - "table2Group": { - "10007": { - "dbId": 10005, - "grpId": 10008 - }, - "10040": { - "dbId": 10005, - "grpId": 10008 - } - }, - "group2Schema": { - "10005.10008": { - "groupId": { - "dbId": 10005, - "grpId": 10008 - }, - "distributionColTypes": [{ - "type": "INT", - "len": -1, - "isAssignedStrLenInColDefinition": false, - "precision": 0, - "scale": 0 - }], - "bucketsNum": 10, - "replicationNum": 2 - } - }, - "group2BackendsPerBucketSeq": { - "10005.10008": [ - [10004, 10002], - [10003, 10002], - [10002, 10004], - [10003, 10002], - [10002, 10004], - [10003, 10002], - [10003, 10004], - [10003, 10004], - [10003, 10004], - [10002, 10004] - ] - }, - "unstableGroups": [] + "msg": "success", + "code": 0, + "data": { + "infos": [ + ["10003.12002", "10003_group1", "10037, 10043", "1", "1", "int(11)", "true"] + ], + "unstableGroupIds": [], + "allGroupIds": [{ + "dbId": 10003, + "grpId": 12002 + }] }, - "status": "OK" + "count": 0 } ``` 2. Mark Group as Stable or Unstable @@ -436,7 +397,7 @@ The API is implemented on the FE side and accessed using `fe_host: fe_http_port` The interface can force the number distribution of a group. ``` - POST /api/colocate/bucketseq?db_id=10005&group_id= 10008 + POST /api/colocate/bucketseq?db_id=10005&group_id=10008 Body: [[10004,10002],[10003,10002],[10002,10004],[10003,10002],[10002,10004],[10003,10002],[10003,10004],[10003,10004],[10003,10004],[10002,10004]] diff --git a/docs/zh-CN/administrator-guide/colocation-join.md b/docs/zh-CN/administrator-guide/colocation-join.md index c701efd7c7d8b6..1f4a998f2193de 100644 --- a/docs/zh-CN/administrator-guide/colocation-join.md +++ b/docs/zh-CN/administrator-guide/colocation-join.md @@ -358,58 +358,19 @@ Doris 提供了几个和 Colocation Join 有关的 HTTP Restful API，用于查 返回以 Json 格式表示内部 Colocation 信息。 { - "colocate_meta": { - "groupName2Id": { - "g1": { - "dbId": 10005, - "grpId": 10008 - } - }, - "group2Tables": {}, - "table2Group": { - "10007": { - "dbId": 10005, - "grpId": 10008 - }, - "10040": { - "dbId": 10005, - "grpId": 10008 - } - }, - "group2Schema": { - "10005.10008": { - "groupId": { - "dbId": 10005, - "grpId": 10008 - }, - "distributionColTypes": [{ - "type": "INT", - "len": -1, - "isAssignedStrLenInColDefinition": false, - "precision": 0, - "scale": 0 - }], - "bucketsNum": 10, - "replicationNum": 2 - } - }, - "group2BackendsPerBucketSeq": { - "10005.10008": [ - [10004, 10002], - [10003, 10002], - [10002, 10004], - [10003, 10002], - [10002, 10004], - [10003, 10002], - [10003, 10004], - [10003, 10004], - [10003, 10004], - [10002, 10004] - ] - }, - "unstableGroups": [] + "msg": "success", + "code": 0, + "data": { + "infos": [ + ["10003.12002", "10003_group1", "10037, 10043", "1", "1", "int(11)", "true"] + ], + "unstableGroupIds": [], + "allGroupIds": [{ + "dbId": 10003, + "grpId": 12002 + }] }, - "status": "OK" + "count": 0 } ``` @@ -436,7 +397,7 @@ Doris 提供了几个和 Colocation Join 有关的 HTTP Restful API，用于查 该接口可以强制设置某一 Group 的数分布。 ``` - POST /api/colocate/bucketseq?db_id=10005&group_id= 10008 + POST /api/colocate/bucketseq?db_id=10005&group_id=10008 Body: [[10004,10002],[10003,10002],[10002,10004],[10003,10002],[10002,10004],[10003,10002],[10003,10004],[10003,10004],[10003,10004],[10002,10004]] @@ -445,4 +406,4 @@ Doris 提供了几个和 Colocation Join 有关的 HTTP Restful API，用于查 ``` 其中 Body 是以嵌套数组表示的 BucketsSequence 以及每个 Bucket 中分片分布所在 BE 的 id。 - 注意，使用该命令，可能需要将 FE 的配置 `disable_colocate_relocate` 和 `disable_colocate_balance` 设为 true。即关闭系统自动的 Colocation 副本修复和均衡。否则可能在修改后，会被系统自动重置。 \ No newline at end of file + 注意，使用该命令，可能需要将 FE 的配置 `disable_colocate_relocate` 和 `disable_colocate_balance` 设为 true。即关闭系统自动的 Colocation 副本修复和均衡。否则可能在修改后，会被系统自动重置。 diff --git a/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java b/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java index 76f320f9f58bb7..7d8c8a40f5db40 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java +++ b/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java @@ -91,7 +91,7 @@ public static void start(String dorisHomeDir, String pidDir, String[] args) { Log4jConfig.initLogging(dorisHomeDir + "/conf/"); // set dns cache ttl - java.security.Security.setProperty("networkaddress.cache.ttl" , "60"); + java.security.Security.setProperty("networkaddress.cache.ttl", "60"); // check command line options checkCommandLineOptions(cmdLineOpts); @@ -111,16 +111,25 @@ public static void start(String dorisHomeDir, String pidDir, String[] args) { // 3. HttpServer for HTTP Server QeService qeService = new QeService(Config.query_port, Config.mysql_service_nio_enabled, ExecuteEnv.getInstance().getScheduler()); FeServer feServer = new FeServer(Config.rpc_port); - HttpServer httpServer = new HttpServer( - Config.http_port, - Config.http_max_line_length, - Config.http_max_header_size, - Config.http_max_chunk_size - ); - httpServer.setup(); + feServer.start(); - httpServer.start(); + + if (!Config.enable_http_server_v2) { + HttpServer httpServer = new HttpServer( + Config.http_port, + Config.http_max_line_length, + Config.http_max_header_size, + Config.http_max_chunk_size + ); + httpServer.setup(); + httpServer.start(); + } else { + org.apache.doris.httpv2.HttpServer httpServer2 = new org.apache.doris.httpv2.HttpServer(); + httpServer2.setPort(Config.http_port); + httpServer2.start(dorisHomeDir); + } + qeService.start(); ThreadPoolManager.registerAllThreadPoolMetric(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java index e8825ba13e633d..dd0e3d60bdf3cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java @@ -1270,4 +1270,12 @@ public class Config extends ConfigBase { */ @ConfField(mutable = true, masterOnly = true) public static double default_max_filter_ratio = 0; + + /** + * HTTP Server V2 is implemented by SpringBoot. + * It uses an architecture that separates front and back ends. + * Only enable httpv2 can user to use the new Frontend UI interface + */ + @ConfField + public static boolean enable_http_server_v2 = false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java index c87096e1433491..cfcd5f7cd29c14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java @@ -292,20 +292,42 @@ public HeartbeatResponse call() { try { String result = Util.getResultForUrl(url, null, 2000, 2000); /* - * return: + * Old return: * {"replayedJournalId":191224,"queryPort":9131,"rpcPort":9121,"status":"OK","msg":"Success"} * {"replayedJournalId":0,"queryPort":0,"rpcPort":0,"status":"FAILED","msg":"not ready"} + * + * New return: + * {"msg":"success","code":0,"data":{"queryPort":9333,"rpcPort":9222,"replayedJournalId":197},"count":0} + * {"msg":"not ready","code":1,"data":null,"count":0} */ JSONObject root = new JSONObject(result); - String status = root.getString("status"); - if (!"OK".equals(status)) { - return new FrontendHbResponse(fe.getNodeName(), root.getString("msg")); + if (root.has("status")) { + // old return + String status = root.getString("status"); + if (!"OK".equals(status)) { + return new FrontendHbResponse(fe.getNodeName(), root.getString("msg")); + } else { + long replayedJournalId = root.getLong(BootstrapFinishAction.REPLAYED_JOURNAL_ID); + int queryPort = root.getInt(BootstrapFinishAction.QUERY_PORT); + int rpcPort = root.getInt(BootstrapFinishAction.RPC_PORT); + return new FrontendHbResponse(fe.getNodeName(), queryPort, rpcPort, replayedJournalId, + System.currentTimeMillis()); + } + } else if (root.has("code")) { + // new return + int code = root.getInt("code"); + if (code != 0) { + return new FrontendHbResponse(fe.getNodeName(), root.getString("msg")); + } else { + JSONObject dataObj = root.getJSONObject("data"); + long replayedJournalId = dataObj.getLong(BootstrapFinishAction.REPLAYED_JOURNAL_ID); + int queryPort = dataObj.getInt(BootstrapFinishAction.QUERY_PORT); + int rpcPort = dataObj.getInt(BootstrapFinishAction.RPC_PORT); + return new FrontendHbResponse(fe.getNodeName(), queryPort, rpcPort, replayedJournalId, + System.currentTimeMillis()); + } } else { - long replayedJournalId = root.getLong(BootstrapFinishAction.REPLAYED_JOURNAL_ID); - int queryPort = root.getInt(BootstrapFinishAction.QUERY_PORT); - int rpcPort = root.getInt(BootstrapFinishAction.RPC_PORT); - return new FrontendHbResponse(fe.getNodeName(), queryPort, rpcPort, replayedJournalId, - System.currentTimeMillis()); + throw new Exception("invalid return value: " + result); } } catch (Exception e) { return new FrontendHbResponse(fe.getNodeName(), From 463365b4d0756d2560d7d74034c04464841cf4c5 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 16 Sep 2020 18:38:13 +0800 Subject: [PATCH 02/12] fix UI bug --- .../administrator-guide/config/fe_config.md | 19 ++++- .../administrator-guide/config/fe_config.md | 16 +++- .../java/org/apache/doris/common/Config.java | 9 +++ .../httpv2/rest/ExtraBasepathAction.java | 74 +++++++++++++++++++ ui/src/pages/configuration/index.tsx | 2 +- ui/src/pages/home/index.tsx | 2 +- ui/src/pages/layout/index.tsx | 2 +- ui/src/pages/login/index.tsx | 7 +- ui/src/pages/logs/index.tsx | 2 +- .../content/components/data-prev.tsx | 2 +- .../playground/content/content-result.tsx | 4 +- .../playground/content/content-structure.tsx | 2 +- ui/src/pages/playground/content/index.tsx | 2 +- ui/src/pages/playground/data-import/index.tsx | 7 +- ui/src/pages/playground/tree/index.tsx | 2 +- ui/src/pages/query-profile/index.tsx | 2 +- ui/src/pages/session/index.tsx | 2 +- ui/src/pages/system/index.tsx | 2 +- ui/src/utils/api.ts | 9 ++- ui/src/utils/request.tsx | 10 ++- 20 files changed, 148 insertions(+), 29 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ExtraBasepathAction.java diff --git a/docs/en/administrator-guide/config/fe_config.md b/docs/en/administrator-guide/config/fe_config.md index 8b18254ddfdc77..a8b37cac4967e1 100644 --- a/docs/en/administrator-guide/config/fe_config.md +++ b/docs/en/administrator-guide/config/fe_config.md @@ -706,7 +706,22 @@ The function is still in the experimental stage, so the default value is false. Used to set default database data quota size, default is 1T. -### 'default_max_filter_ratio' +### `default_max_filter_ratio` -Used to set default max filter ratio of load Job. It will be overridden by 'max_filter_ratio' of the load job properties，default value is 0, value range 0-1. +Used to set default max filter ratio of load Job. It will be overridden by `max_filter_ratio` of the load job properties，default value is 0, value range 0-1. +### `enable_http_server_v2` + +Whether to enable the V2 version of the HTTP Server implementation. The new HTTP Server is implemented using SpringBoot. And realize the separation of front and back ends. +Only when it is turned on, can you use the new UI interface under the `ui/` directory. + +Default is false. + +### `http_api_extra_base_path` + +In some deployment environments, user need to specify an additional base path as the unified prefix of the HTTP API. This parameter is used by the user to specify additional prefixes. +After setting, user can get the parameter value through the `GET /api/basepath` interface. +And the new UI will also try to get this base path first to assemble the URL. +Only valid when `enable_http_server_v2` is true. + +The default is empty, that is, not set. diff --git a/docs/zh-CN/administrator-guide/config/fe_config.md b/docs/zh-CN/administrator-guide/config/fe_config.md index 0eb2248060de1e..0ece531b36bab2 100644 --- a/docs/zh-CN/administrator-guide/config/fe_config.md +++ b/docs/zh-CN/administrator-guide/config/fe_config.md @@ -700,8 +700,22 @@ thrift_client_timeout_ms 的值被设置为大于0来避免线程卡在java.net. 用于设置database data的默认quota值，单位为 bytes，默认1T. -### 'default_max_filter_ratio' +### `default_max_filter_ratio` 默认的最大容忍可过滤（数据不规范等原因）的数据比例。它将被Load Job 中设置的"max_filter_ratio"覆盖，默认0，取值范围0-1. +### `enable_http_server_v2` +是否启用的 V2 版本的 HTTP Server 实现。新的 HTTP Server 采用 SpringBoot 实现。并且实现了前后端分离。 +只有当开启后，才能使用 `ui/` 目录下的新版 UI 界面。 + +默认为 false。 + +### `http_api_extra_base_path` + +一些部署环境下，需要指定额外的 base path 作为 HTTP API 的统一前缀。这个参数用于用户指定额外的前缀。 +设置后，可以通过 `GET /api/basepath` 接口获取这个参数值。 +新版本的UI也会先尝试获取这个base path来拼接URL。 +仅在 `enable_http_server_v2` 为 true 的情况下才有效。 + +默认为空，即不设置。 diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java index dd0e3d60bdf3cf..97c13c8b0c6d8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java @@ -1278,4 +1278,13 @@ public class Config extends ConfigBase { */ @ConfField public static boolean enable_http_server_v2 = false; + + /* + * Base path is the URL prefix for all API paths. + * Some deployment environments need to configure additional base path to match resources. + * This Api will return the path configured in Config.http_api_extra_base_path. + * Default is empty, which means not set. + */ + @ConfField + public static String http_api_extra_base_path = ""; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ExtraBasepathAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ExtraBasepathAction.java new file mode 100644 index 00000000000000..581bbe3e4ba3cc --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ExtraBasepathAction.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.common.Config; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; + +import org.apache.parquet.Strings; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Api for getting the base path of api + * Base path is the URL prefix for all API paths. + * Some deployment environments need to configure additional base path to match resources. + * This Api will return the path configured in Config.http_api_extra_base_path. + */ +@RestController +public class ExtraBasepathAction { + @RequestMapping(path = "/api/basepath", method = RequestMethod.GET) + public ResponseEntity execute(HttpServletRequest request, HttpServletResponse response) { + BasepathResponse resp = new BasepathResponse(); + resp.path = Config.http_api_extra_base_path; + if (Strings.isNullOrEmpty(Config.http_api_extra_base_path)) { + resp.enable = false; + } else { + resp.enable = true; + } + return ResponseEntityBuilder.ok(resp); + } + + public static class BasepathResponse { + public boolean enable; // enable is false mean no extra base path configured. + public String path; + + public boolean isEnable() { + return enable; + } + + public void setEnable(boolean enable) { + this.enable = enable; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + } +} + + diff --git a/ui/src/pages/configuration/index.tsx b/ui/src/pages/configuration/index.tsx index 8c29fd7680e696..0152d8403940ec 100644 --- a/ui/src/pages/configuration/index.tsx +++ b/ui/src/pages/configuration/index.tsx @@ -20,7 +20,7 @@ import React, {useState, useEffect} from 'react'; import {Typography, Button, Row, Col} from 'antd'; const {Title} = Typography; -import {getConfig} from 'Utils/api'; +import {getConfig} from 'Src/api/api'; import Table from 'Src/components/table'; export default function Configuration(params: any) { const [allTableData, setAllTableData] = useState({}); diff --git a/ui/src/pages/home/index.tsx b/ui/src/pages/home/index.tsx index 4ea95a060e674d..d55fecf2a16db8 100644 --- a/ui/src/pages/home/index.tsx +++ b/ui/src/pages/home/index.tsx @@ -20,7 +20,7 @@ import React, {useState, useEffect} from 'react'; import {Typography, Divider, BackTop, Spin} from 'antd'; const {Title, Paragraph, Text} = Typography; -import {getHardwareInfo} from 'Utils/api'; +import {getHardwareInfo} from 'Src/api/api'; export default function Home(params: any) { const [hardwareData , setHardwareData] = useState({}); diff --git a/ui/src/pages/layout/index.tsx b/ui/src/pages/layout/index.tsx index 466cf95d403cfe..d2947393974b3a 100644 --- a/ui/src/pages/layout/index.tsx +++ b/ui/src/pages/layout/index.tsx @@ -29,7 +29,7 @@ import {renderRoutes} from 'react-router-config'; import {useHistory} from 'react-router-dom'; import {useTranslation} from 'react-i18next'; import routes from 'Src/router'; -import {logOut} from 'Utils/api'; +import {logOut} from 'Src/api/api'; import './index.css'; import styles from './index.less'; const {Header, Content, Footer} = Layout; diff --git a/ui/src/pages/login/index.tsx b/ui/src/pages/login/index.tsx index 402e38c441fb10..5498f413524ece 100644 --- a/ui/src/pages/login/index.tsx +++ b/ui/src/pages/login/index.tsx @@ -22,6 +22,7 @@ import {Form, Input, Button, Checkbox} from 'antd'; import request from 'Utils/request'; import {useHistory} from 'react-router-dom'; import {useTranslation} from 'react-i18next'; +import {login} from 'Src/api/api'; import styles from './index.less'; import './cover.less'; function Login(){ @@ -44,12 +45,6 @@ function Login(){ msg: string; code: number; } - function login(data: any): Promise> { - return request('/rest/v1/login', { - method: 'POST', - headers:{Authorization: data.password?`Basic ${btoa(data.username+':'+data.password)}`:`Basic ${btoa(data.username+':')}`}, - }); - } const onFinish = values => { login(values).then(res=>{ if(res.code===200){ diff --git a/ui/src/pages/logs/index.tsx b/ui/src/pages/logs/index.tsx index a4024de8e2c531..12d7f55a765af8 100644 --- a/ui/src/pages/logs/index.tsx +++ b/ui/src/pages/logs/index.tsx @@ -20,7 +20,7 @@ import React,{useState, useEffect, useRef} from 'react'; import {Typography, Divider, Row, Col, Input, BackTop} from 'antd'; const {Title, Paragraph, Text} = Typography; -import {getLog} from 'Src/utils/api'; +import {getLog} from 'Src/api/api'; const {Search} = Input; import {Result} from '@src/interfaces/http.interface'; export default function Logs(params: any) { diff --git a/ui/src/pages/playground/content/components/data-prev.tsx b/ui/src/pages/playground/content/components/data-prev.tsx index 55a1aa52f81407..6f662a2099fe24 100644 --- a/ui/src/pages/playground/content/components/data-prev.tsx +++ b/ui/src/pages/playground/content/components/data-prev.tsx @@ -18,7 +18,7 @@ */ import React,{useState,useEffect} from 'react'; -import {AdHocAPI} from 'Utils/api'; +import {AdHocAPI} from 'Src/api/api'; import {getDbName} from 'Utils/utils'; import {Row, Empty} from 'antd'; import {FlatBtn} from 'Components/flatbtn'; diff --git a/ui/src/pages/playground/content/content-result.tsx b/ui/src/pages/playground/content/content-result.tsx index 68f806d7568914..21acb36f0f337f 100644 --- a/ui/src/pages/playground/content/content-result.tsx +++ b/ui/src/pages/playground/content/content-result.tsx @@ -176,10 +176,10 @@ export function AdhocContentResult(props) { {tableData.map((item,index) => ( - {item.map(tdData => ( + {item.map((tdData, index) => ( {tdData == '\\N'?'-':tdData} diff --git a/ui/src/pages/playground/content/content-structure.tsx b/ui/src/pages/playground/content/content-structure.tsx index e2866c828244e2..7e9c4b4c34058a 100644 --- a/ui/src/pages/playground/content/content-structure.tsx +++ b/ui/src/pages/playground/content/content-structure.tsx @@ -24,7 +24,7 @@ import React from 'react'; import {FlatBtn} from 'Components/flatbtn'; import {TABLE_DELAY} from 'Constants'; import {useRequest} from '@umijs/hooks'; -import {AdHocAPI} from 'Utils/api'; +import {AdHocAPI} from 'Src/api/api'; import {getDbName} from 'Utils/utils'; import {Result} from '@src/interfaces/http.interface'; import {DataPrev} from './components/data-prev'; diff --git a/ui/src/pages/playground/content/index.tsx b/ui/src/pages/playground/content/index.tsx index 61e4b28b32488e..16dd8535466c92 100644 --- a/ui/src/pages/playground/content/index.tsx +++ b/ui/src/pages/playground/content/index.tsx @@ -29,7 +29,7 @@ import {PlayCircleFilled} from '@ant-design/icons'; import {Switch, Route, Redirect} from 'react-router'; import {AdhocContentResult} from './content-result'; import {useRequest} from '@umijs/hooks'; -import {AdHocAPI} from 'Utils/api'; +import {AdHocAPI} from 'Src/api/api'; import {Result} from '@src/interfaces/http.interface'; import {isSuccess, getDbName, getTimeNow} from 'Utils/utils'; import {CodeMirrorWithFullscreen} from 'Components/codemirror-with-fullscreen/codemirror-with-fullscreen'; diff --git a/ui/src/pages/playground/data-import/index.tsx b/ui/src/pages/playground/data-import/index.tsx index 9a6dc566fe70cd..dac1fe9e1c36cc 100644 --- a/ui/src/pages/playground/data-import/index.tsx +++ b/ui/src/pages/playground/data-import/index.tsx @@ -18,11 +18,10 @@ */ import React,{useState, useEffect, useLayoutEffect} from 'react'; -import {AdHocAPI} from 'Utils/api'; import {getDbName} from 'Utils/utils'; import {Typography, Steps, Button, message, Form, Input, Select, Upload, Table, Empty} from 'antd'; import {UploadOutlined} from '@ant-design/icons'; -import {AdHocAPI, doUp, getUploadData, deleteUploadData} from 'Utils/api'; +import {AdHocAPI, doUp, getUploadData, deleteUploadData} from 'Src/api/api'; const {Step} = Steps; const {Option} = Select; import {getAllTableData} from './import-func'; @@ -324,10 +323,10 @@ export default function DataImport(props: any) { {prevData?.map((item,index) => ( - {item.map(tdData => ( + {item.map((tdData,i) => ( {tdData == '\\N'?'-':tdData} diff --git a/ui/src/pages/playground/tree/index.tsx b/ui/src/pages/playground/tree/index.tsx index 1baccbad456ff9..638c6c408493fa 100644 --- a/ui/src/pages/playground/tree/index.tsx +++ b/ui/src/pages/playground/tree/index.tsx @@ -20,7 +20,7 @@ import React, {useState,useEffect} from 'react'; import {Tree, Spin, Space} from 'antd'; import {TableOutlined, HddOutlined} from '@ant-design/icons'; -import {AdHocAPI} from 'Utils/api'; +import {AdHocAPI} from 'Src/api/api'; import { AdhocContentRouteKeyEnum, } from '../adhoc.data'; diff --git a/ui/src/pages/query-profile/index.tsx b/ui/src/pages/query-profile/index.tsx index 815dbc24ba0dc9..15b6d85a38cbf6 100644 --- a/ui/src/pages/query-profile/index.tsx +++ b/ui/src/pages/query-profile/index.tsx @@ -20,7 +20,7 @@ import React, {useState, useEffect, useRef} from 'react'; import {Typography, Button, Row, Col} from 'antd'; const {Text, Title, Paragraph} = Typography; -import {queryProfile} from 'Utils/api'; +import {queryProfile} from 'Src/api/api'; import Table from 'Src/components/table'; import {useHistory} from 'react-router-dom'; export default function QueryProfile(params: any) { diff --git a/ui/src/pages/session/index.tsx b/ui/src/pages/session/index.tsx index 7b49775af0dbf0..393cf324d78bb0 100644 --- a/ui/src/pages/session/index.tsx +++ b/ui/src/pages/session/index.tsx @@ -20,7 +20,7 @@ import React, {useState, useEffect} from 'react'; import {Typography, Button, Row, Col} from 'antd'; const {Text, Title, Paragraph} = Typography; -import {getSession} from 'Utils/api'; +import {getSession} from 'Src/api/api'; import Table from 'Src/components/table'; // import {useHistory} from 'react-router-dom'; export default function Session(params: any) { diff --git a/ui/src/pages/system/index.tsx b/ui/src/pages/system/index.tsx index 6af60da5a98137..26720af8751f5b 100644 --- a/ui/src/pages/system/index.tsx +++ b/ui/src/pages/system/index.tsx @@ -21,7 +21,7 @@ import React, {useState, useEffect} from 'react'; import {Typography, Button, Row, Col} from 'antd'; const {Text, Title, Paragraph} = Typography; -import {getSystem} from 'Utils/api'; +import {getSystem} from 'Src/api/api'; import Table from 'Src/components/table'; import {useHistory} from 'react-router-dom'; export default function System(params: any) { diff --git a/ui/src/utils/api.ts b/ui/src/utils/api.ts index cf7c9a814ffcba..704368c38eb905 100644 --- a/ui/src/utils/api.ts +++ b/ui/src/utils/api.ts @@ -18,6 +18,13 @@ import {API_BASE} from 'Constants'; import request from 'Utils/request'; import {Result} from '@src/interfaces/http.interface'; +//login +export function login(data: any): Promise> { + return request('/rest/v1/login', { + method: 'POST', + headers:{Authorization: data.password?`Basic ${btoa(data.username+':'+data.password)}`:`Basic ${btoa(data.username+':')}`}, + }); +} //logout export function logOut(): Promise> { return request(`/rest/v1/logout`,{ @@ -123,5 +130,5 @@ export const AdHocAPI = { logOut, getHardwareInfo, getUploadData, - deleteUploadData + deleteUploadData, }; \ No newline at end of file diff --git a/ui/src/utils/request.tsx b/ui/src/utils/request.tsx index 9a7cf7ba16c516..de137e7ac88cb3 100644 --- a/ui/src/utils/request.tsx +++ b/ui/src/utils/request.tsx @@ -26,7 +26,7 @@ import React from 'react'; import {message, Modal} from 'antd'; import {ExclamationCircleOutlined} from '@ant-design/icons'; import {Trans} from 'react-i18next'; - +let basePath; function checkStatus(response) { if (response.status >= 200 && response.status < 300) { return response; @@ -65,6 +65,10 @@ function checkStatus(response) { * @return {Object} */ export default async function request(url, options = {}, tipSuccess = false, tipError = true, fullResponse = false) { + if(!basePath){ + const data = await fetch('/api/basepath') + basePath = await data.json(); + } const newOptions = {credentials: 'include', ...options}; if (newOptions.method === 'POST' || newOptions.method === 'PUT') { newOptions.headers = newOptions.isUpload @@ -79,6 +83,9 @@ export default async function request(url, options = {}, tipSuccess = false, tip if (typeof newOptions.body === 'object' && !newOptions.isUpload) { newOptions.body = JSON.stringify(newOptions.body); } + if (basePath.data?.path && basePath.data?.enable) { + url = basePath.data.path + url + } const response = await fetch(url, newOptions); if ( response.url.includes('dataIntegrationApi') @@ -113,4 +120,3 @@ export default async function request(url, options = {}, tipSuccess = false, tip } return data; } - From 28241feee3390efd2b873af6288ef2556096d6a1 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 16 Sep 2020 18:53:28 +0800 Subject: [PATCH 03/12] Fix bug --- ui/src/{utils => api}/api.ts | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ui/src/{utils => api}/api.ts (100%) diff --git a/ui/src/utils/api.ts b/ui/src/api/api.ts similarity index 100% rename from ui/src/utils/api.ts rename to ui/src/api/api.ts From 98cc3c9c18b9b72982e4875e74f1b06d14960b21 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 23 Sep 2020 19:21:38 +0800 Subject: [PATCH 04/12] add 401 json --- .../apache/doris/httpv2/config/WebConfigurer.java | 1 - .../doris/httpv2/entity/ResponseEntityBuilder.java | 12 ++++++++---- .../apache/doris/httpv2/rest/RestApiStatusCode.java | 6 +++++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/WebConfigurer.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/WebConfigurer.java index 272e84b0543a74..2968604c8a612a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/WebConfigurer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/WebConfigurer.java @@ -56,7 +56,6 @@ public void addViewControllers(ViewControllerRegistry registry) { registry.addViewController("/notFound").setViewName("forward:/index.html"); } - @Bean public WebServerFactoryCustomizer containerCustomizer() { return container -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseEntityBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseEntityBuilder.java index 1b46c090ec0559..c3d7ee53bac422 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseEntityBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseEntityBuilder.java @@ -28,7 +28,8 @@ public class ResponseEntityBuilder { public static ResponseEntity badRequest(Object data) { - return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(data); + ResponseBody body = new ResponseBody().code(RestApiStatusCode.BAD_REQUEST).msg().data(data); + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(body); } public static ResponseEntity okWithCommonError(String msg) { @@ -47,14 +48,17 @@ public static ResponseEntity ok() { } public static ResponseEntity unauthorized(Object data) { - return ResponseEntity.status(HttpStatus.UNAUTHORIZED).body(data); + ResponseBody body = new ResponseBody().code(RestApiStatusCode.UNAUTHORIZED).data(data); + return ResponseEntity.status(HttpStatus.UNAUTHORIZED).body(body); } public static ResponseEntity internalError(Object data) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(data); + ResponseBody body = new ResponseBody().code(RestApiStatusCode.INTERNAL_ERROR).data(data); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(body); } public static ResponseEntity notFound(Object data) { - return ResponseEntity.status(HttpStatus.NOT_FOUND).body(data); + ResponseBody body = new ResponseBody().code(RestApiStatusCode.NOT_FOUND).data(data); + return ResponseEntity.status(HttpStatus.NOT_FOUND).body(body); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestApiStatusCode.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestApiStatusCode.java index d260cc36b56cc3..612e425f84b2e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestApiStatusCode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestApiStatusCode.java @@ -19,7 +19,11 @@ public enum RestApiStatusCode { OK(0), - COMMON_ERROR(1); + COMMON_ERROR(1), + UNAUTHORIZED(2), + INTERNAL_ERROR(3), + NOT_FOUND(4), + BAD_REQUEST(5); public int code; From b4c78099822919b0eb37df5556714673300788b7 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 24 Sep 2020 12:44:19 +0800 Subject: [PATCH 05/12] change all to 200 --- .../apache/doris/httpv2/config/WebConfigurer.java | 2 +- .../doris/httpv2/entity/ResponseEntityBuilder.java | 13 +++++++------ .../apache/doris/httpv2/rest/RestApiStatusCode.java | 8 ++++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/WebConfigurer.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/WebConfigurer.java index 2968604c8a612a..d9a3572a3a63ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/WebConfigurer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/WebConfigurer.java @@ -53,7 +53,7 @@ public void addCorsMappings(CorsRegistry registry) { @Override public void addViewControllers(ViewControllerRegistry registry) { - registry.addViewController("/notFound").setViewName("forward:/index.html"); + registry.addViewController("/notFound").setStatusCode(HttpStatus.OK).setViewName("forward:/index.html"); } @Bean diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseEntityBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseEntityBuilder.java index c3d7ee53bac422..f557525c2bbcd6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseEntityBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseEntityBuilder.java @@ -24,12 +24,13 @@ /** * A utility class for creating a ResponseEntity easier. + * All response will return with http code 200, and a internal code represent the real code. */ public class ResponseEntityBuilder { public static ResponseEntity badRequest(Object data) { - ResponseBody body = new ResponseBody().code(RestApiStatusCode.BAD_REQUEST).msg().data(data); - return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(body); + ResponseBody body = new ResponseBody().code(RestApiStatusCode.BAD_REQUEST).data(data); + return ResponseEntity.status(HttpStatus.OK).body(body); } public static ResponseEntity okWithCommonError(String msg) { @@ -49,16 +50,16 @@ public static ResponseEntity ok() { public static ResponseEntity unauthorized(Object data) { ResponseBody body = new ResponseBody().code(RestApiStatusCode.UNAUTHORIZED).data(data); - return ResponseEntity.status(HttpStatus.UNAUTHORIZED).body(body); + return ResponseEntity.status(HttpStatus.OK).body(body); } public static ResponseEntity internalError(Object data) { - ResponseBody body = new ResponseBody().code(RestApiStatusCode.INTERNAL_ERROR).data(data); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(body); + ResponseBody body = new ResponseBody().code(RestApiStatusCode.INTERNAL_SERVER_ERROR).data(data); + return ResponseEntity.status(HttpStatus.OK).body(body); } public static ResponseEntity notFound(Object data) { ResponseBody body = new ResponseBody().code(RestApiStatusCode.NOT_FOUND).data(data); - return ResponseEntity.status(HttpStatus.NOT_FOUND).body(body); + return ResponseEntity.status(HttpStatus.OK).body(body); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestApiStatusCode.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestApiStatusCode.java index 612e425f84b2e1..6952f2abb7ce17 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestApiStatusCode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestApiStatusCode.java @@ -20,10 +20,10 @@ public enum RestApiStatusCode { OK(0), COMMON_ERROR(1), - UNAUTHORIZED(2), - INTERNAL_ERROR(3), - NOT_FOUND(4), - BAD_REQUEST(5); + UNAUTHORIZED(401), + BAD_REQUEST(403), + NOT_FOUND(404), + INTERNAL_SERVER_ERROR(500); public int code; From 38ab3a73dba8ce29acd1d265a07a733116930a8a Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 24 Sep 2020 15:39:13 +0800 Subject: [PATCH 06/12] unify response body --- .../httpv2/controller/BaseController.java | 56 ++++++++++--------- .../httpv2/entity/ResponseEntityBuilder.java | 10 ++-- 2 files changed, 36 insertions(+), 30 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/BaseController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/BaseController.java index fbeb773ece1b60..3bbea54f02c329 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/BaseController.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/BaseController.java @@ -59,40 +59,45 @@ public class BaseController { public static final String PALO_SESSION_ID = "PALO_SESSION_ID"; private static final int PALO_SESSION_EXPIRED_TIME = 3600 * 24; // one day - // We first check cookie, if not admin, we check http's authority header public void checkAuthWithCookie(HttpServletRequest request, HttpServletResponse response) { checkWithCookie(request, response, true); } public ActionAuthorizationInfo checkWithCookie(HttpServletRequest request, HttpServletResponse response, boolean checkAuth) { - ActionAuthorizationInfo authInfo = checkCookie(request, response, checkAuth); - if (authInfo != null) { + // First we check if the request has Authorization header. + String encodedAuthString = request.getHeader("Authorization"); + if (encodedAuthString != null) { + // If has Authorization header, check auth info + ActionAuthorizationInfo authInfo = getAuthorizationInfo(request); + UserIdentity currentUser = checkPassword(authInfo); + + if (checkAuth) { + checkGlobalAuth(currentUser, PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, + PaloPrivilege.NODE_PRIV), CompoundPredicate.Operator.OR)); + } + + SessionValue value = new SessionValue(); + value.currentUser = currentUser; + value.password = authInfo.password; + addSession(request, response, value); + + ConnectContext ctx = new ConnectContext(null); + ctx.setQualifiedUser(authInfo.fullUserName); + ctx.setRemoteIP(authInfo.remoteIp); + ctx.setCurrentUserIdentity(currentUser); + ctx.setCatalog(Catalog.getCurrentCatalog()); + ctx.setCluster(SystemInfoService.DEFAULT_CLUSTER); + ctx.setThreadLocalInfo(); + LOG.debug("check auth without cookie success for user: {}, thread: {}", + currentUser, Thread.currentThread().getId()); return authInfo; } - // cookie is invalid. check auth info in request - authInfo = getAuthorizationInfo(request); - UserIdentity currentUser = checkPassword(authInfo); - - if (checkAuth) { - checkGlobalAuth(currentUser, PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, - PaloPrivilege.NODE_PRIV), CompoundPredicate.Operator.OR)); + // No Authorization header, check cookie + ActionAuthorizationInfo authInfo = checkCookie(request, response, checkAuth); + if (authInfo == null) { + throw new UnauthorizedException("Cookie is invalid"); } - - SessionValue value = new SessionValue(); - value.currentUser = currentUser; - value.password = authInfo.password; - addSession(request, response, value); - - ConnectContext ctx = new ConnectContext(null); - ctx.setQualifiedUser(authInfo.fullUserName); - ctx.setRemoteIP(authInfo.remoteIp); - ctx.setCurrentUserIdentity(currentUser); - ctx.setCatalog(Catalog.getCurrentCatalog()); - ctx.setCluster(SystemInfoService.DEFAULT_CLUSTER); - ctx.setThreadLocalInfo(); - LOG.debug("check auth without cookie success for user: {}, thread: {}", - currentUser, Thread.currentThread().getId()); return authInfo; } @@ -294,3 +299,4 @@ protected String getCurrentFrontendURL() { return "http://" + FrontendOptions.getLocalHostAddress() + ":" + Config.http_port; } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseEntityBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseEntityBuilder.java index f557525c2bbcd6..9b1034dfd04c4d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseEntityBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ResponseEntityBuilder.java @@ -29,12 +29,12 @@ public class ResponseEntityBuilder { public static ResponseEntity badRequest(Object data) { - ResponseBody body = new ResponseBody().code(RestApiStatusCode.BAD_REQUEST).data(data); + ResponseBody body = new ResponseBody().code(RestApiStatusCode.BAD_REQUEST).msg("Bad Request").data(data); return ResponseEntity.status(HttpStatus.OK).body(body); } public static ResponseEntity okWithCommonError(String msg) { - ResponseBody body = new ResponseBody().code(RestApiStatusCode.COMMON_ERROR).commonError(msg); + ResponseBody body = new ResponseBody().code(RestApiStatusCode.COMMON_ERROR).msg("Error").data(msg); return ResponseEntity.status(HttpStatus.OK).body(body); } @@ -49,17 +49,17 @@ public static ResponseEntity ok() { } public static ResponseEntity unauthorized(Object data) { - ResponseBody body = new ResponseBody().code(RestApiStatusCode.UNAUTHORIZED).data(data); + ResponseBody body = new ResponseBody().code(RestApiStatusCode.UNAUTHORIZED).msg("Unauthorized").data(data); return ResponseEntity.status(HttpStatus.OK).body(body); } public static ResponseEntity internalError(Object data) { - ResponseBody body = new ResponseBody().code(RestApiStatusCode.INTERNAL_SERVER_ERROR).data(data); + ResponseBody body = new ResponseBody().code(RestApiStatusCode.INTERNAL_SERVER_ERROR).msg("Internal Error").data(data); return ResponseEntity.status(HttpStatus.OK).body(body); } public static ResponseEntity notFound(Object data) { - ResponseBody body = new ResponseBody().code(RestApiStatusCode.NOT_FOUND).data(data); + ResponseBody body = new ResponseBody().code(RestApiStatusCode.NOT_FOUND).msg("Not Found").data(data); return ResponseEntity.status(HttpStatus.OK).body(body); } } From a38d1a59cc791a1784813320deaee98bc1297020 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 28 Sep 2020 15:16:53 +0800 Subject: [PATCH 07/12] fix basepath --- ui/README.md | 2 +- ui/config/helpers.js | 4 +- ui/config/paths.js | 2 +- ui/config/webpack.common.js | 2 +- ui/config/webpack.dev.js | 2 +- ui/public/locales/en-us.json | 6 +- ui/public/locales/zh-cn.json | 8 ++- ui/src/App.tsx | 9 ++- ui/src/api/api.ts | 2 +- ui/src/components/table/index.tsx | 4 +- ui/src/components/table/table.utils.tsx | 10 +-- ui/src/index.html | 12 +++- ui/src/pages/404/index.tsx | 34 +++++++++ ui/src/pages/layout/index.tsx | 15 ++-- ui/src/pages/login/index.tsx | 2 +- ui/src/pages/logs/index.tsx | 2 +- .../content/components/data-prev.tsx | 2 +- .../playground/content/content-result.tsx | 6 +- .../playground/content/content-structure.tsx | 11 ++- ui/src/pages/playground/content/index.tsx | 5 +- ui/src/pages/playground/data-import/index.tsx | 32 ++++----- ui/src/pages/playground/tree/index.tsx | 1 - ui/src/pages/query-profile/index.tsx | 7 +- ui/src/pages/system/index.tsx | 1 + ui/src/router/index.ts | 19 +++-- ui/src/router/renderRouter.tsx | 8 ++- ui/src/utils/request.tsx | 70 +++++++++++++++---- ui/src/utils/utils.ts | 36 ++++++---- 28 files changed, 218 insertions(+), 96 deletions(-) create mode 100644 ui/src/pages/404/index.tsx diff --git a/ui/README.md b/ui/README.md index b4861ef5f5f098..6caeed6e8a7b99 100644 --- a/ui/README.md +++ b/ui/README.md @@ -35,7 +35,7 @@ Start server. ```bash $ npm run dev -# visit http://localhost:8233 +# visit http://localhost:8030 ``` Submit code diff --git a/ui/config/helpers.js b/ui/config/helpers.js index e0250a53a81a08..c968258160e8c0 100644 --- a/ui/config/helpers.js +++ b/ui/config/helpers.js @@ -25,7 +25,5 @@ const path = require('path'); const rootPath = path.resolve(__dirname, '..'); -const pingoPath = path.resolve(__dirname, '../../'); const root = (...args) => path.join(...[rootPath].concat(args)); - -module.exports = {root, pingoPath}; +module.exports = {root}; diff --git a/ui/config/paths.js b/ui/config/paths.js index 18772a64db4cae..0ab397b392d0e6 100644 --- a/ui/config/paths.js +++ b/ui/config/paths.js @@ -37,5 +37,5 @@ module.exports = { Services: helpers.root('/src/services'), Constants: helpers.root('/src/constants'), '@hooks': helpers.root('/src/hooks'), - '@src': helpers.root('/src') + '@src': helpers.root('/src'), }; diff --git a/ui/config/webpack.common.js b/ui/config/webpack.common.js index fb2ccfd5dce2ed..f5a4b713dfcf7a 100644 --- a/ui/config/webpack.common.js +++ b/ui/config/webpack.common.js @@ -45,7 +45,7 @@ module.exports = { entry: paths.entryApp, output: { path: paths.distSrc, - publicPath: '/', + // publicPath: '', filename: '[name].[hash].js', chunkFilename: '[name].[hash].js' }, diff --git a/ui/config/webpack.dev.js b/ui/config/webpack.dev.js index 768cb7f3d70514..03628af0d5ef1c 100644 --- a/ui/config/webpack.dev.js +++ b/ui/config/webpack.dev.js @@ -44,7 +44,7 @@ module.exports = merge(baseConfig, { host: 'localhost', open: true, contentBase: path.join(__dirname, 'dist'), - port: 8233, + port: 8030, proxy: { '/api': { target: 'http://127.0.0.1:8030', diff --git a/ui/public/locales/en-us.json b/ui/public/locales/en-us.json index 8f12e77bebdc64..6894f33e5d2b35 100644 --- a/ui/public/locales/en-us.json +++ b/ui/public/locales/en-us.json @@ -2,6 +2,7 @@ "username": "Username", "password": "Password", "signOut": "Sign out", + "loginWarning":"Incorrect username or password", "exitSuccessfully":"Exit successfully", "editor": "Editor", @@ -28,7 +29,7 @@ "endTime":"End Time", "currentDatabase":"Current Database", "executionFailed":"Execution failed", - "uploadWarning":"Please upload files", + "uploadWarning":"Please select file", "upload": "Upload", "delimiterWarning":"Please select a separator", "uploadedFiles":"Uploaded Files", @@ -44,5 +45,6 @@ "loadButton":"Import", "successfulOperation":"The operation was successful", "tips":"Tips", - "fileSizeWarning": "File size cannot exceed 100M" + "fileSizeWarning": "File size cannot exceed 100M", + "selectWarning": "Please select a table" } diff --git a/ui/public/locales/zh-cn.json b/ui/public/locales/zh-cn.json index 0b4faaf99fe3a5..080c7be82de5ce 100644 --- a/ui/public/locales/zh-cn.json +++ b/ui/public/locales/zh-cn.json @@ -2,7 +2,8 @@ "username": "用户名", "password": "密码", "exitSuccessfully": "退出成功", - + "loginWarning":"账号或密码错误", + "editor": "编辑器", "format": "格式化", "clear": "清空编辑器", @@ -28,7 +29,7 @@ "endTime":"结束时间", "currentDatabase": "当前数据库", "executionFailed": "上传失败", - "uploadWarning": "请上传文件", + "uploadWarning": "请选择文件", "upload": "上传", "delimiterWarning": "请选择分隔符", "uploadedFiles": "已上传文件列表", @@ -44,5 +45,6 @@ "loadButton": "导入", "successfulOperation": "操作成功", "tips": "提示", - "fileSizeWarning": "文件大小不能超过100m" + "fileSizeWarning": "文件大小不能超过100m", + "selectWarning": "请选择表" } diff --git a/ui/src/App.tsx b/ui/src/App.tsx index 60f7f55ae53f02..fbcc772fab7a5f 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -23,15 +23,18 @@ * @since 2020/08/19 */ import React from 'react'; -import {BrowserRouter as Router, Switch, Redirect} from 'react-router-dom'; +import {BrowserRouter as Router, Switch} from 'react-router-dom'; // import {renderRoutes} from 'react-router-config'; import routes from './router'; import renderRoutes from './router/renderRouter'; import 'antd/dist/antd.css'; - +import {getBasePath} from 'Src/utils/utils'; +let basePath = getBasePath(); function App() { return ( - + {renderRoutes(routes.routes)} diff --git a/ui/src/api/api.ts b/ui/src/api/api.ts index 704368c38eb905..935958b3d357a8 100644 --- a/ui/src/api/api.ts +++ b/ui/src/api/api.ts @@ -73,7 +73,7 @@ export function getSession(data: any): Promise> { } //config export function getConfig(data: any): Promise> { - return request('rest/v1/config/fe/'); + return request('/rest/v1/config/fe/'); } //query begin export function getDatabaseList(data: any): Promise> { diff --git a/ui/src/components/table/index.tsx b/ui/src/components/table/index.tsx index cdccdb435e3a7b..f775739e5522d3 100644 --- a/ui/src/components/table/index.tsx +++ b/ui/src/components/table/index.tsx @@ -24,7 +24,7 @@ import {getColumns, filterTableData} from './table.utils.tsx'; import './index.less'; export default function SortFilterTable(props: any) { - const {isFilter=false, isSort=false, allTableData, isInner, isSystem=false} = props; + const {isFilter=false, isSort=false, allTableData, isInner, isSystem=false, path=''} = props; const [tableData, setTableData] = useState([]); const [localColumns, setColumns] = useState([]); // function onChange(pagination, filters, sorter, extra) { @@ -39,7 +39,7 @@ export default function SortFilterTable(props: any) { ); useEffect(() => { if(allTableData.rows&&allTableData.column_names){ - setColumns(getColumns(allTableData.column_names, isSort, isInner, allTableData.href_columns||allTableData.href_column)); + setColumns(getColumns(allTableData.column_names, isSort, isInner, allTableData.href_columns||allTableData.href_column, path)); setTableData(allTableData.rows); } }, [allTableData]); diff --git a/ui/src/components/table/table.utils.tsx b/ui/src/components/table/table.utils.tsx index e1e6a0e33b4f98..fc2a17d90494c4 100644 --- a/ui/src/components/table/table.utils.tsx +++ b/ui/src/components/table/table.utils.tsx @@ -25,16 +25,16 @@ function sortItems(a: any,b: any, item: string) { } return a[item].localeCompare(b[item]); } -function getLinkItem(text, record, index, isInner, item, hrefColumn){ +function getLinkItem(text, record, index, isInner, item, hrefColumn, path){ if (isInner && hrefColumn && (hrefColumn.indexOf(item) !== -1)&&record.__hrefPaths) { if (record.__hrefPaths[hrefColumn.indexOf(item)].includes('http')) { return {text}; } - return {text}; + return {text}; } return text === '\\N' ? '-' : text; } -export function getColumns(params: string[], isSort: boolean, isInner, hrefColumn) { +export function getColumns(params: string[], isSort: boolean, isInner, hrefColumn, path) { if(!params||params.length === 0){return [];} let arr = params.map(item=> { if (isSort) { @@ -43,14 +43,14 @@ export function getColumns(params: string[], isSort: boolean, isInner, hrefColum dataIndex: item, className: 'pr-25', sorter: (a,b)=>sortItems(a, b, item), - render:(text, record, index)=>getLinkItem(text,record, index, isInner, item, hrefColumn), + render:(text, record, index)=>getLinkItem(text,record, index, isInner, item, hrefColumn, path), }; } return { title: item, dataIndex: item, className: 'pr-25', - render:(text, record, index)=>getLinkItem(text, record, index, isInner, item, hrefColumn), + render:(text, record, index)=>getLinkItem(text, record, index, isInner, item, hrefColumn, path), }; }); return arr; diff --git a/ui/src/index.html b/ui/src/index.html index a7a39a3255012a..cae196ecdae97e 100644 --- a/ui/src/index.html +++ b/ui/src/index.html @@ -19,7 +19,17 @@ - + Apache Doris diff --git a/ui/src/pages/404/index.tsx b/ui/src/pages/404/index.tsx new file mode 100644 index 00000000000000..29c838f3f0842c --- /dev/null +++ b/ui/src/pages/404/index.tsx @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from 'react'; +import {Result, Button} from 'antd'; +import {useHistory} from 'react-router-dom'; +export default function Backend(params: any) { + const history = useHistory(); + return( + history.push('/home')}>Back home} + /> + ); +} + diff --git a/ui/src/pages/layout/index.tsx b/ui/src/pages/layout/index.tsx index d2947393974b3a..d87e3e383ff389 100644 --- a/ui/src/pages/layout/index.tsx +++ b/ui/src/pages/layout/index.tsx @@ -23,7 +23,7 @@ * @since 2020/08/19 */ import React, {useState} from 'react'; -import {Layout, Menu, Dropdown, message} from 'antd'; +import {Layout, Menu, Dropdown, notification} from 'antd'; import { CaretDownOutlined, LogoutOutlined} from '@ant-design/icons'; import {renderRoutes} from 'react-router-config'; import {useHistory} from 'react-router-dom'; @@ -33,7 +33,6 @@ import {logOut} from 'Src/api/api'; import './index.css'; import styles from './index.less'; const {Header, Content, Footer} = Layout; - function Layouts(props: any) { let { t } = useTranslation(); const [route, setRoute] = useState(props.route.routes); @@ -42,17 +41,21 @@ function Layouts(props: any) { //Jump page function handleClick(e) { setCurrent(e.key); - if (e.key === '/System' ) { + if (e.key.includes('/System')) { history.push(`${e.key}?path=/`); return; } if (location.pathname === e.key) { location.reload(); } - history.push(e.key); if(location.pathname.includes('Playground')){ + history.push(e.key); location.reload(); } + history.push(e.key); + // if(location.pathname.includes('Playground')){ + // location.reload(); + // } } function clearAllCookie() { var keys = document.cookie.match(/[^ =;]+(?=\=)/g); @@ -65,7 +68,7 @@ function Layouts(props: any) { logOut().then((res)=>{ localStorage.setItem('username',''); clearAllCookie(); - message.success(t('exitSuccessfully')) + notification.success({message: t('exitSuccessfully')}) history.push('/login'); }) } @@ -91,7 +94,7 @@ function Layouts(props: any) { {routes?.routes[1]?.routes?.map(item => { - if (item.path !== '/login'&&item.path !== '/home') { + if (item.title !== 'Login'&&item.title !== 'Home') { return ( {item.title} diff --git a/ui/src/pages/login/index.tsx b/ui/src/pages/login/index.tsx index 5498f413524ece..a7454cb2350d53 100644 --- a/ui/src/pages/login/index.tsx +++ b/ui/src/pages/login/index.tsx @@ -50,7 +50,7 @@ function Login(){ if(res.code===200){ history.push('/home'); localStorage.setItem('username', username) - } + } }); }; diff --git a/ui/src/pages/logs/index.tsx b/ui/src/pages/logs/index.tsx index 12d7f55a765af8..9ca37f2860a57e 100644 --- a/ui/src/pages/logs/index.tsx +++ b/ui/src/pages/logs/index.tsx @@ -29,7 +29,7 @@ export default function Logs(params: any) { const [LogContents, setLogContents] = useState({}); function getLogData(data){ getLog(data).then(res=>{ - if(res.data){ + if(res.data && res.msg === 'success'){ if(res.data.LogConfiguration){ setLogConfiguration(res.data.LogConfiguration); } diff --git a/ui/src/pages/playground/content/components/data-prev.tsx b/ui/src/pages/playground/content/components/data-prev.tsx index 6f662a2099fe24..96ab1e0f29e88b 100644 --- a/ui/src/pages/playground/content/components/data-prev.tsx +++ b/ui/src/pages/playground/content/components/data-prev.tsx @@ -32,7 +32,7 @@ export function DataPrev(props: any) { db_name, body:{stmt:`SELECT * FROM ${db_name}.${tbl_name} LIMIT 10`}, }).then(res=>{ - if (res && res.data) { + if (res && res.msg === 'success') { setTableData(res.data); } }) diff --git a/ui/src/pages/playground/content/content-result.tsx b/ui/src/pages/playground/content/content-result.tsx index 21acb36f0f337f..703f73097ad421 100644 --- a/ui/src/pages/playground/content/content-result.tsx +++ b/ui/src/pages/playground/content/content-result.tsx @@ -60,9 +60,9 @@ export function AdhocContentResult(props) { if (runningQueryInfo.data?.type === 'exec_status') { setResStatus(runningQueryInfo.data.status) } else { - const tableData = runningQueryInfo.data?(runningQueryInfo.data?.data).slice(0,20):[]; + const tableData = (runningQueryInfo.data && typeof(runningQueryInfo.data) === 'object')?(runningQueryInfo.data?.data).slice(0,20):[]; setTableDate(tableData); - setTotal(runningQueryInfo.data?.data.length); + setTotal((runningQueryInfo.data && typeof(runningQueryInfo.data) === 'object')?runningQueryInfo.data.length:0); } setRunningQueryInfo(runningQueryInfo); },[location.state]); @@ -123,7 +123,7 @@ export function AdhocContentResult(props) { ) : ( } - text={"执行失败: "+runningQueryInfo.msg} + text={"执行失败: "+runningQueryInfo.msg +' '+ runningQueryInfo.data} color="red" style={{ marginBottom: 10, diff --git a/ui/src/pages/playground/content/content-structure.tsx b/ui/src/pages/playground/content/content-structure.tsx index 7e9c4b4c34058a..703a68c449cc83 100644 --- a/ui/src/pages/playground/content/content-structure.tsx +++ b/ui/src/pages/playground/content/content-structure.tsx @@ -18,7 +18,7 @@ */ import React, {useState,useEffect} from 'react'; -import {Tabs, Row, Table} from 'antd'; +import {Tabs, Row, Table, notification} from 'antd'; import {TabPaneType,QUERY_TABTYPE} from '../adhoc.data'; import React from 'react'; import {FlatBtn} from 'Components/flatbtn'; @@ -57,10 +57,17 @@ export function ContentStructure(props: any) { }, [cols]) const history = useHistory(); + function goImport(){ + if(db_name && tbl_name){ + history.push('/Playground/import/'+db_name+'-'+tbl_name); + } else { + notification.error({message: t('selectWarning')}); + } + } return ( {if (key ===QUERY_TABTYPE.key3) {history.push('/Playground/import/'+db_name+'-'+tbl_name);}}} + onChange={key => {if (key ===QUERY_TABTYPE.key3) {goImport()}}} > diff --git a/ui/src/pages/playground/content/index.tsx b/ui/src/pages/playground/content/index.tsx index 16dd8535466c92..c4c5c1d90ec340 100644 --- a/ui/src/pages/playground/content/index.tsx +++ b/ui/src/pages/playground/content/index.tsx @@ -24,7 +24,7 @@ import { CODEMIRROR_OPTIONS, AdhocContentRouteKeyEnum, } from '../adhoc.data'; -import {Button, Row, Col, message} from 'antd'; +import {Button, Row, Col, notification} from 'antd'; import {PlayCircleFilled} from '@ant-design/icons'; import {Switch, Route, Redirect} from 'react-router'; import {AdhocContentResult} from './content-result'; @@ -46,7 +46,6 @@ require('react-resizable/css/styles.css'); let editorInstance: any; let isQueryTableClicked = false; let isFieldNameInserted = false; - export function AdHocContent(props: any) { let { t } = useTranslation(); const {match} = props; @@ -80,7 +79,7 @@ export function AdHocContent(props: any) { } }, onError: () => { - message.error(t('errMsg')); + notification.error({message: t('errMsg')}); runSQLSuccessSubject.next(false); }, }, diff --git a/ui/src/pages/playground/data-import/index.tsx b/ui/src/pages/playground/data-import/index.tsx index dac1fe9e1c36cc..5e58151581cee4 100644 --- a/ui/src/pages/playground/data-import/index.tsx +++ b/ui/src/pages/playground/data-import/index.tsx @@ -19,7 +19,7 @@ import React,{useState, useEffect, useLayoutEffect} from 'react'; import {getDbName} from 'Utils/utils'; -import {Typography, Steps, Button, message, Form, Input, Select, Upload, Table, Empty} from 'antd'; +import {Typography, Steps, Button, notification, Form, Input, Select, Upload, Table, Empty} from 'antd'; import {UploadOutlined} from '@ant-design/icons'; import {AdHocAPI, doUp, getUploadData, deleteUploadData} from 'Src/api/api'; const {Step} = Steps; @@ -31,8 +31,11 @@ import {useTranslation} from 'react-i18next'; import 'antd/lib/style/themes/default.less'; import getColumns from '../content/getColumns'; import './index.less'; +import 'antd/dist/antd.css'; +import {getBasePath} from 'Src/utils/utils'; export default function DataImport(props: any) { let { t } = useTranslation(); + let basePath = getBasePath(); const history = useHistory(); const [header, setHeader] = useState([]) const [rowId, setRowId] = useState() @@ -66,18 +69,15 @@ export default function DataImport(props: any) { }; const uploadData = { name: 'file', - action: `/api/default_cluster/${db_name}/${tbl_name}/upload`, + action: `${basePath}/api/default_cluster/${db_name}/${tbl_name}/upload`, data:{ column_separator, preview:'true', }, - headers: { - authorization: 'authorization-text', - }, fileList, beforeUpload(file){ if (file.size/1024/1024 > 100) { - message.error(t('fileSizeWarning')); + notification.error({message: t('fileSizeWarning')}); return false; } return true @@ -87,10 +87,10 @@ export default function DataImport(props: any) { // } if (info.file.status === 'done') { - message.success(`${info.file.name} file uploaded successfully`); + notification.success({message: `${info.file.name} file uploaded successfully`}); getUploadList() } else if (info.file.status === 'error') { - message.error(`${info.file.name} file upload failed.`); + notification.error({message: `${info.file.name} file upload failed.`}); setHeaderUploadData([]); setColsUploadData([]) } @@ -116,7 +116,7 @@ export default function DataImport(props: any) { function next() { const num = current + 1; if (current === 1 && !prevBackData) { - message.error(t('uploadWarning')); + notification.error({message: t('uploadWarning')}); return; } setCurrent( num ); @@ -136,7 +136,7 @@ export default function DataImport(props: any) { res => { // const endTime = getTimeNow(); const {db_name, tbl_name} = getDbName(); - if (res) { + if (res && res.msg === 'success') { let cols = res.data[tbl_name]?.schema; setCols(cols); setHeader(getColumns(cols[0], false, false)) @@ -147,7 +147,7 @@ export default function DataImport(props: any) { } ).catch( () => { - message.error(t('errMsg')); + notification.error({message:t('errMsg')}); } ) } @@ -165,9 +165,9 @@ export default function DataImport(props: any) { column_separator: prevBackData.columnSeparator }; doUp(params).then(res=>{ - if(res){ + if(res && res.msg === 'success'){ if(res.data){ - message.success(`${res.msg}`); + notification.success({message: `${res.msg}`}); ImportResult(res.data,()=>{ history.push(`/Playground/structure/${db_name}-${tbl_name}`); }); @@ -176,7 +176,7 @@ export default function DataImport(props: any) { }); }) .catch(errorInfo => { - message.error(`${errorInfo}`); + notification.error({message: `${errorInfo}`}); }); } useEffect(() => { @@ -195,7 +195,7 @@ export default function DataImport(props: any) { file_uuid:data.uuid, preview:true }).then((res)=>{ - if (res.data) { + if (res.data && res.msg === 'success') { const data = res.data; setPrevBackData(data); setPrevData(getAllTableData(data.maxColNum , data.lines)); @@ -213,7 +213,7 @@ export default function DataImport(props: any) { db_name, tbl_name, }).then((res)=>{ - if(res.data){ + if(res.data && res.msg === 'success'){ let data = res.data; setHeaderUploadData(getColumns(data[0], deleteUpload, true)); setColsUploadData(data) diff --git a/ui/src/pages/playground/tree/index.tsx b/ui/src/pages/playground/tree/index.tsx index 638c6c408493fa..658fe82dbdfb53 100644 --- a/ui/src/pages/playground/tree/index.tsx +++ b/ui/src/pages/playground/tree/index.tsx @@ -32,7 +32,6 @@ interface DataNode { } const initTreeDate: DataNode[] = []; - function updateTreeData(list: DataNode[], key: React.Key, children: DataNode[]): DataNode[] { return list.map(node => { if (node.key === key) { diff --git a/ui/src/pages/query-profile/index.tsx b/ui/src/pages/query-profile/index.tsx index 15b6d85a38cbf6..7beb2aea162e4e 100644 --- a/ui/src/pages/query-profile/index.tsx +++ b/ui/src/pages/query-profile/index.tsx @@ -31,7 +31,7 @@ export default function QueryProfile(params: any) { const history = useHistory(); const doQueryProfile = function(){ const param = { - path:location.pathname.slice(14), + path: getLastPath(), }; queryProfile(param).then(res=>{ if (res && res.msg === 'success') { @@ -59,6 +59,11 @@ export default function QueryProfile(params: any) { useEffect(() => { doQueryProfile(); }, [location.pathname]); + function getLastPath(){ + let arr = location.pathname.split('/'); + let str = arr.pop(); + return str === 'QueryProfile' ? '' : str; + } function goPrev(){ if (location.pathname === '/QueryProfile/') {return;} history.push('/QueryProfile/'); diff --git a/ui/src/pages/system/index.tsx b/ui/src/pages/system/index.tsx index 26720af8751f5b..35e0154012000a 100644 --- a/ui/src/pages/system/index.tsx +++ b/ui/src/pages/system/index.tsx @@ -80,6 +80,7 @@ export default function System(params: any) { isSort={true} isFilter={true} isInner={true} + path = 'System' isSystem = {true} allTableData={allTableData} /> diff --git a/ui/src/router/index.ts b/ui/src/router/index.ts index c94d28bfb83df6..1d54fde677b36b 100644 --- a/ui/src/router/index.ts +++ b/ui/src/router/index.ts @@ -27,16 +27,16 @@ const Logs = asyncComponent(() => import('../pages/logs')); const QueryProfile = asyncComponent(() => import('../pages/query-profile')); const Session = asyncComponent(() => import('../pages/session')); const Configuration = asyncComponent(() => import('../pages/configuration')); -const Ha = asyncComponent(() => import('../pages/ha')); -const Help = asyncComponent(() => import('../pages/help')); +// const Ha = asyncComponent(() => import('../pages/ha')); +// const Help = asyncComponent(() => import('../pages/help')); +const Page404 = asyncComponent(() => import('../pages/404')); const DataImport = asyncComponent(() => import('../pages/playground/data-import')); - export default { routes: [ { path: '/login', component: Login, - title: '登录', + title: 'Login', }, { path: '/', @@ -45,7 +45,7 @@ export default { { path: '/home', component: Home, - title: '首页', + title: 'Home', }, { path: '/Playground', @@ -93,6 +93,10 @@ export default { component: Configuration, title: 'Configuration', }, + { + path: '*', + component: Page404, + }, // { // path: '/ha', // component: Ha, @@ -105,10 +109,15 @@ export default { // }, ], }, + { + path: '*', + component: Page404, + }, { path: '/', redirect: '/home', component: Layout, }, + ], }; \ No newline at end of file diff --git a/ui/src/router/renderRouter.tsx b/ui/src/router/renderRouter.tsx index a3f8ad2e6e61cd..58a3c422789eda 100644 --- a/ui/src/router/renderRouter.tsx +++ b/ui/src/router/renderRouter.tsx @@ -19,9 +19,11 @@ import React from 'react'; import { Route, Redirect, Switch } from 'react-router-dom'; -import router from '.'; +import {getBasePath} from 'Src/utils/utils'; + let isLogin = document.cookie; const renderRoutes = (routes, authPath = '/login') => { + let basepath = getBasePath(); if(routes){ return ( @@ -32,8 +34,8 @@ const renderRoutes = (routes, authPath = '/login') => { exact={route.exact} strict={route.strict} render= { props =>{ - if(props.location.pathname === '/'){ - return ; + if(props.location.pathname === basepath+'/'){ + return ; } if (isLogin) { return route.render ? ( diff --git a/ui/src/utils/request.tsx b/ui/src/utils/request.tsx index de137e7ac88cb3..96c479708df64b 100644 --- a/ui/src/utils/request.tsx +++ b/ui/src/utils/request.tsx @@ -23,10 +23,10 @@ * @since 2020/08/19 */ import React from 'react'; -import {message, Modal} from 'antd'; +import {notification, Modal} from 'antd'; import {ExclamationCircleOutlined} from '@ant-design/icons'; import {Trans} from 'react-i18next'; -let basePath; +import {getBasePath} from 'Src/utils/utils'; function checkStatus(response) { if (response.status >= 200 && response.status < 300) { return response; @@ -37,7 +37,7 @@ function checkStatus(response) { icon: , content: loginExpirtMsg, onOk() { - window.location.href = window.location.origin + '/login'; + window.location.href = window.location.origin +getBasePath()+ '/login'; }, onCancel() { // @@ -47,7 +47,7 @@ function checkStatus(response) { } const error = new Error(response.statusText); error.response = response; - message.error(response.statusText); + notification.error(response.statusText); throw error; } @@ -65,10 +65,22 @@ function checkStatus(response) { * @return {Object} */ export default async function request(url, options = {}, tipSuccess = false, tipError = true, fullResponse = false) { - if(!basePath){ - const data = await fetch('/api/basepath') - basePath = await data.json(); + if(!localStorage.getItem('username') && url.includes('login') === false){ + clearAllCookie(); + Modal.confirm({ + title: tips, + icon: , + content: loginExpirtMsg, + onOk() { + window.location.href = window.location.origin +getBasePath()+ '/login'; + }, + onCancel() { + // + } + }); + return; } + const basePath = getBasePath(); const newOptions = {credentials: 'include', ...options}; if (newOptions.method === 'POST' || newOptions.method === 'PUT') { newOptions.headers = newOptions.isUpload @@ -83,8 +95,8 @@ export default async function request(url, options = {}, tipSuccess = false, tip if (typeof newOptions.body === 'object' && !newOptions.isUpload) { newOptions.body = JSON.stringify(newOptions.body); } - if (basePath.data?.path && basePath.data?.enable) { - url = basePath.data.path + url + if (basePath && basePath!=='/') { + url = basePath + url } const response = await fetch(url, newOptions); if ( @@ -104,14 +116,35 @@ export default async function request(url, options = {}, tipSuccess = false, tip } const data = await response.json(); if ('code' in data || 'msg' in data) { - const code = data.code; - const msg = data.msg; - if (msg === 'success' || code === 0 || code === 200) { + const {code, msg} = data; + if (code === 401 && data.data === 'Cookie is invalid') { + Modal.confirm({ + title: tips, + icon: , + content: loginExpirtMsg, + onOk() { + window.location.href = window.location.origin +getBasePath()+ '/login'; + }, + onCancel() { + // + } + }); + } else if (code === 401 && data.data !== 'Cookie is invalid') { + notification.error({ + message:loginWarning + }); + }else if (msg === 'success' || code === 0 || code === 200) { if (tipSuccess) { - message.success(successfulOperation, msg); + notification.success({ + message:successfulOperation, + description: msg + }); } - } else if (tipError && code !== 0 && msg !== 'success') { - message.error(msg); + } else if (tipError && code !== 0 && msg !== '') { + notification.error({ + message: msg, + description: data.data + }); } } @@ -120,3 +153,10 @@ export default async function request(url, options = {}, tipSuccess = false, tip } return data; } +function clearAllCookie() { + var keys = document.cookie.match(/[^ =;]+(?=\=)/g); + if(keys) { + for(var i = keys.length; i--;) + document.cookie = keys[i] + '=0;expires=' + new Date(0).toUTCString() + } +} \ No newline at end of file diff --git a/ui/src/utils/utils.ts b/ui/src/utils/utils.ts index 3355540c64fa11..584c288edc37a4 100644 --- a/ui/src/utils/utils.ts +++ b/ui/src/utils/utils.ts @@ -25,23 +25,23 @@ function isSuccess(response) { return false; } - let status = response.status; - if (status == null) { - status = response.msg; - } + let {code, msg} = response; - if (isNaN(status)) { - return status === 'success'; + if (code === 0 && msg === 'success') { + return true } - return status === 0; + return false; } function getDbName(params) { - const infoArr = location.pathname.split('-'); - const db_name = infoArr[0].split('/')[3]; - const tbl_name = infoArr[1]; + const infoArr = location.pathname.split('/'); + const str = infoArr[infoArr.length-1]; const res = {}; - res.db_name = db_name; - res.tbl_name = tbl_name; + if(str && str !=='Playground'){ + const db_name = str.split('-')[0]; + const tbl_name = str.split('-')[1]; + res.db_name = db_name; + res.tbl_name = tbl_name; + } return res; } function getTimeNow() { @@ -74,5 +74,13 @@ function getTimeNow() { } return fmt; } - -module.exports = {isSuccess, getDbName, getTimeNow}; \ No newline at end of file +function getBasePath(){ + let arr = location.pathname.split('/'); + let res = ''; + if(arr.length>5){ + arr = arr.slice(0,5); + res = arr.join('/'); + } + return res; +} +module.exports = {isSuccess, getDbName, getTimeNow, getBasePath}; \ No newline at end of file From 10766b07c5a1b5761c7659c628cafa0161babc3c Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 28 Sep 2020 19:03:42 +0800 Subject: [PATCH 08/12] bug fix --- fe/fe-core/src/main/java/org/apache/doris/load/Load.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java index 687d0290f0bd1a..546061691052a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java @@ -2020,7 +2020,7 @@ public LinkedList> getLoadJobInfosByDb(long dbId, String dbName jobInfo.add(TimeUtils.longToTimeString(loadJob.getLoadFinishTimeMs())); // tracking url jobInfo.add(status.getTrackingUrl()); - // job details + // job detail(not used for hadoop load, just return an empty string) jobInfo.add(""); loadJobInfos.add(jobInfo); From 1b9ea5f72ac4248525d637c071d4563e18126f89 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 9 Oct 2020 20:51:11 +0800 Subject: [PATCH 09/12] add new http client --- fe/fe-core/pom.xml | 5 ++ .../org/apache/doris/common/util/Util.java | 11 +++ .../doris/httpv2/rest/RestBaseController.java | 11 +-- .../doris/httpv2/util/LoadSubmitter.java | 86 +++++++++++++++++-- .../apache/doris/httpv2/util/TmpFileMgr.java | 11 +-- fe/pom.xml | 6 ++ 6 files changed, 115 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 8b91f86ef03810..ec91130705c88e 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -569,6 +569,11 @@ under the License. 5.5.0 + + org.apache.httpcomponents + httpclient + + diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java index 9e4ac29b5dc1c0..6dc3a4447aa5c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java @@ -22,6 +22,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.qe.ConnectContext; +import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -56,6 +57,8 @@ public class Util { private static final String[] ORDINAL_SUFFIX = new String[] { "th", "st", "nd", "rd", "th", "th", "th", "th", "th", "th" }; + private static final List REGEX_ESCAPES = Lists.newArrayList("\\", "$", "(", ")", "*", "+", ".", "[", "]", "?", "^", "{", "}", "|"); + static { TYPE_STRING_MAP.put(PrimitiveType.TINYINT, "tinyint(4)"); TYPE_STRING_MAP.put(PrimitiveType.SMALLINT, "smallint(6)"); @@ -482,5 +485,13 @@ public static InputStream getInputStreamFromUrl(String urlStr, String encodedAut public static boolean showHiddenColumns() { return ConnectContext.get() != null && ConnectContext.get().getSessionVariable().showHiddenColumns(); } + + public static String escapeSingleRegex(String s) { + Preconditions.checkArgument(s.length() == 1); + if (REGEX_ESCAPES.contains(s)) { + return "\\" + s; + } + return s; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java index 89fcc230bf39f8..f634c1f2ed94c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java @@ -26,15 +26,13 @@ import org.apache.doris.system.SystemInfoService; import org.apache.doris.thrift.TNetworkAddress; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.web.servlet.view.RedirectView; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; @@ -42,6 +40,9 @@ import java.io.OutputStream; import java.net.URI; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + public class RestBaseController extends BaseController { protected static final String NS_KEY = "ns"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/LoadSubmitter.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/LoadSubmitter.java index 9bdaaf56c78924..bec7a510a3ba0c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/LoadSubmitter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/LoadSubmitter.java @@ -22,18 +22,28 @@ import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.httpv2.rest.UploadAction; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import com.google.common.base.Strings; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.entity.FileEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.DefaultRedirectStrategy; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; +import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.lang.reflect.Type; @@ -67,6 +77,69 @@ public Worker(UploadAction.LoadContext loadContext) { @Override public SubmitResult call() throws Exception { + try { + return load(); + } catch (Throwable e) { + LOG.warn("failed to submit load. label: {}", loadContext.label, e); + throw e; + } + } + + private SubmitResult sendData(String content) throws Exception { + String loadUrlStr = String.format(STREAM_LOAD_URL_PATTERN, "127.0.0.1", Config.http_port, loadContext.db, loadContext.tbl); + + final HttpClientBuilder httpClientBuilder = HttpClients + .custom() + .setRedirectStrategy(new DefaultRedirectStrategy() { + @Override + protected boolean isRedirectable(String method) { + return true; + } + }); + + try (CloseableHttpClient client = httpClientBuilder.build()) { + HttpPut put = new HttpPut(loadUrlStr); + put.setHeader(HttpHeaders.EXPECT, "100-continue"); + put.setHeader(HttpHeaders.AUTHORIZATION, basicAuthHeader()); + if (!Strings.isNullOrEmpty(loadContext.columns)) { + put.setHeader("columns", loadContext.columns); + } + if (!Strings.isNullOrEmpty(loadContext.columnSeparator)) { + put.setHeader("column_separator", loadContext.columnSeparator); + } + if (!Strings.isNullOrEmpty(loadContext.label)) { + put.setHeader("label", loadContext.label); + } + + FileEntity entity = new FileEntity(checkAndGetFile(loadContext.file)); + put.setEntity(entity); + + try (CloseableHttpResponse response = client.execute(put)) { + String loadResult = ""; + if (response.getEntity() != null) { + loadResult = EntityUtils.toString(response.getEntity()); + } + final int statusCode = response.getStatusLine().getStatusCode(); + + if (statusCode != 200) { + throw new IOException( + String.format("Stream load failed, statusCode=%s load result=%s", statusCode, loadResult)); + } + + Type type = new TypeToken() { + }.getType(); + SubmitResult result = new Gson().fromJson(loadResult, type); + return result; + } + } + } + + private String basicAuthHeader() { + String auth = String.format("%s:%s", ClusterNamespace.getNameFromFullName(loadContext.user), loadContext.passwd); + return Base64.getEncoder().encodeToString(auth.getBytes(StandardCharsets.UTF_8)); + } + + private SubmitResult load() throws Exception { String auth = String.format("%s:%s", ClusterNamespace.getNameFromFullName(loadContext.user), loadContext.passwd); String authEncoding = Base64.getEncoder().encodeToString(auth.getBytes(StandardCharsets.UTF_8)); @@ -88,10 +161,12 @@ public SubmitResult call() throws Exception { } conn.setDoOutput(true); conn.setDoInput(true); + conn.setReadTimeout(0); + conn.setConnectTimeout(5000); File loadFile = checkAndGetFile(loadContext.file); - try(BufferedOutputStream bos = new BufferedOutputStream(conn.getOutputStream()); - BufferedInputStream bis = new BufferedInputStream(new FileInputStream(loadFile));) { + try (BufferedOutputStream bos = new BufferedOutputStream(conn.getOutputStream()); + BufferedInputStream bis = new BufferedInputStream(new FileInputStream(loadFile));) { int i; while ((i = bis.read()) > 0) { bos.write(i); @@ -126,6 +201,7 @@ public static class SubmitResult { public String TxnId; public String Label; public String Status; + public String ExistingJobStatus; public String Message; public String NumberTotalRows; public String NumberLoadedRows; diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/TmpFileMgr.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/TmpFileMgr.java index 0d1cc79915fefa..1b527ebfb5c01e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/TmpFileMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/TmpFileMgr.java @@ -19,14 +19,14 @@ import org.apache.doris.common.util.Util; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.springframework.web.multipart.MultipartFile; - import com.google.common.base.Joiner; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.web.multipart.MultipartFile; + import java.io.BufferedReader; import java.io.File; import java.io.FileReader; @@ -185,11 +185,12 @@ public void save(MultipartFile file) throws IOException { public void setPreview() throws IOException { lines = Lists.newArrayList(); + String escapedColSep = Util.escapeSingleRegex(columnSeparator); try (FileReader fr = new FileReader(absPath); BufferedReader bf = new BufferedReader(fr)) { String str; while ((str = bf.readLine()) != null) { - String[] cols = str.split(columnSeparator); + String[] cols = str.split(escapedColSep, -1); // -1 to keep the last empty column lines.add(Lists.newArrayList(cols)); if (cols.length > maxColNum) { maxColNum = cols.length; diff --git a/fe/pom.xml b/fe/pom.xml index c7b95371e6d129..fe9a5b364cc17e 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -653,6 +653,12 @@ under the License. 2.4.5 provided + + + org.apache.httpcomponents + httpclient + 4.5.3 + From 5be35362f19e0deee21d78685102d6bc80ed0003 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 9 Oct 2020 23:47:22 +0800 Subject: [PATCH 10/12] fix bug --- .../fe/statement-execution-action.md | 8 +- .../fe/statement-execution-action.md | 8 +- fe/fe-core/pom.xml | 5 - .../doris/httpv2/util/LoadSubmitter.java | 100 +++++------------- .../doris/httpv2/util/StatementSubmitter.java | 45 +++++--- fe/pom.xml | 6 -- 6 files changed, 66 insertions(+), 106 deletions(-) diff --git a/docs/en/administrator-guide/http-actions/fe/statement-execution-action.md b/docs/en/administrator-guide/http-actions/fe/statement-execution-action.md index 103a70577c9db2..828319fc330027 100644 --- a/docs/en/administrator-guide/http-actions/fe/statement-execution-action.md +++ b/docs/en/administrator-guide/http-actions/fe/statement-execution-action.md @@ -75,13 +75,14 @@ None "name": "k1", "type": "INT" }], - "status": {} + "status": {}, + "time": 10 }, "count": 0 } ``` - * The type field is `result_set`, which means the result set is returned. The results need to be obtained and displayed based on the meta and data fields. The meta field describes the column information returned. The data field returns the result row. The column type in each row needs to be judged by the content of the meta field. The status field returns some information of MySQL, such as the number of alarm rows, status code, etc. + * The type field is `result_set`, which means the result set is returned. The results need to be obtained and displayed based on the meta and data fields. The meta field describes the column information returned. The data field returns the result row. The column type in each row needs to be judged by the content of the meta field. The status field returns some information of MySQL, such as the number of alarm rows, status code, etc. The time field return the execution time, unit is millisecond. * Return execution result @@ -93,7 +94,8 @@ None "type": "exec_status", "status": {} }, - "count": 0 + "count": 0, + "time": 10 } ``` diff --git a/docs/zh-CN/administrator-guide/http-actions/fe/statement-execution-action.md b/docs/zh-CN/administrator-guide/http-actions/fe/statement-execution-action.md index 30df1db34cc5e4..805f239649fe3c 100644 --- a/docs/zh-CN/administrator-guide/http-actions/fe/statement-execution-action.md +++ b/docs/zh-CN/administrator-guide/http-actions/fe/statement-execution-action.md @@ -75,13 +75,14 @@ Statement Execution Action 用于执行语句并返回结果。 "name": "k1", "type": "INT" }], - "status": {} + "status": {}, + "time": 10 }, "count": 0 } ``` - * type 字段为 `result_set` 表示返回结果集。需要根据 meta 和 data 字段获取并展示结果。meta 字段描述返回的列信息。data 字段返回结果行。其中每一行的中的列类型，需要通过 meta 字段内容判断。status 字段返回 MySQL 的一些信息，如告警行数，状态码等。 + * type 字段为 `result_set` 表示返回结果集。需要根据 meta 和 data 字段获取并展示结果。meta 字段描述返回的列信息。data 字段返回结果行。其中每一行的中的列类型，需要通过 meta 字段内容判断。status 字段返回 MySQL 的一些信息，如告警行数，状态码等。time 字段返回语句执行时间，单位毫秒。 * 返回执行结果 @@ -91,7 +92,8 @@ Statement Execution Action 用于执行语句并返回结果。 "code": 0, "data": { "type": "exec_status", - "status": {} + "status": {}, + "time": 10 }, "count": 0 } diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index ec91130705c88e..8b91f86ef03810 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -569,11 +569,6 @@ under the License. 5.5.0 - - org.apache.httpcomponents - httpclient - - diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/LoadSubmitter.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/LoadSubmitter.java index bec7a510a3ba0c..1cac7e3d02272a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/LoadSubmitter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/LoadSubmitter.java @@ -17,33 +17,26 @@ package org.apache.doris.httpv2.util; +import org.apache.doris.catalog.Catalog; import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.httpv2.rest.UploadAction; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import com.google.common.base.Strings; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; -import org.apache.http.HttpHeaders; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.entity.FileEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.DefaultRedirectStrategy; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; -import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.lang.reflect.Type; @@ -51,6 +44,7 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.Base64; +import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; @@ -85,68 +79,16 @@ public SubmitResult call() throws Exception { } } - private SubmitResult sendData(String content) throws Exception { - String loadUrlStr = String.format(STREAM_LOAD_URL_PATTERN, "127.0.0.1", Config.http_port, loadContext.db, loadContext.tbl); - - final HttpClientBuilder httpClientBuilder = HttpClients - .custom() - .setRedirectStrategy(new DefaultRedirectStrategy() { - @Override - protected boolean isRedirectable(String method) { - return true; - } - }); - - try (CloseableHttpClient client = httpClientBuilder.build()) { - HttpPut put = new HttpPut(loadUrlStr); - put.setHeader(HttpHeaders.EXPECT, "100-continue"); - put.setHeader(HttpHeaders.AUTHORIZATION, basicAuthHeader()); - if (!Strings.isNullOrEmpty(loadContext.columns)) { - put.setHeader("columns", loadContext.columns); - } - if (!Strings.isNullOrEmpty(loadContext.columnSeparator)) { - put.setHeader("column_separator", loadContext.columnSeparator); - } - if (!Strings.isNullOrEmpty(loadContext.label)) { - put.setHeader("label", loadContext.label); - } - - FileEntity entity = new FileEntity(checkAndGetFile(loadContext.file)); - put.setEntity(entity); - - try (CloseableHttpResponse response = client.execute(put)) { - String loadResult = ""; - if (response.getEntity() != null) { - loadResult = EntityUtils.toString(response.getEntity()); - } - final int statusCode = response.getStatusLine().getStatusCode(); - - if (statusCode != 200) { - throw new IOException( - String.format("Stream load failed, statusCode=%s load result=%s", statusCode, loadResult)); - } - - Type type = new TypeToken() { - }.getType(); - SubmitResult result = new Gson().fromJson(loadResult, type); - return result; - } - } - } - - private String basicAuthHeader() { - String auth = String.format("%s:%s", ClusterNamespace.getNameFromFullName(loadContext.user), loadContext.passwd); - return Base64.getEncoder().encodeToString(auth.getBytes(StandardCharsets.UTF_8)); - } - private SubmitResult load() throws Exception { - String auth = String.format("%s:%s", ClusterNamespace.getNameFromFullName(loadContext.user), loadContext.passwd); - String authEncoding = Base64.getEncoder().encodeToString(auth.getBytes(StandardCharsets.UTF_8)); + // choose a backend to submit the stream load + Backend be = selectOneBackend(); - String loadUrlStr = String.format(STREAM_LOAD_URL_PATTERN, "127.0.0.1", Config.http_port, loadContext.db, loadContext.tbl); + String loadUrlStr = String.format(STREAM_LOAD_URL_PATTERN, be.getHost(), be.getHttpPort(), loadContext.db, loadContext.tbl); URL loadUrl = new URL(loadUrlStr); HttpURLConnection conn = (HttpURLConnection) loadUrl.openConnection(); conn.setRequestMethod("PUT"); + String auth = String.format("%s:%s", ClusterNamespace.getNameFromFullName(loadContext.user), loadContext.passwd); + String authEncoding = Base64.getEncoder().encodeToString(auth.getBytes(StandardCharsets.UTF_8)); conn.setRequestProperty("Authorization", "Basic " + authEncoding); conn.addRequestProperty("Expect", "100-continue"); conn.addRequestProperty("Content-Type", "text/plain; charset=UTF-8"); @@ -161,8 +103,6 @@ private SubmitResult load() throws Exception { } conn.setDoOutput(true); conn.setDoInput(true); - conn.setReadTimeout(0); - conn.setConnectTimeout(5000); File loadFile = checkAndGetFile(loadContext.file); try (BufferedOutputStream bos = new BufferedOutputStream(conn.getOutputStream()); @@ -195,6 +135,20 @@ private File checkAndGetFile(TmpFileMgr.TmpFile tmpFile) { File file = new File(tmpFile.absPath); return file; } + + private Backend selectOneBackend() throws DdlException { + List backendIds = Catalog.getCurrentSystemInfo().seqChooseBackendIds( + 1, true, false, SystemInfoService.DEFAULT_CLUSTER); + if (backendIds == null) { + throw new DdlException("No alive backend"); + } + + Backend backend = Catalog.getCurrentSystemInfo().getBackend(backendIds.get(0)); + if (backend == null) { + throw new DdlException("No alive backend"); + } + return backend; + } } public static class SubmitResult { diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java index 65f888656e31bd..af2707f2e7993e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java @@ -26,6 +26,7 @@ import org.apache.doris.analysis.SqlParser; import org.apache.doris.analysis.SqlScanner; import org.apache.doris.analysis.StatementBase; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.util.SqlParserUtils; @@ -97,17 +98,17 @@ public ExecutionResultSet call() throws Exception { try { Class.forName(JDBC_DRIVER); conn = DriverManager.getConnection(dbUrl, queryCtx.user, queryCtx.passwd); - + long startTime = System.currentTimeMillis(); if (stmtBase instanceof QueryStmt || stmtBase instanceof ShowStmt) { stmt = conn.prepareStatement(queryCtx.stmt, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); ResultSet rs = stmt.executeQuery(queryCtx.stmt); - ExecutionResultSet resultSet = generateResultSet(rs); + ExecutionResultSet resultSet = generateResultSet(rs, startTime); rs.close(); return resultSet; } else if (stmtBase instanceof InsertStmt || stmtBase instanceof DdlStmt || stmtBase instanceof ExportStmt) { stmt = conn.createStatement(); stmt.execute(queryCtx.stmt); - ExecutionResultSet resultSet = generateExecStatus(); + ExecutionResultSet resultSet = generateExecStatus(startTime); return resultSet; } else { throw new Exception("Unsupported statement type"); @@ -131,19 +132,20 @@ public ExecutionResultSet call() throws Exception { /** * Result json sample: * { - * "type": "result_set", - * "data": [ - * [1], - * [2] - * ], - * "meta": [{ - * "name": "k1", - * "type": "INT" + * "type": "result_set", + * "data": [ + * [1], + * [2] + * ], + * "meta": [{ + * "name": "k1", + * "type": "INT" * }], - * "status": {} + * "status": {}, + * "time" : 10 * } */ - private ExecutionResultSet generateResultSet(ResultSet rs) throws SQLException { + private ExecutionResultSet generateResultSet(ResultSet rs, long startTime) throws SQLException { Map result = Maps.newHashMap(); result.put("type", TYPE_RESULT_SET); if (rs == null) { @@ -174,20 +176,23 @@ private ExecutionResultSet generateResultSet(ResultSet rs) throws SQLException { } result.put("meta", metaFields); result.put("data", rows); + result.put("time", (System.currentTimeMillis() - startTime)); return new ExecutionResultSet(result); } /** * Result json sample: * { - * "type": "exec_status", - * "status": {} + * "type": "exec_status", + * "status": {}, + * "time" : 10 * } */ - private ExecutionResultSet generateExecStatus() throws SQLException { + private ExecutionResultSet generateExecStatus(long startTime) throws SQLException { Map result = Maps.newHashMap(); result.put("type", TYPE_EXEC_STATUS); result.put("status", Maps.newHashMap()); + result.put("time", (System.currentTimeMillis() - startTime)); return new ExecutionResultSet(result); } @@ -195,6 +200,13 @@ private StatementBase analyzeStmt(String stmtStr) throws Exception { SqlParser parser = new SqlParser(new SqlScanner(new StringReader(stmtStr))); try { return SqlParserUtils.getFirstStmt(parser); + } catch (AnalysisException e) { + String errorMessage = parser.getErrorMsg(stmtStr); + if (errorMessage == null) { + throw e; + } else { + throw new AnalysisException(errorMessage, e); + } } catch (Exception e) { throw new Exception("error happens when parsing stmt: " + e.getMessage()); } @@ -215,3 +227,4 @@ public StmtContext(String stmt, String user, String passwd, long limit) { } } } + diff --git a/fe/pom.xml b/fe/pom.xml index fe9a5b364cc17e..c7b95371e6d129 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -653,12 +653,6 @@ under the License. 2.4.5 provided - - - org.apache.httpcomponents - httpclient - 4.5.3 - From 246b69696ba9d52f032ffd39db3ae4fb5b41a8f8 Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 10 Oct 2020 09:37:21 +0800 Subject: [PATCH 11/12] fix bug --- .../fe/statement-execution-action.md | 44 +++++++++---------- .../fe/statement-execution-action.md | 44 +++++++++---------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/docs/en/administrator-guide/http-actions/fe/statement-execution-action.md b/docs/en/administrator-guide/http-actions/fe/statement-execution-action.md index 828319fc330027..71770634eed58c 100644 --- a/docs/en/administrator-guide/http-actions/fe/statement-execution-action.md +++ b/docs/en/administrator-guide/http-actions/fe/statement-execution-action.md @@ -63,22 +63,22 @@ None ``` { - "msg": "success", - "code": 0, - "data": { - "type": "result_set", - "data": [ - [1], - [2] - ], - "meta": [{ - "name": "k1", - "type": "INT" - }], - "status": {}, + "msg": "success", + "code": 0, + "data": { + "type": "result_set", + "data": [ + [1], + [2] + ], + "meta": [{ + "name": "k1", + "type": "INT" + }], + "status": {}, "time": 10 - }, - "count": 0 + }, + "count": 0 } ``` @@ -88,13 +88,13 @@ None ``` { - "msg": "success", - "code": 0, - "data": { - "type": "exec_status", - "status": {} - }, - "count": 0, + "msg": "success", + "code": 0, + "data": { + "type": "exec_status", + "status": {} + }, + "count": 0, "time": 10 } ``` diff --git a/docs/zh-CN/administrator-guide/http-actions/fe/statement-execution-action.md b/docs/zh-CN/administrator-guide/http-actions/fe/statement-execution-action.md index 805f239649fe3c..a11a960bc8cf25 100644 --- a/docs/zh-CN/administrator-guide/http-actions/fe/statement-execution-action.md +++ b/docs/zh-CN/administrator-guide/http-actions/fe/statement-execution-action.md @@ -63,22 +63,22 @@ Statement Execution Action 用于执行语句并返回结果。 ``` { - "msg": "success", - "code": 0, - "data": { - "type": "result_set", - "data": [ - [1], - [2] - ], - "meta": [{ - "name": "k1", - "type": "INT" - }], - "status": {}, + "msg": "success", + "code": 0, + "data": { + "type": "result_set", + "data": [ + [1], + [2] + ], + "meta": [{ + "name": "k1", + "type": "INT" + }], + "status": {}, "time": 10 - }, - "count": 0 + }, + "count": 0 } ``` @@ -88,14 +88,14 @@ Statement Execution Action 用于执行语句并返回结果。 ``` { - "msg": "success", - "code": 0, - "data": { - "type": "exec_status", - "status": {}, + "msg": "success", + "code": 0, + "data": { + "type": "exec_status", + "status": {}, "time": 10 - }, - "count": 0 + }, + "count": 0 } ``` From 764d0bc32d8d779faf957cc9b373f9f056016a98 Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 10 Oct 2020 17:09:35 +0800 Subject: [PATCH 12/12] fix prepare state bug --- build.sh | 14 ++++++++++---- .../doris/httpv2/util/StatementSubmitter.java | 5 ++++- ui/public/locales/en-us.json | 3 ++- ui/public/locales/zh-cn.json | 3 ++- ui/src/i18n.tsx | 2 +- ui/src/pages/layout/index.tsx | 12 +++++++++++- .../pages/playground/content/content-result.tsx | 16 ++++++++-------- ui/src/pages/playground/content/index.tsx | 8 ++++---- ui/src/utils/request.tsx | 9 ++++++++- 9 files changed, 50 insertions(+), 22 deletions(-) diff --git a/build.sh b/build.sh index 89b61079a1042a..a0f6c034ae0041 100755 --- a/build.sh +++ b/build.sh @@ -211,12 +211,18 @@ function build_ui() { fi echo "Build Frontend UI" - cd ${DORIS_HOME}/ui - ${NPM} install - ${NPM} run build + ui_dist=${DORIS_HOME}/ui/dist/ + if [[ ! -z ${CUSTOM_UI_DIST} ]]; then + ui_dist=${CUSTOM_UI_DIST} + else + cd ${DORIS_HOME}/ui + ${NPM} install + ${NPM} run build + fi + echo "ui dist: ${ui_dist}" rm -rf ${DORIS_HOME}/fe/fe-core/src/main/resources/static/ mkdir -p ${DORIS_HOME}/fe/fe-core/src/main/resources/static - cp -r ${DORIS_HOME}/ui/dist/* ${DORIS_HOME}/fe/fe-core/src/main/resources/static + cp -r ${ui_dist}/* ${DORIS_HOME}/fe/fe-core/src/main/resources/static } # FE UI must be built before building FE diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java index af2707f2e7993e..ae51fb8548e551 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java @@ -41,6 +41,7 @@ import java.io.StringReader; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; @@ -101,7 +102,9 @@ public ExecutionResultSet call() throws Exception { long startTime = System.currentTimeMillis(); if (stmtBase instanceof QueryStmt || stmtBase instanceof ShowStmt) { stmt = conn.prepareStatement(queryCtx.stmt, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - ResultSet rs = stmt.executeQuery(queryCtx.stmt); + // set fetch size to MIN_VALUE to enable streaming result set to avoid OOM. + ((PreparedStatement) stmt).setFetchSize(Integer.MIN_VALUE); + ResultSet rs = ((PreparedStatement) stmt).executeQuery(); ExecutionResultSet resultSet = generateResultSet(rs, startTime); rs.close(); return resultSet; diff --git a/ui/public/locales/en-us.json b/ui/public/locales/en-us.json index 6894f33e5d2b35..ba4200138f5460 100644 --- a/ui/public/locales/en-us.json +++ b/ui/public/locales/en-us.json @@ -46,5 +46,6 @@ "successfulOperation":"The operation was successful", "tips":"Tips", "fileSizeWarning": "File size cannot exceed 100M", - "selectWarning": "Please select a table" + "selectWarning": "Please select a table", + "executionTime": "Execution Time" } diff --git a/ui/public/locales/zh-cn.json b/ui/public/locales/zh-cn.json index 080c7be82de5ce..24a7335735b180 100644 --- a/ui/public/locales/zh-cn.json +++ b/ui/public/locales/zh-cn.json @@ -46,5 +46,6 @@ "successfulOperation": "操作成功", "tips": "提示", "fileSizeWarning": "文件大小不能超过100m", - "selectWarning": "请选择表" + "selectWarning": "请选择表", + "executionTime": "执行时间" } diff --git a/ui/src/i18n.tsx b/ui/src/i18n.tsx index 006938b69914a1..331c465a0bcfb6 100644 --- a/ui/src/i18n.tsx +++ b/ui/src/i18n.tsx @@ -24,10 +24,10 @@ import zhCnTrans from '../public/locales/zh-cn.json'; import { initReactI18next } from 'react-i18next'; - i18n.use(LanguageDetector) .use(initReactI18next) .init({ + lng: localStorage.getItem('I18N_LANGUAGE') || "en", resources: { en: { translation: enUsTrans diff --git a/ui/src/pages/layout/index.tsx b/ui/src/pages/layout/index.tsx index d87e3e383ff389..8c14ac4b1aa1a1 100644 --- a/ui/src/pages/layout/index.tsx +++ b/ui/src/pages/layout/index.tsx @@ -23,7 +23,7 @@ * @since 2020/08/19 */ import React, {useState} from 'react'; -import {Layout, Menu, Dropdown, notification} from 'antd'; +import {Layout, Menu, Dropdown, notification, Button} from 'antd'; import { CaretDownOutlined, LogoutOutlined} from '@ant-design/icons'; import {renderRoutes} from 'react-router-config'; import {useHistory} from 'react-router-dom'; @@ -72,6 +72,15 @@ function Layouts(props: any) { history.push('/login'); }) } + function changeLanguage(){ + if (localStorage.getItem('I18N_LANGUAGE') === 'zh-CN'){ + localStorage.setItem('I18N_LANGUAGE','en'); + location.reload() + } else { + localStorage.setItem('I18N_LANGUAGE','zh-CN'); + location.reload() + } + } const menu = ( @@ -85,6 +94,7 @@ function Layouts(props: any) {
{history.replace('/home');setCurrent('')}}>
+ {/* */} diff --git a/ui/src/pages/playground/content/content-result.tsx b/ui/src/pages/playground/content/content-result.tsx index 703f73097ad421..627fa99370332f 100644 --- a/ui/src/pages/playground/content/content-result.tsx +++ b/ui/src/pages/playground/content/content-result.tsx @@ -62,7 +62,7 @@ export function AdhocContentResult(props) { } else { const tableData = (runningQueryInfo.data && typeof(runningQueryInfo.data) === 'object')?(runningQueryInfo.data?.data).slice(0,20):[]; setTableDate(tableData); - setTotal((runningQueryInfo.data && typeof(runningQueryInfo.data) === 'object')?runningQueryInfo.data.length:0); + setTotal((runningQueryInfo.data && typeof(runningQueryInfo.data) === 'object')?runningQueryInfo.data?.data.length:0); } setRunningQueryInfo(runningQueryInfo); },[location.state]); @@ -140,13 +140,13 @@ export function AdhocContentResult(props) { {runningQueryInfo.tbl_name} */} - {t('startingTime')}: - {runningQueryInfo.beginTime} + {t('executionTime')}: + {runningQueryInfo.data?.time + ' ms'} - + {/* {t('endTime')}: {runningQueryInfo.beginTime} - + */} { ...getELe(resStatus) } @@ -166,8 +166,8 @@ export function AdhocContentResult(props) { - {runningQueryInfo.data?.meta?.map(item => ( - ))} @@ -179,7 +179,7 @@ export function AdhocContentResult(props) { {item.map((tdData, index) => ( diff --git a/ui/src/pages/playground/content/index.tsx b/ui/src/pages/playground/content/index.tsx index c4c5c1d90ec340..bc0f035c9eec12 100644 --- a/ui/src/pages/playground/content/index.tsx +++ b/ui/src/pages/playground/content/index.tsx @@ -54,7 +54,7 @@ export function AdHocContent(props: any) { }); const [code, setCode] = useState(''); const editorAreaHeight = +(localStorage.getItem('editorAreaHeight') || 300); - const beginTime = getTimeNow(); + // const beginTime = getTimeNow(); const runQuery = useRequest>( () => AdHocAPI.doQuery({ @@ -64,16 +64,16 @@ export function AdHocContent(props: any) { { manual: true, onSuccess: res => { - const endTime = getTimeNow(); + // const endTime = getTimeNow(); const {db_name, tbl_name} = getDbName(); if (isSuccess(res)) { res.sqlCode = code; - res = {...res, db_name, tbl_name, beginTime, endTime} + res = {...res, db_name, tbl_name} props.history.push({pathname:`/Playground/result/${db_name}-${tbl_name}`,state: res}); runSQLSuccessSubject.next(true); } else { res.sqlCode = code; - res = {...res, db_name, tbl_name, beginTime, endTime} + res = {...res, db_name, tbl_name} props.history.push({pathname:`/Playground/result/${db_name}-${tbl_name}`,state: res}); runSQLSuccessSubject.next(false); } diff --git a/ui/src/utils/request.tsx b/ui/src/utils/request.tsx index 96c479708df64b..1620828b625069 100644 --- a/ui/src/utils/request.tsx +++ b/ui/src/utils/request.tsx @@ -27,6 +27,8 @@ import {notification, Modal} from 'antd'; import {ExclamationCircleOutlined} from '@ant-design/icons'; import {Trans} from 'react-i18next'; import {getBasePath} from 'Src/utils/utils'; +import SyntaxHighlighter from 'react-syntax-highlighter'; +import {docco} from 'react-syntax-highlighter/dist/esm/styles/hljs'; function checkStatus(response) { if (response.status >= 200 && response.status < 300) { return response; @@ -141,9 +143,14 @@ export default async function request(url, options = {}, tipSuccess = false, tip }); } } else if (tipError && code !== 0 && msg !== '') { + let item = ( + + {data.data} + + ) notification.error({ message: msg, - description: data.data + description: item }); } }
+ {runningQueryInfo.data?.meta?.map((item, index) => ( + {item.name} {tdData == '\\N'?'-':tdData}