From eadac7a75d3c74f92d4bb8cd666fc354843968e0 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 14 Apr 2022 16:22:54 +0800 Subject: [PATCH 1/3] [fix](ut)(compile) Fix BE compile bug and FE unit test --- .../main/java/org/apache/doris/PaloFe.java | 2 +- .../org/apache/doris/common/Log4jConfig.java | 4 +- .../org/apache/doris/httpv2/HttpServer.java | 10 ++++- .../doris/journal/bdbje/BDBDebugger.java | 2 +- .../apache/doris/http/DorisHttpTestCase.java | 38 +++++++++++++++---- .../doris/http/TableQueryPlanActionTest.java | 29 +++++++------- .../doris/http/TableRowCountActionTest.java | 8 ++-- .../doris/http/TableSchemaActionTest.java | 4 +- .../ExtractCommonFactorsRuleFunctionTest.java | 26 +++++-------- 9 files changed, 73 insertions(+), 50 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java b/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java index 13e6f5b22ea904..5a279080dc38ca 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java +++ b/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java @@ -142,7 +142,7 @@ public static void start(String dorisHomeDir, String pidDir, String[] args) { httpServer.setWorkers(Config.jetty_server_workers); httpServer.setMaxThreads(Config.jetty_threadPool_maxThreads); httpServer.setMaxThreads(Config.jetty_threadPool_minThreads); - httpServer.start(dorisHomeDir); + httpServer.start(); qeService.start(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java index 6f2dbf5f81ac58..470636cb5d3f54 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java @@ -19,6 +19,8 @@ import org.apache.doris.httpv2.config.SpringLog4j2Config; +import com.google.common.collect.Maps; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.config.ConfigurationSource; @@ -26,8 +28,6 @@ import org.apache.logging.log4j.core.lookup.Interpolator; import org.apache.logging.log4j.core.lookup.StrSubstitutor; -import com.google.common.collect.Maps; - import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.Map; diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java index 97cd932049086f..0d43e2f6795992 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java @@ -19,6 +19,7 @@ import org.apache.doris.PaloFe; import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; import org.apache.doris.httpv2.config.SpringLog4j2Config; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -77,7 +78,7 @@ protected SpringApplicationBuilder configure(SpringApplicationBuilder applicatio return application.sources(HttpServer.class); } - public void start(String dorisHome) { + public void start() { Map properties = new HashMap<>(); properties.put("server.port", port); properties.put("server.servlet.context-path", "/"); @@ -101,7 +102,12 @@ public void start(String dorisHome) { System.setProperty("spring.http.multipart.location", PaloFe.DORIS_HOME_DIR); } System.setProperty("spring.banner.image.location", "doris-logo.png"); - properties.put("logging.config", Config.custom_config_dir + "/" + SpringLog4j2Config.SPRING_LOG_XML_FILE); + if (FeConstants.runningUnitTest) { + // this is currently only used for unit test + properties.put("logging.config", getClass().getClassLoader().getResource("log4j2.xml").getPath()); + } else { + properties.put("logging.config", Config.custom_config_dir + "/" + SpringLog4j2Config.SPRING_LOG_XML_FILE); + } new SpringApplicationBuilder() .sources(HttpServer.class) .properties(properties) diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java index 35533433d4d615..2286c65417c62b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java @@ -89,7 +89,7 @@ private void startService(String dorisHomeDir) throws IllegalArgException, IOExc HttpServer httpServer = new HttpServer(); httpServer.setPort(Config.http_port); - httpServer.start(dorisHomeDir); + httpServer.start(); // MySQl server QeService qeService = new QeService(Config.query_port, Config.mysql_service_nio_enabled, ExecuteEnv.getInstance().getScheduler()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java b/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java index 71f4adaf247277..be5596250d5a35 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java +++ b/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java @@ -38,9 +38,9 @@ import org.apache.doris.catalog.TabletInvertedIndex; import org.apache.doris.catalog.TabletMeta; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.ExceptionChecker.ThrowingRunnable; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.httpv2.HttpServer; import org.apache.doris.httpv2.IllegalArgException; @@ -53,6 +53,7 @@ import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TStorageType; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import org.junit.After; @@ -60,7 +61,10 @@ import org.junit.Before; import org.junit.BeforeClass; +import java.io.File; +import java.io.IOException; import java.net.ServerSocket; +import java.nio.file.Files; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -113,7 +117,19 @@ abstract public class DorisHttpTestCase { protected String rootAuth = Credentials.basic("root", ""); - public static final String DORIS_HOME_DIR = System.getenv("DORIS_HOME"); + private static final String DORIS_HOME; + + static { + String dorisHome = System.getenv("DORIS_HOME"); + if (Strings.isNullOrEmpty(dorisHome)) { + try { + dorisHome = Files.createTempDirectory("DORIS_HOME").toAbsolutePath().toString(); + } catch (IOException e) { + e.printStackTrace(); + } + } + DORIS_HOME = dorisHome; + } @Mocked private static EditLog editLog; @@ -296,15 +312,21 @@ public static void initHttpServer() throws IllegalArgException, InterruptedExcep } } + FeConstants.runningUnitTest = true; httpServer = new HttpServer(); - httpServer.setPort(Config.http_port); - httpServer.setMaxHttpPostSize(Config.jetty_server_max_http_post_size); - httpServer.setAcceptors(Config.jetty_server_acceptors); - httpServer.setSelectors(Config.jetty_server_selectors); - httpServer.setWorkers(Config.jetty_server_workers); - httpServer.start(DORIS_HOME_DIR); + httpServer.setPort(HTTP_PORT); + httpServer.setMaxHttpPostSize(100 * 1024 * 1024); + httpServer.setAcceptors(2); + httpServer.setSelectors(4); + httpServer.setWorkers(0); + httpServer.start(); } + @AfterClass + public static void afterClass() { + File file = new File(DORIS_HOME); + file.delete(); + } @Before diff --git a/fe/fe-core/src/test/java/org/apache/doris/http/TableQueryPlanActionTest.java b/fe/fe-core/src/test/java/org/apache/doris/http/TableQueryPlanActionTest.java index 8915411dcd927f..395a7d295796bd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/http/TableQueryPlanActionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/http/TableQueryPlanActionTest.java @@ -58,9 +58,9 @@ public void testQueryPlanAction() throws IOException, TException { String respStr = response.body().string(); JSONObject jsonObject = (JSONObject) JSONValue.parse(respStr); System.out.println(respStr); - Assert.assertEquals(200, (long) jsonObject.get("status")); + Assert.assertEquals(200, (long) ((JSONObject) jsonObject.get("data")).get("status")); - JSONObject partitionsObject = (JSONObject) jsonObject.get("partitions"); + JSONObject partitionsObject = (JSONObject) ((JSONObject) jsonObject.get("data")).get("partitions"); Assert.assertNotNull(partitionsObject); for (Object tabletKey : partitionsObject.keySet()) { JSONObject tabletObject = (JSONObject) partitionsObject.get(tabletKey); @@ -70,7 +70,7 @@ public void testQueryPlanAction() throws IOException, TException { Assert.assertEquals(testSchemaHash, (long) tabletObject.get("schemaHash")); } - String queryPlan = (String) jsonObject.get("opaqued_query_plan"); + String queryPlan = (String) ((JSONObject) jsonObject.get("data")).get("opaqued_query_plan"); Assert.assertNotNull(queryPlan); byte[] binaryPlanInfo = Base64.getDecoder().decode(queryPlan); TDeserializer deserializer = new TDeserializer(); @@ -93,8 +93,8 @@ public void testNoSqlFailure() throws IOException { System.out.println(respStr); Assert.assertNotNull(respStr); JSONObject jsonObject = (JSONObject) JSONValue.parse(respStr); - Assert.assertEquals(400, (long) jsonObject.get("status")); - String exception = (String) jsonObject.get("exception"); + Assert.assertEquals(403, (long) jsonObject.get("code")); + String exception = (String) jsonObject.get("data"); Assert.assertNotNull(exception); Assert.assertEquals("POST body must contains [sql] root object", exception); } @@ -112,8 +112,8 @@ public void testEmptySqlFailure() throws IOException { System.out.println(respStr); Assert.assertNotNull(respStr); JSONObject jsonObject = (JSONObject) JSONValue.parse(respStr); - Assert.assertEquals(400, (long) jsonObject.get("status")); - String exception = (String) jsonObject.get("exception"); + Assert.assertEquals(403, (long) jsonObject.get("code")); + String exception = (String) jsonObject.get("data"); Assert.assertNotNull(exception); Assert.assertEquals("POST body must contains [sql] root object", exception); } @@ -131,8 +131,8 @@ public void testInconsistentResource() throws IOException { System.out.println(respStr); Assert.assertNotNull(respStr); JSONObject jsonObject = (JSONObject) JSONValue.parse(respStr); - Assert.assertEquals(400, (long) jsonObject.get("status")); - String exception = (String) jsonObject.get("exception"); + Assert.assertEquals(400, (long) ((JSONObject) jsonObject.get("data")).get("status")); + String exception = (String) ((JSONObject) jsonObject.get("data")).get("exception"); Assert.assertNotNull(exception); Assert.assertTrue(exception.startsWith("requested database and table must consistent with sql")); } @@ -149,15 +149,15 @@ public void testMalformedJson() throws IOException { String respStr = response.body().string(); Assert.assertNotNull(respStr); JSONObject jsonObject = (JSONObject) JSONValue.parse(respStr); - Assert.assertEquals(400, (long) jsonObject.get("status")); - String exception = (String) jsonObject.get("exception"); + Assert.assertEquals(403, (long) jsonObject.get("code")); + String exception = (String) jsonObject.get("data"); Assert.assertNotNull(exception); Assert.assertTrue(exception.startsWith("malformed json")); } @Test public void testNotOlapTableFailure() throws IOException { - RequestBody body = RequestBody.create(JSON, "{ \"sql\" : \" select k1,k2 from " + DB_NAME + ".es_table" + " \" }"); + RequestBody body = RequestBody.create("{ \"sql\" : \" select k1,k2 from " + DB_NAME + ".es_table" + " \" }", JSON); Request request = new Request.Builder() .post(body) .addHeader("Authorization", rootAuth) @@ -167,8 +167,9 @@ public void testNotOlapTableFailure() throws IOException { String respStr = response.body().string(); Assert.assertNotNull(respStr); JSONObject jsonObject = (JSONObject) JSONValue.parse(respStr); - Assert.assertEquals(403, (long) jsonObject.get("status")); - String exception = (String) jsonObject.get("exception"); + System.out.println(respStr); + Assert.assertEquals(1, (long) jsonObject.get("code")); + String exception = (String) jsonObject.get("data"); Assert.assertTrue(exception.contains("table type is not OLAP")); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/http/TableRowCountActionTest.java b/fe/fe-core/src/test/java/org/apache/doris/http/TableRowCountActionTest.java index d697c5348dc46e..2f8250747f550b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/http/TableRowCountActionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/http/TableRowCountActionTest.java @@ -40,8 +40,10 @@ public void testTableCount() throws IOException { .build(); Response response = networkClient.newCall(request).execute(); - JSONObject jsonObject = (JSONObject) JSONValue.parse(response.body().string()); - Assert.assertEquals(200, (long) jsonObject.get("status")); - Assert.assertEquals(2000, (long) jsonObject.get("size")); + String res = response.body().string(); + System.out.println(res); + JSONObject jsonObject = (JSONObject) JSONValue.parse(res); + Assert.assertEquals(200, (long) ((JSONObject) jsonObject.get("data")).get("status")); + Assert.assertEquals(2000, (long) ((JSONObject) jsonObject.get("data")).get("size")); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/http/TableSchemaActionTest.java b/fe/fe-core/src/test/java/org/apache/doris/http/TableSchemaActionTest.java index cb96a10328b5a1..e3f6346164bf5f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/http/TableSchemaActionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/http/TableSchemaActionTest.java @@ -45,8 +45,8 @@ public void testGetTableSchema() throws IOException { String respStr = response.body().string(); Assert.assertNotNull(respStr); JSONObject object = (JSONObject) JSONValue.parse(respStr); - Assert.assertEquals(200, (long) object.get("status")); - JSONArray propArray = (JSONArray) object.get("properties"); + Assert.assertEquals(200, (long) ((JSONObject) object.get("data")).get("status")); + JSONArray propArray = (JSONArray) ((JSONObject) object.get("data")).get("properties"); // k1, k2 Assert.assertEquals(2, propArray.size()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/rewrite/ExtractCommonFactorsRuleFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/rewrite/ExtractCommonFactorsRuleFunctionTest.java index a25e035db1c460..d462de7fc64059 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/rewrite/ExtractCommonFactorsRuleFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/rewrite/ExtractCommonFactorsRuleFunctionTest.java @@ -18,7 +18,6 @@ package org.apache.doris.rewrite; -import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; import org.apache.doris.utframe.DorisAssert; import org.apache.doris.utframe.UtFrameUtils; @@ -256,62 +255,55 @@ public void testRewriteLikePredicate() throws Exception { // tinyint String sql = "select * from tb3 where k1 like '%4%';"; LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery()); - dorisAssert.query(sql).explainContains("CAST(`k1` AS CHARACTER) LIKE '%4%'"); + dorisAssert.query(sql).explainContains("`k1` LIKE '%4%'"); // smallint sql = "select * from tb3 where k2 like '%4%';"; LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery()); - dorisAssert.query(sql).explainContains("CAST(`k2` AS CHARACTER) LIKE '%4%'"); + dorisAssert.query(sql).explainContains("`k2` LIKE '%4%'"); // int sql = "select * from tb3 where k3 like '%4%';"; LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery()); - dorisAssert.query(sql).explainContains("CAST(`k3` AS CHARACTER) LIKE '%4%'"); + dorisAssert.query(sql).explainContains("`k3` LIKE '%4%'"); // bigint sql = "select * from tb3 where k4 like '%4%';"; LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery()); - dorisAssert.query(sql).explainContains("CAST(`k4` AS CHARACTER) LIKE '%4%'"); + dorisAssert.query(sql).explainContains("`k4` LIKE '%4%'"); // largeint sql = "select * from tb3 where k5 like '%4%';"; LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery()); - dorisAssert.query(sql).explainContains("CAST(`k5` AS CHARACTER) LIKE '%4%'"); + dorisAssert.query(sql).explainContains("`k5` LIKE '%4%'"); } - @Test(expected = AnalysisException.class) + @Test public void testRewriteLikePredicateDate() throws Exception { // date String sql = "select * from tb3 where k6 like '%4%';"; LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery()); - dorisAssert.query(sql).explainQuery(); - Assert.fail("No exception throws."); } - @Test(expected = AnalysisException.class) + @Test public void testRewriteLikePredicateDateTime() throws Exception { // datetime String sql = "select * from tb3 where k7 like '%4%';"; LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery()); dorisAssert.query(sql).explainQuery(); - Assert.fail("No exception throws."); } - @Test(expected = AnalysisException.class) + @Test public void testRewriteLikePredicateFloat() throws Exception { // date String sql = "select * from tb3 where k8 like '%4%';"; LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery()); - dorisAssert.query(sql).explainQuery(); - Assert.fail("No exception throws."); } - @Test(expected = AnalysisException.class) + @Test public void testRewriteLikePredicateDouble() throws Exception { // date String sql = "select * from tb3 where k9 like '%4%';"; LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery()); - dorisAssert.query(sql).explainQuery(); - Assert.fail("No exception throws."); } } From 52592e7c8bd721b4b7ae58eeda3613b268f1c5cf Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 14 Apr 2022 16:28:51 +0800 Subject: [PATCH 2/3] fix ut --- be/src/olap/rowset/column_reader.cpp | 4 ++-- be/src/olap/rowset/run_length_integer_reader.cpp | 4 ++-- be/src/tools/meta_tool.cpp | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/be/src/olap/rowset/column_reader.cpp b/be/src/olap/rowset/column_reader.cpp index 42f301f521dca7..327d6152874f25 100644 --- a/be/src/olap/rowset/column_reader.cpp +++ b/be/src/olap/rowset/column_reader.cpp @@ -190,7 +190,7 @@ Status StringColumnDirectReader::next_vector(ColumnVector* column_vector, uint32 string_buffer_size += length; } - uint8_t* allocated_mem; + uint8_t* allocated_mem = nullptr; res = mem_pool->allocate_safely(string_buffer_size, allocated_mem); if (!res.ok()) { return res; @@ -229,7 +229,7 @@ Status StringColumnDirectReader::next_vector(ColumnVector* column_vector, uint32 } } - uint8_t* allocated_mem; + uint8_t* allocated_mem = nullptr; res = mem_pool->allocate_safely(string_buffer_size, allocated_mem); if (!res.ok()) { return res; diff --git a/be/src/olap/rowset/run_length_integer_reader.cpp b/be/src/olap/rowset/run_length_integer_reader.cpp index 47cd9d755fe47e..4fcd6019d425cb 100644 --- a/be/src/olap/rowset/run_length_integer_reader.cpp +++ b/be/src/olap/rowset/run_length_integer_reader.cpp @@ -30,7 +30,7 @@ Status RunLengthIntegerReader::_read_values() { Status res = Status::OK(); // read the first 2 bits and determine the encoding type - uint8_t first_byte; + uint8_t first_byte = 0; res = _input->read((char*)&first_byte); if (!res.ok()) { @@ -311,7 +311,7 @@ Status RunLengthIntegerReader::_read_direct_values(uint8_t first_byte) { // extract the run length int32_t len = (first_byte & 0x01) << 8; - uint8_t byte; + uint8_t byte = 0; res = _input->read((char*)&byte); if (!res.ok()) { diff --git a/be/src/tools/meta_tool.cpp b/be/src/tools/meta_tool.cpp index 41923b277d48f6..52ee67b6629187 100644 --- a/be/src/tools/meta_tool.cpp +++ b/be/src/tools/meta_tool.cpp @@ -114,7 +114,7 @@ void get_meta(DataDir* data_dir) { std::string value; Status s = TabletMetaManager::get_json_meta(data_dir, FLAGS_tablet_id, FLAGS_schema_hash, &value); - if (s == doris::Status::OLAPInternalError(OLAP_ERR_META_KEY_NOT_FOUND)) { + if (s == doris::Status::OLAPInternalError(doris::OLAP_ERR_META_KEY_NOT_FOUND)) { std::cout << "no tablet meta for tablet_id:" << FLAGS_tablet_id << ", schema_hash:" << FLAGS_schema_hash << std::endl; return; From 6b0791009a072c8a500503247b20ec1b7b5635da Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 14 Apr 2022 16:30:43 +0800 Subject: [PATCH 3/3] remove error --- be/src/olap/rowset/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/olap/rowset/CMakeLists.txt b/be/src/olap/rowset/CMakeLists.txt index 208ff212eaed52..2f4720e5ac9a85 100644 --- a/be/src/olap/rowset/CMakeLists.txt +++ b/be/src/olap/rowset/CMakeLists.txt @@ -47,4 +47,4 @@ add_library(Rowset STATIC beta_rowset_reader.cpp beta_rowset_writer.cpp) -target_compile_options(Rowset PUBLIC "-Wno-error=maybe-uninitialized") +target_compile_options(Rowset PUBLIC)