From 6db91ca4636ce3b9786003596d4ef13e370e3829 Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 30 Apr 2022 22:55:05 +0800 Subject: [PATCH 1/2] [fix](truncate) fix bug that truncate partition throw NPE partition name is case insensitive --- .gitignore | 1 - .../org/apache/doris/catalog/Catalog.java | 2 +- .../org/apache/doris/catalog/OlapTable.java | 6 ++- .../apache/doris/common/CaseSensibility.java | 2 +- .../doris/catalog/TruncateTableTest.java | 41 ++++++++++++++++++ .../src/test/resources/help-resource.zip | Bin 0 -> 1056 bytes 6 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 fe/fe-core/src/test/resources/help-resource.zip diff --git a/.gitignore b/.gitignore index aed8a347ba08d2..9a42e446d6b162 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ *.iml *.swp *.jar -*.zip *.gz *.log *.so.tmp diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index dd89a77a9413e1..d0cab71cad6bed 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -6814,7 +6814,7 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti // check partitions for (Map.Entry entry : origPartitions.entrySet()) { Partition partition = copiedTbl.getPartition(entry.getValue()); - if (partition == null || !partition.getName().equals(entry.getKey())) { + if (partition == null || !partition.getName().equalsIgnoreCase(entry.getKey())) { throw new DdlException("Partition [" + entry.getKey() + "] is changed"); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index e108fb82d78057..9f068335abbaeb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -1286,11 +1286,13 @@ public OlapTable selectiveCopy(Collection reservedPartitions, IndexExtSt return copied; } - Set partNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + Set partNames = Sets.newHashSet(); partNames.addAll(copied.getPartitionNames()); + // partition name is case insensitive: + Set lowerReservedPartitionNames = reservedPartitions.stream().map(n -> n.toLowerCase()).collect(Collectors.toSet()); for (String partName : partNames) { - if (!reservedPartitions.contains(partName)) { + if (!lowerReservedPartitionNames.contains(partName.toLowerCase())) { copied.dropPartitionAndReserveTablet(partName); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/CaseSensibility.java b/fe/fe-core/src/main/java/org/apache/doris/common/CaseSensibility.java index ebd7b63b5b46b7..d0545a14dda643 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/CaseSensibility.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/CaseSensibility.java @@ -22,7 +22,7 @@ public enum CaseSensibility { DATABASE(true), TABLE(true), ROLLUP(true), - PARTITION(true), + PARTITION(false), COLUMN(false), USER(true), ROLE(false), diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java index db8951da4e8d8b..80aec2011f2826 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java @@ -56,6 +56,24 @@ public static void setup() throws Exception { "properties('replication_num' = '1');"; createDb(createDbStmtStr); createTable(createTableStr); + + String createTable2 = "CREATE TABLE test.case_sensitive_table (\n" + + " `date_id` date NULL COMMENT \"\",\n" + + " `column2` tinyint(4) NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`date_id`, `column2`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE(`date_id`)\n" + + "(\n" + + "PARTITION p20211006 VALUES [('2021-10-06'), ('2021-10-07')),\n" + + "PARTITION P20211007 VALUES [('2021-10-07'), ('2021-10-08')),\n" + + "PARTITION P20211008 VALUES [('2021-10-08'), ('2021-10-09')))\n" + + "DISTRIBUTED BY HASH(`column2`) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\"\n" + + ");"; + + createTable(createTable2); } @AfterClass @@ -64,6 +82,29 @@ public static void tearDown() { file.delete(); } + @Test + public void testTruncateWithCaseInsensitivePartitionName() throws Exception { + Database db = Catalog.getCurrentCatalog().getDbNullable("default_cluster:test"); + OlapTable tbl = db.getOlapTableOrDdlException("case_sensitive_table"); + long p20211006Id = tbl.getPartition("P20211006").getId(); + long p20211007Id = tbl.getPartition("P20211007").getId(); + long p20211008Id = tbl.getPartition("p20211008").getId(); + // truncate p20211008(real name is P20211008) + String truncateStr = "TRUNCATE TABLE test.case_sensitive_table PARTITION p20211008; \n"; + TruncateTableStmt truncateTableStmt = (TruncateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(truncateStr, connectContext); + Catalog.getCurrentCatalog().truncateTable(truncateTableStmt); + Assert.assertNotEquals(p20211008Id, tbl.getPartition("p20211008").getId()); + // 2. truncate P20211007 + truncateStr = "TRUNCATE TABLE test.case_sensitive_table PARTITION P20211007; \n"; + truncateTableStmt = (TruncateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(truncateStr, connectContext); + Catalog.getCurrentCatalog().truncateTable(truncateTableStmt); + Assert.assertEquals(3, tbl.getPartitionInfo().idToDataProperty.size()); + Assert.assertNotEquals(p20211007Id, tbl.getPartition("p20211007").getId()); + Assert.assertEquals(p20211006Id, tbl.getPartition("p20211006").getId()); + Assert.assertNotNull(tbl.getPartition("p20211006")); + Assert.assertNotNull(tbl.getPartition("P20211006")); + } + @Test public void testTruncateTable() throws Exception { String stmtStr = "ALTER TABLE test.tbl ADD PARTITION p20210902 VALUES [('2021-09-02'), ('2021-09-03')) DISTRIBUTED BY HASH(`k1`) BUCKETS 3;"; diff --git a/fe/fe-core/src/test/resources/help-resource.zip b/fe/fe-core/src/test/resources/help-resource.zip new file mode 100644 index 0000000000000000000000000000000000000000..7ab08e7bf20a74548c98387858b6c02cb7009e6b GIT binary patch literal 1056 zcmWIWW@h1H00EV4^FqK3D8b1f!;qYxSCX1nQmh{u!pXp#;**;+4~R=ExEUB(zA`c} zu!sN^2f&T6Ml(Vj*@)u89No0iyyTM1{5+_E6LA{|GMIw_WZ?W|ggT{XOCQ16w!SBL3GQRjxMov{BOQ)iD%G=16JTFp^>b|k~Qy$|1!Q{6|(a%j*ewmi-mZN?ENo7W`FO3(PBzeVdzljCh~@mbG1qJ8haU)s<7 z_(pw|n8}~{lC#e${88O2PWagumzY=ron-_ilDAwO}Bk@n&LE+wvw-IOlatC-b zGRZOHO8XMfTp+;k))B-+$_cEHoPd!F5XNEV17zczk&Q#o6+pv5sTZr^xbg(VaA0<1 fSkgEJx8d*{g3At8HlSt(W*`&*8ukmA=NK3OeJ^Ul literal 0 HcmV?d00001 From 14f8615c2572a5f6284257d5db091450bd491068 Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 1 May 2022 17:28:56 +0800 Subject: [PATCH 2/2] fix by review --- .../src/main/java/org/apache/doris/catalog/OlapTable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 9f068335abbaeb..c7795d5e2745ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -1290,7 +1290,7 @@ public OlapTable selectiveCopy(Collection reservedPartitions, IndexExtSt partNames.addAll(copied.getPartitionNames()); // partition name is case insensitive: - Set lowerReservedPartitionNames = reservedPartitions.stream().map(n -> n.toLowerCase()).collect(Collectors.toSet()); + Set lowerReservedPartitionNames = reservedPartitions.stream().map(String::toLowerCase).collect(Collectors.toSet()); for (String partName : partNames) { if (!lowerReservedPartitionNames.contains(partName.toLowerCase())) { copied.dropPartitionAndReserveTablet(partName);