From 81b45b044414c0761a9de74614a9f604e4e3ce19 Mon Sep 17 00:00:00 2001 From: yangzhg Date: Tue, 17 Nov 2020 13:38:00 +0800 Subject: [PATCH 1/4] disable the creation of segment v1 table --- be/src/common/config.h | 2 +- .../src/main/java/org/apache/doris/common/Config.java | 8 ++++++++ .../org/apache/doris/common/util/PropertyAnalyzer.java | 4 ++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/be/src/common/config.h b/be/src/common/config.h index 0be21a4f1c7e2d..c54cb5cd11bfb9 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -550,7 +550,7 @@ namespace config { // config for default rowset type // Valid configs: ALPHA, BETA - CONF_String(default_rowset_type, "ALPHA"); + CONF_String(default_rowset_type, "BETA"); // Maximum size of a single message body in all protocols CONF_Int64(brpc_max_body_size, "209715200"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java index 9de89e1f819a8a..39d8318e3529a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java @@ -1293,4 +1293,12 @@ public class Config extends ConfigBase { */ @ConfField public static String http_api_extra_base_path = ""; + + /** + * Whether to support the creation of alpha rowset tables. + * The default is false and should only be used in emergency situations, + * this config should be remove in some future version + */ + @ConfField + public static boolean enable_alpha_rowset = false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index dcc3ea091c2b91..15250e65ec0075 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -411,6 +411,10 @@ public static TStorageFormat analyzeStorageFormat(Map properties } if (storageFormat.equalsIgnoreCase("v1")) { + if (!Config.enable_alpha_rowset) { + throw new AnalysisException("StorageFormat V1 has been deprecated since version 0.14," + + " please use V2 instead"); + } return TStorageFormat.V1; } else if (storageFormat.equalsIgnoreCase("v2")) { return TStorageFormat.V2; From 6b56f7259e55936cdb7d5b0b3146c377c3209858 Mon Sep 17 00:00:00 2001 From: yangzhg Date: Tue, 17 Nov 2020 14:16:51 +0800 Subject: [PATCH 2/4] fix unit test --- .../doris/common/util/PropertyAnalyzer.java | 2 +- .../apache/doris/alter/AlterJobV2Test.java | 2 +- .../doris/common/PropertyAnalyzerTest.java | 23 +++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 15250e65ec0075..9b726c6c480a76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -412,7 +412,7 @@ public static TStorageFormat analyzeStorageFormat(Map properties if (storageFormat.equalsIgnoreCase("v1")) { if (!Config.enable_alpha_rowset) { - throw new AnalysisException("StorageFormat V1 has been deprecated since version 0.14," + + throw new AnalysisException("Storage format V1 has been deprecated since version 0.14," + " please use V2 instead"); } return TStorageFormat.V1; diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterJobV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterJobV2Test.java index 0c0e3f494abd09..e513d04e2f7304 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterJobV2Test.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterJobV2Test.java @@ -63,7 +63,7 @@ public static void beforeClass() throws Exception { createTable("CREATE TABLE test.schema_change_test(k1 int, k2 int, k3 int) distributed by hash(k1) buckets 3 properties('replication_num' = '1');"); - createTable("CREATE TABLE test.segmentv2(k1 int, k2 int, v1 int sum) distributed by hash(k1) buckets 3 properties('replication_num' = '1', 'storage_format' = 'v1');"); + createTable("CREATE TABLE test.segmentv2(k1 int, k2 int, v1 int sum) distributed by hash(k1) buckets 3 properties('replication_num' = '1', 'storage_format' = 'v2');"); } @AfterClass diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java index ddbf6c1dbc5676..9239a711b0c86b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java @@ -26,6 +26,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.thrift.TStorageFormat; import org.apache.doris.thrift.TStorageMedium; import com.google.common.collect.Lists; @@ -33,14 +34,19 @@ import com.google.common.collect.Sets; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import java.text.SimpleDateFormat; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; public class PropertyAnalyzerTest { + @Rule + public ExpectedException expectedEx = ExpectedException.none(); @Test public void testBfColumns() throws AnalysisException { @@ -144,4 +150,21 @@ public void testStorageMedium() throws AnalysisException { DateLiteral dateLiteral = new DateLiteral(tomorrowTimeStr, Type.DATETIME); Assert.assertEquals(dateLiteral.unixTimestamp(TimeUtils.getTimeZone()), dataProperty.getCooldownTimeMs()); } + + @Test + public void testStorageFormat() throws AnalysisException { + HashMap propertiesV1 = Maps.newHashMap(); + HashMap propertiesV2 = Maps.newHashMap(); + HashMap propertiesDefault = Maps.newHashMap(); + propertiesV1.put(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT, "v1"); + propertiesV2.put(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT, "v2"); + propertiesDefault.put(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT, "default"); + Assert.assertEquals(TStorageFormat.V2, PropertyAnalyzer.analyzeStorageFormat(null)); + Assert.assertEquals(TStorageFormat.V2, PropertyAnalyzer.analyzeStorageFormat(propertiesV2)); + Assert.assertEquals(TStorageFormat.V2, PropertyAnalyzer.analyzeStorageFormat(propertiesDefault)); + expectedEx.expect(AnalysisException.class); + expectedEx.expectMessage("Storage format V1 has been deprecated since version 0.14," + + " please use V2 instead"); + PropertyAnalyzer.analyzeStorageFormat(propertiesV1); + } } From 76d0081d9201b7c75e8e820bec8b6a61f1bc10fb Mon Sep 17 00:00:00 2001 From: yangzhg Date: Tue, 17 Nov 2020 14:23:11 +0800 Subject: [PATCH 3/4] fix unit test --- .../src/test/java/org/apache/doris/alter/AlterJobV2Test.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterJobV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterJobV2Test.java index e513d04e2f7304..e1ed50f53c2af3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterJobV2Test.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterJobV2Test.java @@ -133,7 +133,7 @@ public void testAlterSegmentV2() throws Exception { Assert.assertNotNull(db); OlapTable tbl = (OlapTable) db.getTable("segmentv2"); Assert.assertNotNull(tbl); - Assert.assertEquals(TStorageFormat.V1, tbl.getTableProperty().getStorageFormat()); + Assert.assertEquals(TStorageFormat.V2, tbl.getTableProperty().getStorageFormat()); // 1. create a rollup r1 String alterStmtStr = "alter table test.segmentv2 add rollup r1(k2, v1)"; From 8e5e3112a0d3e6bd8f9d4173c7975cc5f0fe1aeb Mon Sep 17 00:00:00 2001 From: yangzhg Date: Tue, 17 Nov 2020 14:30:33 +0800 Subject: [PATCH 4/4] fix unit test --- .../test/java/org/apache/doris/alter/AlterJobV2Test.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterJobV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterJobV2Test.java index e1ed50f53c2af3..c0a0266bb22ba6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterJobV2Test.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterJobV2Test.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.qe.ConnectContext; @@ -53,6 +54,7 @@ public static void beforeClass() throws Exception { FeConstants.runningUnitTest = true; UtFrameUtils.createMinDorisCluster(runningDir); + Config.enable_alpha_rowset = true; // create connect context connectContext = UtFrameUtils.createDefaultCtx(); @@ -63,7 +65,7 @@ public static void beforeClass() throws Exception { createTable("CREATE TABLE test.schema_change_test(k1 int, k2 int, k3 int) distributed by hash(k1) buckets 3 properties('replication_num' = '1');"); - createTable("CREATE TABLE test.segmentv2(k1 int, k2 int, v1 int sum) distributed by hash(k1) buckets 3 properties('replication_num' = '1', 'storage_format' = 'v2');"); + createTable("CREATE TABLE test.segmentv2(k1 int, k2 int, v1 int sum) distributed by hash(k1) buckets 3 properties('replication_num' = '1', 'storage_format' = 'v1');"); } @AfterClass @@ -128,12 +130,14 @@ public void testRollup() throws Exception { } @Test + @Deprecated public void testAlterSegmentV2() throws Exception { + // TODO this test should remove after we disable segment v1 completely Database db = Catalog.getCurrentCatalog().getDb("default_cluster:test"); Assert.assertNotNull(db); OlapTable tbl = (OlapTable) db.getTable("segmentv2"); Assert.assertNotNull(tbl); - Assert.assertEquals(TStorageFormat.V2, tbl.getTableProperty().getStorageFormat()); + Assert.assertEquals(TStorageFormat.V1, tbl.getTableProperty().getStorageFormat()); // 1. create a rollup r1 String alterStmtStr = "alter table test.segmentv2 add rollup r1(k2, v1)";