From 46a7f9d18dd4188117ac728db7abfa0e08493dca Mon Sep 17 00:00:00 2001 From: qijianliang01 Date: Wed, 17 Mar 2021 11:16:35 +0800 Subject: [PATCH 1/6] add list partition support Change-Id: Ie0b5c1b518bdd4f27e0eedc08dd71e8b1048e2ee --- be/src/exec/tablet_info.cpp | 48 +- be/src/exec/tablet_info.h | 15 +- be/test/exec/tablet_info_test.cpp | 413 +++++++++++ .../alter-table/alter-table-temp-partition.md | 49 +- .../administrator-guide/dynamic-partition.md | 2 + .../load-data/delete-manual.md | 2 +- docs/en/getting-started/best-practice.md | 15 +- docs/en/getting-started/data-partition.md | 173 ++++- .../Data Definition/CREATE TABLE.md | 216 ++++-- .../sql-statements/Data Definition/RESTORE.md | 2 +- .../Data Manipulation/DELETE.md | 2 +- .../alter-table/alter-table-temp-partition.md | 49 +- .../administrator-guide/dynamic-partition.md | 2 + .../load-data/delete-manual.md | 2 +- docs/zh-CN/getting-started/best-practice.md | 16 +- docs/zh-CN/getting-started/data-partition.md | 173 ++++- .../Data Definition/CREATE TABLE.md | 235 +++++-- .../sql-statements/Data Definition/RESTORE.md | 2 +- .../Data Manipulation/DELETE.md | 2 +- fe/fe-core/src/main/cup/sql_parser.cup | 98 ++- .../doris/alter/SchemaChangeHandler.java | 6 +- .../doris/analysis/AddPartitionClause.java | 6 +- .../apache/doris/analysis/BoolLiteral.java | 8 + .../doris/analysis/CreateTableStmt.java | 9 +- .../doris/analysis/ListPartitionDesc.java | 110 +++ .../apache/doris/analysis/LiteralExpr.java | 2 +- .../apache/doris/analysis/PartitionDesc.java | 85 ++- .../doris/analysis/PartitionKeyDesc.java | 64 +- .../doris/analysis/RangePartitionDesc.java | 120 +--- ...tionDesc.java => SinglePartitionDesc.java} | 14 +- .../apache/doris/analysis/StringLiteral.java | 9 +- .../org/apache/doris/backup/RestoreJob.java | 69 +- .../org/apache/doris/catalog/Catalog.java | 147 ++-- .../doris/catalog/CatalogRecycleBin.java | 71 +- .../doris/catalog/ListPartitionInfo.java | 212 ++++++ .../doris/catalog/ListPartitionItem.java | 148 ++++ .../org/apache/doris/catalog/OlapTable.java | 164 +++-- .../apache/doris/catalog/PartitionInfo.java | 143 +++- .../apache/doris/catalog/PartitionItem.java | 33 + .../apache/doris/catalog/PartitionKey.java | 43 ++ .../apache/doris/catalog/PartitionType.java | 7 +- .../doris/catalog/RangePartitionInfo.java | 172 +---- .../doris/catalog/RangePartitionItem.java | 83 +++ .../clone/DynamicPartitionScheduler.java | 24 +- .../apache/doris/common/FeMetaVersion.java | 4 +- .../common/proc/EsPartitionsProcDir.java | 2 +- .../doris/common/proc/PartitionsProcDir.java | 16 +- .../apache/doris/common/util/ListUtil.java | 84 +++ .../apache/doris/common/util/RangeUtils.java | 38 +- .../elasticsearch/EsShardPartitions.java | 8 +- .../elasticsearch/EsTablePartitions.java | 7 +- .../load/loadv2/SparkLoadPendingTask.java | 12 +- .../doris/persist/PartitionPersistInfo.java | 17 +- .../apache/doris/planner/DataSplitSink.java | 2 +- .../org/apache/doris/planner/EsScanNode.java | 5 +- .../doris/planner/ListPartitionPruner.java | 149 ++-- .../apache/doris/planner/OlapScanNode.java | 32 +- .../apache/doris/planner/OlapTableSink.java | 57 +- .../doris/planner/RangePartitionPruner.java | 9 +- .../org/apache/doris/qe/ConnectProcessor.java | 12 +- .../apache/doris/qe/cache/PartitionRange.java | 9 +- .../doris/task/HadoopLoadPendingTask.java | 2 +- fe/fe-core/src/main/jflex/sql_scanner.flex | 1 + .../analysis/ListPartitionPrunerTest.java | 170 +++++ .../doris/analysis/PartitionKeyDescTest.java | 28 +- .../apache/doris/backup/CatalogMocker.java | 8 +- .../apache/doris/catalog/CatalogTestUtil.java | 11 +- .../apache/doris/catalog/CreateTableTest.java | 251 +++++++ .../doris/catalog/ListPartitionInfoTest.java | 227 ++++++ .../doris/catalog/PartitionKeyTest.java | 58 +- .../doris/catalog/RangePartitionInfoTest.java | 160 ++--- .../doris/catalog/TempPartitionTest.java | 653 +++++++++++++++++- .../doris/common/util/ListUtilTest.java | 64 ++ .../load/loadv2/SparkLoadPendingTaskTest.java | 23 +- .../doris/planner/OlapTableSinkTest.java | 35 + gensrc/thrift/Descriptors.thrift | 1 + gensrc/thrift/Partitions.thrift | 3 + 77 files changed, 4487 insertions(+), 936 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/ListPartitionDesc.java rename fe/fe-core/src/main/java/org/apache/doris/analysis/{SingleRangePartitionDesc.java => SinglePartitionDesc.java} (92%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionItem.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionItem.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/ListPartitionPrunerTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/ListPartitionInfoTest.java diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp index d734a4ca5c9afa..7c2d07463aaf91 100644 --- a/be/src/exec/tablet_info.cpp +++ b/be/src/exec/tablet_info.cpp @@ -122,10 +122,22 @@ std::string OlapTableSchemaParam::debug_string() const { std::string OlapTablePartition::debug_string(TupleDescriptor* tuple_desc) const { std::stringstream ss; + std::stringstream in_keys_ss; + int idx = 0; + in_keys_ss << "["; + for (auto in_key : in_keys) { + if (idx++ > 0) { + in_keys_ss << ","; + } + in_keys_ss << Tuple::to_string(in_key, *tuple_desc); + } + in_keys_ss << "]"; ss << "(id=" << id << ",start_key=" << Tuple::to_string(start_key, *tuple_desc) - << ",end_key=" << Tuple::to_string(end_key, *tuple_desc) << ",num_buckets=" << num_buckets + << ",end_key=" << Tuple::to_string(end_key, *tuple_desc) + << ",in_key=" << in_keys_ss.str() + << ",num_buckets=" << num_buckets << ",indexes=["; - int idx = 0; + idx = 0; for (auto& index : indexes) { if (idx++ > 0) { ss << ","; @@ -211,6 +223,12 @@ Status OlapTablePartitionParam::init() { } else if (t_part.__isset.end_keys) { RETURN_IF_ERROR(_create_partition_keys(t_part.end_keys, &part->end_key)); } + if (t_part.__isset.in_keys) { + part->in_keys.resize(t_part.in_keys.size()); + for (int j = 0; j < t_part.in_keys.size(); j++) { + RETURN_IF_ERROR(_create_partition_keys(t_part.in_keys[j], &part->in_keys[j])); + } + } part->num_buckets = t_part.num_buckets; auto num_indexes = _schema->indexes().size(); @@ -237,14 +255,27 @@ Status OlapTablePartitionParam::init() { } } _partitions.emplace_back(part); - _partitions_map->emplace(part->end_key, part); + if (t_part.__isset.in_keys) { + for (auto in_key : part->in_keys) { + _partitions_map->emplace(in_key, part); + } + } else { + _partitions_map->emplace(part->end_key, part); + } } return Status::OK(); } bool OlapTablePartitionParam::find_tablet(Tuple* tuple, const OlapTablePartition** partition, uint32_t* dist_hashes) const { - auto it = _partitions_map->upper_bound(tuple); + const TOlapTablePartition& t_part = _t_param.partitions[0]; + decltype(_partitions_map->find(tuple)) it; + if (t_part.__isset.in_keys) { + it = _partitions_map->find(tuple); + } else { + it = _partitions_map->upper_bound(tuple); + + } if (it == _partitions_map->end()) { return false; } @@ -312,6 +343,15 @@ Status OlapTablePartitionParam::_create_partition_key(const TExprNode& t_expr, T memcpy(slot, &val, sizeof(val)); break; } + case TExprNodeType::STRING_LITERAL: { + *reinterpret_cast(slot) = StringValue(const_cast(t_expr.string_literal.value.c_str()), + t_expr.string_literal.value.size()); + break; + } + case TExprNodeType::BOOL_LITERAL: { + *reinterpret_cast(slot) = t_expr.bool_literal.value; + break; + } default: { std::stringstream ss; ss << "unsupported partition column node type, type=" << t_expr.node_type; diff --git a/be/src/exec/tablet_info.h b/be/src/exec/tablet_info.h index 194fd9079fbd84..59c296260c4d75 100644 --- a/be/src/exec/tablet_info.h +++ b/be/src/exec/tablet_info.h @@ -94,6 +94,7 @@ struct OlapTablePartition { int64_t id = 0; Tuple* start_key = nullptr; Tuple* end_key = nullptr; + std::vector in_keys; int64_t num_buckets = 0; std::vector indexes; @@ -168,11 +169,23 @@ class OlapTablePartitionParam { // check if this partition contain this key bool _part_contains(OlapTablePartition* part, Tuple* key) const { - if (part->start_key == nullptr) { + if ((part->start_key == nullptr) && (part->in_keys.size() == 0)) { // start_key is nullptr means the lower bound is boundless return true; } OlapTablePartKeyComparator comparator(_partition_slot_descs); + const TOlapTablePartition& t_part = _t_param.partitions[0]; + // when list partition, return true if equals. + if (t_part.__isset.in_keys) { + bool ret = false; + for (auto in_key : part->in_keys) { + ret = !comparator(key, in_key) && !comparator(in_key, key); + if (ret) { + break; + } + } + return ret; + } return !comparator(key, part->start_key); } diff --git a/be/test/exec/tablet_info_test.cpp b/be/test/exec/tablet_info_test.cpp index 6669fb50c7cf03..01d325e7912371 100644 --- a/be/test/exec/tablet_info_test.cpp +++ b/be/test/exec/tablet_info_test.cpp @@ -224,6 +224,419 @@ TEST_F(OlapTablePartitionParamTest, normal) { } } +/* + *PARTITION BY LIST(`k1`) + * ( + * PARTITION p1 VALUES IN ("1", "2"), + * PARTITION p2 VALUES IN ("3"), + * PARTITION p3 VALUES IN ("4", "5", "6") + * ) + * +*/ +TEST_F(OlapTablePartitionParamTest, single_list_partition) { + TDescriptorTable t_desc_tbl; + auto t_schema = get_schema(&t_desc_tbl); + std::shared_ptr schema(new OlapTableSchemaParam()); + auto st = schema->init(t_schema); + ASSERT_TRUE(st.ok()); + LOG(INFO) << schema->debug_string(); + + // 1 | 2 | 3 | 4 | 5 | 6 + std::vector t_node_1; + t_node_1.resize(1); + t_node_1[0].node_type = TExprNodeType::INT_LITERAL; + t_node_1[0].type = t_desc_tbl.slotDescriptors[1].slotType; + t_node_1[0].num_children = 0; + t_node_1[0].__isset.int_literal = true; + t_node_1[0].int_literal.value = 1; + + std::vector t_node_2; + t_node_2.resize(1); + t_node_2[0].node_type = TExprNodeType::INT_LITERAL; + t_node_2[0].type = t_desc_tbl.slotDescriptors[1].slotType; + t_node_2[0].num_children = 0; + t_node_2[0].__isset.int_literal = true; + t_node_2[0].int_literal.value = 2; + + std::vector t_node_3; + t_node_3.resize(1); + t_node_3[0].node_type = TExprNodeType::INT_LITERAL; + t_node_3[0].type = t_desc_tbl.slotDescriptors[1].slotType; + t_node_3[0].num_children = 0; + t_node_3[0].__isset.int_literal = true; + t_node_3[0].int_literal.value = 3; + + std::vector t_node_4; + t_node_4.resize(1); + t_node_4[0].node_type = TExprNodeType::INT_LITERAL; + t_node_4[0].type = t_desc_tbl.slotDescriptors[1].slotType; + t_node_4[0].num_children = 0; + t_node_4[0].__isset.int_literal = true; + t_node_4[0].int_literal.value = 4; + + std::vector t_node_5; + t_node_5.resize(1); + t_node_5[0].node_type = TExprNodeType::INT_LITERAL; + t_node_5[0].type = t_desc_tbl.slotDescriptors[1].slotType; + t_node_5[0].num_children = 0; + t_node_5[0].__isset.int_literal = true; + t_node_5[0].int_literal.value = 5; + + std::vector t_node_6; + t_node_6.resize(1); + t_node_6[0].node_type = TExprNodeType::INT_LITERAL; + t_node_6[0].type = t_desc_tbl.slotDescriptors[1].slotType; + t_node_6[0].num_children = 0; + t_node_6[0].__isset.int_literal = true; + t_node_6[0].int_literal.value = 6; + + TOlapTablePartitionParam t_partition_param; + t_partition_param.db_id = 1; + t_partition_param.table_id = 2; + t_partition_param.version = 0; + t_partition_param.__set_partition_column("c2"); + t_partition_param.__set_distributed_columns({"c1", "c3"}); + t_partition_param.partitions.resize(3); + t_partition_param.partitions[0].id = 10; + t_partition_param.partitions[0].__isset.in_keys = true; + t_partition_param.partitions[0].in_keys.emplace_back(t_node_1); + t_partition_param.partitions[0].in_keys.emplace_back(t_node_2); + t_partition_param.partitions[0].num_buckets = 1; + t_partition_param.partitions[0].indexes.resize(2); + t_partition_param.partitions[0].indexes[0].index_id = 4; + t_partition_param.partitions[0].indexes[0].tablets = {21}; + t_partition_param.partitions[0].indexes[1].index_id = 5; + t_partition_param.partitions[0].indexes[1].tablets = {22}; + + t_partition_param.partitions[1].id = 11; + t_partition_param.partitions[1].__isset.in_keys = true; + t_partition_param.partitions[1].in_keys.emplace_back(t_node_3); + t_partition_param.partitions[1].num_buckets = 2; + t_partition_param.partitions[1].indexes.resize(2); + t_partition_param.partitions[1].indexes[0].index_id = 4; + t_partition_param.partitions[1].indexes[0].tablets = {31, 32}; + t_partition_param.partitions[1].indexes[1].index_id = 5; + t_partition_param.partitions[1].indexes[1].tablets = {33, 34}; + + t_partition_param.partitions[2].id = 12; + t_partition_param.partitions[2].__isset.in_keys = true; + t_partition_param.partitions[2].in_keys.emplace_back(t_node_4); + t_partition_param.partitions[2].in_keys.emplace_back(t_node_5); + t_partition_param.partitions[2].in_keys.emplace_back(t_node_6); + t_partition_param.partitions[2].num_buckets = 4; + t_partition_param.partitions[2].indexes.resize(2); + t_partition_param.partitions[2].indexes[0].index_id = 4; + t_partition_param.partitions[2].indexes[0].tablets = {41, 42, 43, 44}; + t_partition_param.partitions[2].indexes[1].index_id = 5; + t_partition_param.partitions[2].indexes[1].tablets = {45, 46, 47, 48}; + + OlapTablePartitionParam part(schema, t_partition_param); + st = part.init(); + ASSERT_TRUE(st.ok()); + LOG(INFO) << part.debug_string(); + + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + st = DescriptorTbl::create(&pool, t_desc_tbl, &desc_tbl); + ASSERT_TRUE(st.ok()); + RowDescriptor row_desc(*desc_tbl, {0}, {false}); + TupleDescriptor* tuple_desc = desc_tbl->get_tuple_descriptor(0); + auto tracker = std::make_shared(); + RowBatch batch(row_desc, 1024, tracker.get()); + // 12, 1, "abc" + { + Tuple* tuple = (Tuple*)batch.tuple_data_pool()->allocate(tuple_desc->byte_size()); + memset(tuple, 0, tuple_desc->byte_size()); + + *reinterpret_cast(tuple->get_slot(4)) = 12; + *reinterpret_cast(tuple->get_slot(8)) = 1; + StringValue* str_val = reinterpret_cast(tuple->get_slot(16)); + str_val->ptr = (char*)batch.tuple_data_pool()->allocate(10); + str_val->len = 3; + memcpy(str_val->ptr, "abc", str_val->len); + + // 1: + uint32_t dist_hash = 0; + const OlapTablePartition* partition = nullptr; + auto found = part.find_tablet(tuple, &partition, &dist_hash); + ASSERT_TRUE(found); + ASSERT_EQ(10, partition->id); + } + // 13, 3, "abcd" + { + Tuple* tuple = (Tuple*)batch.tuple_data_pool()->allocate(tuple_desc->byte_size()); + memset(tuple, 0, tuple_desc->byte_size()); + + *reinterpret_cast(tuple->get_slot(4)) = 13; + *reinterpret_cast(tuple->get_slot(8)) = 3; + StringValue* str_val = reinterpret_cast(tuple->get_slot(16)); + str_val->ptr = (char*)batch.tuple_data_pool()->allocate(10); + str_val->len = 4; + memcpy(str_val->ptr, "abcd", str_val->len); + + // 3: + uint32_t dist_hash = 0; + const OlapTablePartition* partition = nullptr; + auto found = part.find_tablet(tuple, &partition, &dist_hash); + ASSERT_TRUE(found); + ASSERT_EQ(11, partition->id); + } + // 14, 50, "abcde" + { + Tuple* tuple = (Tuple*)batch.tuple_data_pool()->allocate(tuple_desc->byte_size()); + memset(tuple, 0, tuple_desc->byte_size()); + + *reinterpret_cast(tuple->get_slot(4)) = 14; + *reinterpret_cast(tuple->get_slot(8)) = 50; + StringValue* str_val = reinterpret_cast(tuple->get_slot(16)); + str_val->ptr = reinterpret_cast(batch.tuple_data_pool()->allocate(10)); + str_val->len = 5; + memcpy(str_val->ptr, "abcde", str_val->len); + + // 50: + uint32_t dist_hash = 0; + const OlapTablePartition* partition = nullptr; + auto found = part.find_tablet(tuple, &partition, &dist_hash); + ASSERT_FALSE(found); + } + + // 15, 6, "abcdef" + { + Tuple* tuple = (Tuple*)batch.tuple_data_pool()->allocate(tuple_desc->byte_size()); + memset(tuple, 0, tuple_desc->byte_size()); + + *reinterpret_cast(tuple->get_slot(4)) = 15; + *reinterpret_cast(tuple->get_slot(8)) = 6; + StringValue* str_val = reinterpret_cast(tuple->get_slot(16)); + str_val->ptr = reinterpret_cast(batch.tuple_data_pool()->allocate(10)); + str_val->len = 6; + memcpy(str_val->ptr, "abcdef", str_val->len); + + // 6: + uint32_t dist_hash = 0; + const OlapTablePartition* partition = nullptr; + auto found = part.find_tablet(tuple, &partition, &dist_hash); + ASSERT_TRUE(found); + ASSERT_EQ(12, partition->id); + } +} + +/* + * multi list partition key test + * + * PARTITION BY LIST(c2, c3) + * ( + * PARTITION p1 VALUES IN (("1", "beijing"),("1", "shanghai")), + * PARTITION p2 VALUES IN (("1", "tianjin"),("2", "beijing")), + * PARTITION p3 VALUES IN (("2", "shanghai")) + * ) + * + */ +TEST_F(OlapTablePartitionParamTest, multi_list_partition) { + TDescriptorTable t_desc_tbl; + auto t_schema = get_schema(&t_desc_tbl); + std::shared_ptr schema(new OlapTableSchemaParam()); + auto st = schema->init(t_schema); + ASSERT_TRUE(st.ok()); + LOG(INFO) << schema->debug_string(); + + // 1 + TExprNode node_1; + node_1.node_type = TExprNodeType::INT_LITERAL; + node_1.type = t_desc_tbl.slotDescriptors[1].slotType; + node_1.num_children = 0; + node_1.__isset.int_literal = true; + node_1.int_literal.value = 1; + // 2 + TExprNode node_2; + node_2.node_type = TExprNodeType::INT_LITERAL; + node_2.type = t_desc_tbl.slotDescriptors[1].slotType; + node_2.num_children = 0; + node_2.__isset.int_literal = true; + node_2.int_literal.value = 2; + // beijing + TExprNode node_b; + node_b.node_type = TExprNodeType::STRING_LITERAL; + node_b.type = t_desc_tbl.slotDescriptors[2].slotType; + node_b.num_children = 0; + node_b.__isset.string_literal = true; + node_b.string_literal.value = "beijing"; + // shanghai + TExprNode node_s; + node_s.node_type = TExprNodeType::STRING_LITERAL; + node_s.type = t_desc_tbl.slotDescriptors[2].slotType; + node_s.num_children = 0; + node_s.__isset.string_literal = true; + node_s.string_literal.value = "shanghai"; + // tianjin + TExprNode node_t; + node_t.node_type = TExprNodeType::STRING_LITERAL; + node_t.type = t_desc_tbl.slotDescriptors[2].slotType; + node_t.num_children = 0; + node_t.__isset.string_literal = true; + node_t.string_literal.value = "tianjin"; + + // (1, beijing) + std::vector t_nodes_1; + t_nodes_1.emplace_back(node_1); + t_nodes_1.emplace_back(node_b); + + // (1, shanghai) + std::vector t_nodes_2; + t_nodes_2.emplace_back(node_1); + t_nodes_2.emplace_back(node_s); + + // (1, tianjin) + std::vector t_nodes_3; + t_nodes_3.emplace_back(node_1); + t_nodes_3.emplace_back(node_t); + + // (2, beijing) + std::vector t_nodes_4; + t_nodes_4.emplace_back(node_2); + t_nodes_4.emplace_back(node_b); + + // (2, shanghai) + std::vector t_nodes_5; + t_nodes_5.emplace_back(node_2); + t_nodes_5.emplace_back(node_s); + + TOlapTablePartitionParam t_partition_param; + t_partition_param.db_id = 1; + t_partition_param.table_id = 2; + t_partition_param.version = 0; + t_partition_param.__set_partition_columns({"c2", "c3"}); + t_partition_param.__set_distributed_columns({"c1"}); + t_partition_param.partitions.resize(3); + + // (("1", "beijing"),("1", "shanghai")) + t_partition_param.partitions[0].id = 10; + t_partition_param.partitions[0].__isset.in_keys = true; + t_partition_param.partitions[0].in_keys.resize(2); + t_partition_param.partitions[0].in_keys[0] = t_nodes_1; + t_partition_param.partitions[0].in_keys[1] = t_nodes_2; + t_partition_param.partitions[0].num_buckets = 1; + t_partition_param.partitions[0].indexes.resize(2); + t_partition_param.partitions[0].indexes[0].index_id = 4; + t_partition_param.partitions[0].indexes[0].tablets = {21}; + t_partition_param.partitions[0].indexes[1].index_id = 5; + t_partition_param.partitions[0].indexes[1].tablets = {22}; + + // (("1", "tianjin"),("2", "beijing")) + t_partition_param.partitions[1].id = 11; + t_partition_param.partitions[1].__isset.in_keys = true; + t_partition_param.partitions[1].in_keys.emplace_back(t_nodes_3); + t_partition_param.partitions[1].in_keys.emplace_back(t_nodes_4); + t_partition_param.partitions[1].num_buckets = 2; + t_partition_param.partitions[1].indexes.resize(2); + t_partition_param.partitions[1].indexes[0].index_id = 4; + t_partition_param.partitions[1].indexes[0].tablets = {31, 32}; + t_partition_param.partitions[1].indexes[1].index_id = 5; + t_partition_param.partitions[1].indexes[1].tablets = {33, 34}; + + // (("2", "shanghai"))) + t_partition_param.partitions[2].id = 12; + t_partition_param.partitions[2].__isset.in_keys = true; + t_partition_param.partitions[2].in_keys.emplace_back(t_nodes_5); + t_partition_param.partitions[2].num_buckets = 4; + t_partition_param.partitions[2].indexes.resize(2); + t_partition_param.partitions[2].indexes[0].index_id = 4; + t_partition_param.partitions[2].indexes[0].tablets = {41, 42, 43, 44}; + t_partition_param.partitions[2].indexes[1].index_id = 5; + t_partition_param.partitions[2].indexes[1].tablets = {45, 46, 47, 48}; + + OlapTablePartitionParam part(schema, t_partition_param); + st = part.init(); + LOG(INFO) << st.get_error_msg(); + ASSERT_TRUE(st.ok()); + LOG(INFO) << part.debug_string(); + + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + st = DescriptorTbl::create(&pool, t_desc_tbl, &desc_tbl); + ASSERT_TRUE(st.ok()); + RowDescriptor row_desc(*desc_tbl, {0}, {false}); + TupleDescriptor* tuple_desc = desc_tbl->get_tuple_descriptor(0); + auto tracker = std::make_shared(); + RowBatch batch(row_desc, 1024, tracker.get()); + // 12, 1, "beijing" + { + Tuple* tuple = (Tuple*)batch.tuple_data_pool()->allocate(tuple_desc->byte_size()); + memset(tuple, 0, tuple_desc->byte_size()); + + *reinterpret_cast(tuple->get_slot(4)) = 12; + *reinterpret_cast(tuple->get_slot(8)) = 1; + StringValue* str_val = reinterpret_cast(tuple->get_slot(16)); + str_val->ptr = (char*)batch.tuple_data_pool()->allocate(10); + str_val->len = 7; + memcpy(str_val->ptr, "beijing", str_val->len); + + // 1, beijing + uint32_t dist_hash = 0; + const OlapTablePartition* partition = nullptr; + auto found = part.find_tablet(tuple, &partition, &dist_hash); + ASSERT_TRUE(found); + ASSERT_EQ(10, partition->id); + } + // 13, 2, "shanghai" + { + Tuple* tuple = (Tuple*)batch.tuple_data_pool()->allocate(tuple_desc->byte_size()); + memset(tuple, 0, tuple_desc->byte_size()); + + *reinterpret_cast(tuple->get_slot(4)) = 13; + *reinterpret_cast(tuple->get_slot(8)) = 2; + StringValue* str_val = reinterpret_cast(tuple->get_slot(16)); + str_val->ptr = (char*)batch.tuple_data_pool()->allocate(10); + str_val->len = 8; + memcpy(str_val->ptr, "shanghai", str_val->len); + + // 2, shanghai + uint32_t dist_hash = 0; + const OlapTablePartition* partition = nullptr; + auto found = part.find_tablet(tuple, &partition, &dist_hash); + ASSERT_TRUE(found); + ASSERT_EQ(12, partition->id); + } + // 14, 50, "beijing" + { + Tuple* tuple = (Tuple*)batch.tuple_data_pool()->allocate(tuple_desc->byte_size()); + memset(tuple, 0, tuple_desc->byte_size()); + + *reinterpret_cast(tuple->get_slot(4)) = 14; + *reinterpret_cast(tuple->get_slot(8)) = 50; + StringValue* str_val = reinterpret_cast(tuple->get_slot(16)); + str_val->ptr = reinterpret_cast(batch.tuple_data_pool()->allocate(10)); + str_val->len = 7; + memcpy(str_val->ptr, "beijing", str_val->len); + + // 50, beijing + uint32_t dist_hash = 0; + const OlapTablePartition* partition = nullptr; + auto found = part.find_tablet(tuple, &partition, &dist_hash); + ASSERT_FALSE(found); + } + + // 15, 1, "tianjin" + { + Tuple* tuple = (Tuple*)batch.tuple_data_pool()->allocate(tuple_desc->byte_size()); + memset(tuple, 0, tuple_desc->byte_size()); + + *reinterpret_cast(tuple->get_slot(4)) = 15; + *reinterpret_cast(tuple->get_slot(8)) = 1; + StringValue* str_val = reinterpret_cast(tuple->get_slot(16)); + str_val->ptr = reinterpret_cast(batch.tuple_data_pool()->allocate(10)); + str_val->len = 7; + memcpy(str_val->ptr, "tianjin", str_val->len); + + // 1, tianjin + uint32_t dist_hash = 0; + const OlapTablePartition* partition = nullptr; + auto found = part.find_tablet(tuple, &partition, &dist_hash); + ASSERT_TRUE(found); + ASSERT_EQ(11, partition->id); + } +} + TEST_F(OlapTablePartitionParamTest, to_protobuf) { TDescriptorTable t_desc_tbl; auto t_schema = get_schema(&t_desc_tbl); diff --git a/docs/en/administrator-guide/alter-table/alter-table-temp-partition.md b/docs/en/administrator-guide/alter-table/alter-table-temp-partition.md index 412df27b9401c8..94e735d36b45ab 100644 --- a/docs/en/administrator-guide/alter-table/alter-table-temp-partition.md +++ b/docs/en/administrator-guide/alter-table/alter-table-temp-partition.md @@ -52,6 +52,15 @@ ALTER TABLE tbl2 ADD TEMPORARY PARTITION tp1 VALUES [("2020-01-01"), ("2020-02-0 ALTER TABLE tbl1 ADD TEMPORARY PARTITION tp1 VALUES LESS THAN ("2020-02-01") ("in_memory" = "true", "replication_num" = "1") DISTRIBUTED BY HASH (k1) BUCKETS 5; + +ALTER TABLE tbl3 ADD TEMPORARY PARTITION tp1 VALUES IN ("Beijing", "Shanghai"); + +ALTER TABLE tbl4 ADD TEMPORARY PARTITION tp1 VALUES IN ((1, "Beijing"), (1, "Shanghai")); + +ALTER TABLE tbl3 ADD TEMPORARY PARTITION tp1 VALUES IN ("Beijing", "Shanghai") +("in_memory" = "true", "replication_num" = "1") +DISTRIBUTED BY HASH(k1) BUCKETS 5; + ``` See `HELP ALTER TABLE;` for more help and examples. @@ -97,7 +106,13 @@ The replace operation has two special optional parameters: 1. `strict_range` - The default is true. When this parameter is true, the range union of all formal partitions to be replaced needs to be the same as the range union of the temporary partitions to be replaced. When set to false, you only need to ensure that the range between the new formal partitions does not overlap after replacement. Here are some examples: + The default is true. + + For Range partition, When this parameter is true, the range union of all formal partitions to be replaced needs to be the same as the range union of the temporary partitions to be replaced. When set to false, you only need to ensure that the range between the new formal partitions does not overlap after replacement. + + For List partition, this parameter is always true, and the enumeration values of all full partitions to be replaced must be identical to the enumeration values of the temporary partitions to be replaced. + + Here are some examples: * Example 1 @@ -131,6 +146,38 @@ The replace operation has two special optional parameters: The union of ranges is not the same. If `strict_range` is true, you cannot use tp1 and tp2 to replace p1. If false, and the two partition ranges `[10, 30), [40, 50)` and the other formal partitions do not overlap, they can be replaced. + * Example 3 + + Enumerated values of partitions p1, p2 to be replaced (=> union). + + ``` + (1, 2, 3), (4, 5, 6) => (1, 2, 3, 4, 5, 6) + ``` + + Replace the enumerated values of partitions tp1, tp2, tp3 (=> union). + + ``` + (1, 2, 3), (4), (5, 6) => (1, 2, 3, 4, 5, 6) + ``` + + The enumeration values are the same, you can use tp1, tp2, tp3 to replace p1, p2 + + * Example 4 + + Enumerated values of partitions p1, p2, p3 to be replaced (=> union). + + ``` + (("1", "beijing"), ("1", "shanghai")), (("2", "beijing"), ("2", "shanghai")), (("3", "beijing"), ("3", "shanghai")) => (("1", "beijing"), ("3", "shanghai")) "), ("1", "shanghai"), ("2", "beijing"), ("2", "shanghai"), ("3", "beijing"), ("3", "shanghai")) + ``` + + Replace the enumerated values of partitions tp1, tp2 (=> union). + + ``` + (("1", "beijing"), ("1", "shanghai")), (("2", "beijing"), ("2", "shanghai"), ("3", "beijing"), ("3", "shanghai")) => (("1", "beijing") , ("1", "shanghai"), ("2", "beijing"), ("2", "shanghai"), ("3", "beijing"), ("3", "shanghai")) + ``` + + The enumeration values are the same, you can use tp1, tp2 to replace p1, p2, p3 + 2. `use_temp_partition_name` The default is false. When this parameter is false, and the number of partitions to be replaced is the same as the number of replacement partitions, the name of the formal partition after the replacement remains unchanged. If true, after replacement, the name of the formal partition is the name of the replacement partition. Here are some examples: diff --git a/docs/en/administrator-guide/dynamic-partition.md b/docs/en/administrator-guide/dynamic-partition.md index 1a75491d041b9a..09f3a27af8c7c4 100644 --- a/docs/en/administrator-guide/dynamic-partition.md +++ b/docs/en/administrator-guide/dynamic-partition.md @@ -30,6 +30,8 @@ Dynamic partition is a new feature introduced in Doris version 0.12. It's design At present, the functions of dynamically adding partitions and dynamically deleting partitions are realized. +Dynamic partitioning is only supported for Range partitions. + ## Noun Interpretation * FE: Frontend, the front-end node of Doris. Responsible for metadata management and request access. diff --git a/docs/en/administrator-guide/load-data/delete-manual.md b/docs/en/administrator-guide/load-data/delete-manual.md index 7324a8e4703ff4..8ee465b80c040b 100644 --- a/docs/en/administrator-guide/load-data/delete-manual.md +++ b/docs/en/administrator-guide/load-data/delete-manual.md @@ -67,7 +67,7 @@ Explanation: 2. The column in the WHERE condition can only be the `key` column. 3. Cannot delete when the `key` column does not exist in any rollup table. 4. Each condition in WHERE condition can only be connected by `and`. If you want `or`, you are suggested to write these conditions into two delete statements. -5. If the specified table is a range partitioned table, `PARTITION` must be specified unless the table is a single partition table,. +5. If the specified table is a range or list partitioned table, `PARTITION` must be specified unless the table is a single partition table,. 6. Unlike the insert into command, delete statement cannot specify `label` manually. You can view the concept of `label` in [Insert Into](./insert-into-manual.md) ## Delete Result diff --git a/docs/en/getting-started/best-practice.md b/docs/en/getting-started/best-practice.md index 0d0c76edcf4b8c..930bdb3a8627d4 100644 --- a/docs/en/getting-started/best-practice.md +++ b/docs/en/getting-started/best-practice.md @@ -98,15 +98,22 @@ In the process of using Star Schema, users are advised to use Star Schema to dis ### 1.3 Partitioning and Bucketing -Doris supports two-level partitioned storage. The first layer is RANGE partition and the second layer is HASH bucket. +Doris supports two-level partitioned storage. The first level is partition, which currently supports both RANGE and LIST partition types, and the second layer is HASH bucket. -1.3.1. RANGE Partitioning +1.3.1. Partitioning -The RANGE partition is used to divide data into different intervals, which can be logically understood as dividing the original table into multiple sub-tables. In business, most users will choose to partition on time, which has the following advantages: +Partition is used to divide data into different intervals, which can be logically understood as dividing the original table into multiple sub-tables. Data can be easily managed by partition, for example, to delete data more quickly. + +1.3.1.1. Range Partitioning + +In business, most users will choose to partition on time, which has the following advantages: * Differentiable heat and cold data * Availability of Doris Hierarchical Storage (SSD + SATA) -* Delete data by partition more quickly + +1.3.1.2. List Partitioning + +In business,, users can select cities or other enumeration values for partition. 1.3.2. Hash Bucketing diff --git a/docs/en/getting-started/data-partition.md b/docs/en/getting-started/data-partition.md index a9efe9ac2fdc57..0d41d777b9ffc5 100644 --- a/docs/en/getting-started/data-partition.md +++ b/docs/en/getting-started/data-partition.md @@ -57,7 +57,9 @@ See more help with `HELP CREATE TABLE;`. This section introduces Doris's approach to building tables with an example. ``` -CREATE TABLE IF NOT EXISTS example_db.expamle_tbl +-- Range Partition + +CREATE TABLE IF NOT EXISTS example_db.expamle_range_tbl ( `user_id` LARGEINT NOT NULL COMMENT "user id", `date` DATE NOT NULL COMMENT "Data fill in date time", @@ -86,6 +88,38 @@ PROPERTIES "storage_cooldown_time" = "2018-01-01 12:00:00" ); + +-- List Partition + +CREATE TABLE IF NOT EXISTS example_db.expamle_list_tbl +( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `timestamp` DATETIME NOT NULL COMMENT "数据灌入的时间戳", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间" +) +ENGINE=olap +AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) +PARTITION BY LIST(`city`) +( + PARTITION `p_cn` VALUES IN ("Beijing", "Shanghai", "Hong Kong"), + PARTITION `p_usa` VALUES IN ("New York", "San Francisco"), + PARTITION `p_jp` VALUES IN ("Tokyo") +) +DISTRIBUTED BY HASH(`user_id`) BUCKETS 16 +PROPERTIES +( + "replication_num" = "3", + "storage_medium" = "SSD", + "storage_cooldown_time" = "2018-01-01 12:00:00" +); + ``` ### Column Definition @@ -106,7 +140,7 @@ When defining columns, you can refer to the following suggestions: ### Partitioning and binning -Doris supports two levels of data partitioning. The first layer is Partition, which only supports the division of Range. The second layer is Bucket (Tablet), which only supports the way Hash is divided. +Doris supports two levels of data partitioning. The first layer is Partition, which supports Range and List partitioning. The second layer is the Bucket (Tablet), which only supports Hash partitioning. It is also possible to use only one layer of partitioning. When using a layer partition, only Bucket partitioning is supported. @@ -114,12 +148,15 @@ It is also possible to use only one layer of partitioning. When using a layer pa * The Partition column can specify one or more columns. The partition class must be a KEY column. The use of multi-column partitions is described later in the **Multi-column partitioning** summary.  * Regardless of the type of partition column, double quotes are required when writing partition values. - * Partition columns are usually time columns for easy management of old and new data. * There is no theoretical limit on the number of partitions. * When you do not use Partition to build a table, the system will automatically generate a Partition with the same name as the table name. This Partition is not visible to the user and cannot be modified. + + #### Range Partition + + * Partition columns are usually time columns for easy management of old and new data. * Partition supports only the upper bound by `VALUES LESS THAN (...)`, the system will use the upper bound of the previous partition as the lower bound of the partition, and generate a left closed right open interval. Passing, also supports specifying the upper and lower bounds by `VALUES [...)`, and generating a left closed right open interval. * It is easier to understand by specifying `VALUES [...)`. Here is an example of the change in partition range when adding or deleting partitions using the `VALUES LESS THAN (...)` statement: - * As the example above, when the table is built, the following 3 partitions are automatically generated: + * As in the `example_range_tbl` example above, when the table is built, the following 3 partitions are automatically generated: ``` P201701: [MIN_VALUE, 2017-02-01) P201702: [2017-02-01, 2017-03-01) @@ -176,6 +213,39 @@ It is also possible to use only one layer of partitioning. When using a layer pa You cannot add partitions with overlapping ranges. + #### List Partition + + * The partition column supports the `BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, LARGEINT, DATE, DATETIME, CHAR, VARCHAR` data type, and the partition value is an enumeration value. Partitions can be hit only if the data is one of the target partition enumeration values. + * Partition supports specifying the number of partitions contained in each partition via `VALUES IN (...) ` to specify the enumeration values contained in each partition. + * The following example illustrates how partitions change when adding or deleting partitions. + + * As in the `example_list_tbl` example above, when the table is built, the following three partitions are automatically created. + + ``` + p_cn: ("Beijing", "Shanghai", "Hong Kong") + p_usa: ("New York", "San Francisco") + p_jp: ("Tokyo") + ``` + + * When we add a partition p_uk VALUES IN ("London"), the result of the partition is as follows + + ``` + p_cn: ("Beijing", "Shanghai", "Hong Kong") + p_usa: ("New York", "San Francisco") + p_jp: ("Tokyo") + p_uk: ("London") + ``` + + * When we delete the partition p_jp, the result of the partition is as follows. + + ``` + p_cn: ("Beijing", "Shanghai", "Hong Kong") + p_usa: ("New York", "San Francisco") + p_uk: ("London") + ``` + + You cannot add partitions with overlapping ranges. + 2. Bucket * If a Partition is used, the `DISTRIBUTED ...` statement describes the division rules for the data in each partition. If you do not use Partition, it describes the rules for dividing the data of the entire table. @@ -203,37 +273,70 @@ It is also possible to use only one layer of partitioning. When using a layer pa Doris supports specifying multiple columns as partition columns, examples are as follows: -``` -PARTITION BY RANGE(`date`, `id`) -( - PARTITION `p201701_1000` VALUES LESS THAN ("2017-02-01", "1000"), - PARTITION `p201702_2000` VALUES LESS THAN ("2017-03-01", "2000"), - PARTITION `p201703_all` VALUES LESS THAN ("2017-04-01") -) -``` - -In the above example, we specify `date` (DATE type) and `id` (INT type) as partition columns. The resulting partitions in the above example are as follows: - -``` -*p201701_1000: [(MIN_VALUE, MIN_VALUE), ("2017-02-01", "1000") ) -*p201702_2000: [("2017-02-01", "1000"), ("2017-03-01", "2000") ) -*p201703_all: [("2017-03-01", "2000"), ("2017-04-01", MIN_VALUE)) -``` - -Note that the last partition user defaults only the partition value of the `date` column, so the partition value of the `id` column will be filled with `MIN_VALUE` by default. When the user inserts data, the partition column values ​​are compared in order, and the corresponding partition is finally obtained. Examples are as follows: - -``` -* Data --> Partition -* 2017-01-01, 200 --> p201701_1000 -* 2017-01-01, 2000 --> p201701_1000 -* 2017-02-01, 100 --> p201701_1000 -* 2017-02-01, 2000 --> p201702_2000 -* 2017-02-15, 5000 --> p201702_2000 -* 2017-03-01, 2000 --> p201703_all -* 2017-03-10, 1 --> p201703_all -* 2017-04-01, 1000 --> Unable to import -* 2017-05-01, 1000 --> Unable to import -``` +##### Range Partition + + ``` + PARTITION BY RANGE(`date`, `id`) + ( + PARTITION `p201701_1000` VALUES LESS THAN ("2017-02-01", "1000"), + PARTITION `p201702_2000` VALUES LESS THAN ("2017-03-01", "2000"), + PARTITION `p201703_all` VALUES LESS THAN ("2017-04-01") + ) + ``` + + In the above example, we specify `date` (DATE type) and `id` (INT type) as partition columns. The resulting partitions in the above example are as follows: + + ``` + *p201701_1000: [(MIN_VALUE, MIN_VALUE), ("2017-02-01", "1000") ) + *p201702_2000: [("2017-02-01", "1000"), ("2017-03-01", "2000") ) + *p201703_all: [("2017-03-01", "2000"), ("2017-04-01", MIN_VALUE)) + ``` + + Note that the last partition user defaults only the partition value of the `date` column, so the partition value of the `id` column will be filled with `MIN_VALUE` by default. When the user inserts data, the partition column values ​​are compared in order, and the corresponding partition is finally obtained. Examples are as follows: + + ``` + * Data --> Partition + * 2017-01-01, 200 --> p201701_1000 + * 2017-01-01, 2000 --> p201701_1000 + * 2017-02-01, 100 --> p201701_1000 + * 2017-02-01, 2000 --> p201702_2000 + * 2017-02-15, 5000 --> p201702_2000 + * 2017-03-01, 2000 --> p201703_all + * 2017-03-10, 1 --> p201703_all + * 2017-04-01, 1000 --> Unable to import + * 2017-05-01, 1000 --> Unable to import + ``` + +##### List Partition + + ``` + PARTITION BY LIST(`id`, `city`) + ( + PARTITION `p1_city` VALUES IN (("1", "Beijing"), ("1", "Shanghai")), + PARTITION `p2_city` VALUES IN (("2", "Beijing"), ("2", "Shanghai")), + PARTITION `p3_city` VALUES IN (("3", "Beijing"), ("3", "Shanghai")) + ) + ``` + + In the above example, we specify `id`(INT type) and `city`(VARCHAR type) as partition columns. The above example ends up with the following partitions. + + ``` + * p1_city: [("1", "Beijing"), ("1", "Shanghai")] + * p2_city: [("2", "Beijing"), ("2", "Shanghai")] + * p3_city: [("3", "Beijing"), ("3", "Shanghai")] + ``` + + When the user inserts data, the partition column values will be compared sequentially in order to finally get the corresponding partition. An example is as follows. + + ``` + * Data ---> Partition + * 1, Beijing ---> p1_city + * 1, Shanghai ---> p1_city + * 2, Shanghai ---> p2_city + * 3, Beijing ---> p3_city + * 1, Tianjin ---> Unable to import + * 4, Beijing ---> Unable to import + ``` ### PROPERTIES diff --git a/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md index c03a1420f4f235..56e22d94060be1 100644 --- a/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md +++ b/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md @@ -170,39 +170,83 @@ Syntax: NOTICE: Except for AGGREGATE KEY, no need to specify aggregation type for value columns. 5. partition_desc - Partition has two ways to use: - 1) LESS THAN - Syntax: - - ``` - PARTITION BY RANGE (k1, k2, ...) - ( - PARTITION partition_name1 VALUES LESS THAN MAXVALUE|("value1", "value2", ...), - PARTITION partition_name2 VALUES LESS THAN MAXVALUE|("value1", "value2", ...) - ... - ) - ``` + Currently, both RANGE and LIST partitioning methods are supported. + 5.1 RANGE partition + RANGE Partition has two ways to use: + 1) LESS THAN + Syntax: + + ``` + PARTITION BY RANGE (k1, k2, ...) + ( + PARTITION partition_name1 VALUES LESS THAN MAXVALUE|("value1", "value2", ...), + PARTITION partition_name2 VALUES LESS THAN MAXVALUE|("value1", "value2", ...) + ... + ) + ``` + + Explain: + Use the specified key column and the specified range of values for partitioning. + 1) Partition name only support [A-z0-9_] + 2) Partition key column's type should be: + TINYINT, SMALLINT, INT, BIGINT, LARGEINT, DATE, DATETIME + 3) The range is [closed, open). And the lower bound of first partition is MIN VALUE of specified column type. + 4) NULL values should be save in partition which includes MIN VALUE. + 5) Support multi partition columns, the the default partition value is MIN VALUE. + 2)Fixed Range + Syntax: + ``` + PARTITION BY RANGE (k1, k2, k3, ...) + ( + PARTITION partition_name1 VALUES [("k1-lower1", "k2-lower1", "k3-lower1",...), ("k1-upper1", "k2-upper1", "k3-upper1", ...)), + PARTITION partition_name2 VALUES [("k1-lower1-2", "k2-lower1-2", ...), ("k1-upper1-2", MAXVALUE, )) + "k3-upper1-2", ... + ) + ``` + Explain: + 1)The Fixed Range is more flexible than the LESS THAN, and the left and right intervals are completely determined by the user. + 2)Others are consistent with LESS THAN. + + 5.2 LIST partition + LIST partition is divided into single column partition and multi-column partition + 1) Single column partition + Syntax. + + ``` + PARTITION BY LIST(k1) + ( + PARTITION partition_name1 VALUES IN ("value1", "value2", ...) , + PARTITION partition_name2 VALUES IN ("value1", "value2", ...) + ... + ) + ``` + + Explain: + Use the specified key column and the formulated enumeration value for partitioning. + 1) Partition name only support [A-z0-9_] + 2) Partition key column's type should be: + BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, LARGEINT, DATE, DATETIME, CHAR, VARCHAR + 3) Partition is a collection of enumerated values, partition values cannot be duplicated between partitions + 4) NULL values cannot be imported + 5) partition values cannot be defaulted, at least one must be specified + + 2) Multi-column partition + Syntax. + + ``` + PARTITION BY LIST(k1, k2) + ( + PARTITION partition_name1 VALUES IN (("value1", "value2"), ("value1", "value2"), ...) , + PARTITION partition_name2 VALUES IN (("value1", "value2"), ("value1", "value2"), ...) + ... + ) + ``` + + Explain: + 1) the partition of a multi-column partition is a collection of tuple enumeration values + 2) The number of tuple values per partition must be equal to the number of columns in the partition + 3) The other partitions are synchronized with the single column partition - Explain: - 1) Partition name only support [A-z0-9_] - 2) Partition key column's type should be: - TINYINT, SMALLINT, INT, BIGINT, LARGEINT, DATE, DATETIME - 3) The range is [closed, open). And the lower bound of first partition is MIN VALUE of specified column type. - 4) NULL values should be save in partition which includes MIN VALUE. - 5) Support multi partition columns, the the default partition value is MIN VALUE. - 2)Fixed Range - Syntax: - ``` - PARTITION BY RANGE (k1, k2, k3, ...) - ( - PARTITION partition_name1 VALUES [("k1-lower1", "k2-lower1", "k3-lower1",...), ("k1-upper1", "k2-upper1", "k3-upper1", ...)), - PARTITION partition_name2 VALUES [("k1-lower1-2", "k2-lower1-2", ...), ("k1-upper1-2", MAXVALUE, )) - "k3-upper1-2", ... - ) - ``` - Explain: - 1)The Fixed Range is more flexible than the LESS THAN, and the left and right intervals are completely determined by the user. - 2)Others are consistent with LESS THAN. 6. distribution_desc 1) Hash Syntax: @@ -246,7 +290,7 @@ Syntax: ) ``` - 4) if you want to use the dynamic partitioning feature, specify it in properties + 4) if you want to use the dynamic partitioning feature, specify it in properties. Note: Dynamic partitioning only supports RANGE partitions ``` PROPERTIES ( @@ -269,7 +313,7 @@ Syntax: Dynamic_partition. Buckets: specifies the number of partition buckets that are automatically created ``` -8. rollup_index + 5) You can create multiple Rollups in bulk when building a table grammar: ``` ROLLUP (rollup_name (column_name1, column_name2, ...) @@ -277,7 +321,7 @@ Syntax: [PROPERTIES ("key"="value", ...)],...) ``` - 5) if you want to use the inmemory table feature, specify it in properties + 6) if you want to use the inmemory table feature, specify it in properties ``` PROPERTIES ( @@ -323,7 +367,7 @@ Syntax: ); ``` -3. Create an olap table, with range partitioned, distributed by hash. +3. Create an olap table, with range partitioned, distributed by hash. Records with the same key exist at the same time, set the initial storage medium and cooling time, use default column storage. 1) LESS THAN @@ -381,9 +425,83 @@ Syntax: PROPERTIES( "storage_medium" = "SSD" ); + +4. Create an olap table, with list partitioned, distributed by hash. Records with the same key exist at the same time, set the initial storage medium and cooling time, use default column storage. + + 1) Single column partition + + ``` + CREATE TABLE example_db.table_list + ( + k1 INT, + k2 VARCHAR(128), + k3 SMALLINT, + v1 VARCHAR(2048), + v2 DATETIME DEFAULT "2014-02-04 15:36:00" + ) + ENGINE=olap + DUPLICATE KEY(k1, k2, k3) + PARTITION BY LIST (k1) + ( + PARTITION p1 VALUES IN ("1", "2", "3"), + PARTITION p2 VALUES IN ("4", "5", "6"), + PARTITION p3 VALUES IN ("7", "8", "9") + ) + DISTRIBUTED BY HASH(k2) BUCKETS 32 + PROPERTIES( + "storage_medium" = "SSD", "storage_cooldown_time" = "2022-06-04 00:00:00" + ); + ``` + + Explain: + This statement will divide the data into 3 partitions as follows. + + ``` + ("1", "2", "3") + ("4", "5", "6") + ("7", "8", "9") + ``` + + Data that does not fall within these partition enumeration values will be filtered as illegal data + + 2) Multi-column partition + + ``` + CREATE TABLE example_db.table_list + ( + k1 INT, + k2 VARCHAR(128), + k3 SMALLINT, + v1 VARCHAR(2048), + v2 DATETIME DEFAULT "2014-02-04 15:36:00" + ) + ENGINE=olap + DUPLICATE KEY(k1, k2, k3) + PARTITION BY LIST (k1, k2) + ( + PARTITION p1 VALUES IN (("1", "beijing"), ("1", "shanghai")), + PARTITION p2 VALUES IN (("2", "beijing"), ("2", "shanghai")), + PARTITION p3 VALUES IN (("3", "beijing"), ("3", "shanghai")) + ) + DISTRIBUTED BY HASH(k2) BUCKETS 32 + PROPERTIES( + "storage_medium" = "SSD", "storage_cooldown_time" = "2022-06-04 00:00:00" + ); + ``` + + Explain: + This statement will divide the data into 3 partitions as follows. + + ``` + (("1", "beijing"), ("1", "shanghai")) + (("2", "beijing"), ("2", "shanghai")) + (("3", "beijing"), ("3", "shanghai")) + ``` + + Data that is not within these partition enumeration values will be filtered as illegal data -4. Create a mysql table - 4.1 Create MySQL table directly from external table information +5. Create a mysql table + 5.1 Create MySQL table directly from external table information ``` CREATE EXTERNAL TABLE example_db.table_mysql ( @@ -405,7 +523,7 @@ Syntax: ) ``` - 4.2 Create MySQL table with external ODBC catalog resource + 5.2 Create MySQL table with external ODBC catalog resource ``` CREATE EXTERNAL RESOURCE "mysql_resource" PROPERTIES @@ -435,7 +553,7 @@ Syntax: ) ``` -5. Create a broker table, with file on HDFS, line delimit by "|", column separated by "\n" +6. Create a broker table, with file on HDFS, line delimit by "|", column separated by "\n" ``` CREATE EXTERNAL TABLE example_db.table_broker ( @@ -458,7 +576,7 @@ Syntax: ); ``` -6. Create table will HLL column +7. Create table will HLL column ``` CREATE TABLE example_db.example_table @@ -473,7 +591,7 @@ Syntax: DISTRIBUTED BY HASH(k1) BUCKETS 32; ``` -7. Create a table will BITMAP_UNION column +8. Create a table will BITMAP_UNION column ``` CREATE TABLE example_db.example_table @@ -488,7 +606,7 @@ Syntax: DISTRIBUTED BY HASH(k1) BUCKETS 32; ``` -8. Create 2 colocate join table. +9. Create 2 colocate join table. ``` CREATE TABLE `t1` ( @@ -511,7 +629,7 @@ Syntax: ); ``` -9. Create a broker table, with file on BOS. +10. Create a broker table, with file on BOS. ``` CREATE EXTERNAL TABLE example_db.table_broker ( @@ -529,7 +647,7 @@ Syntax: ); ``` -10. Create a table with a bitmap index +11. Create a table with a bitmap index ``` CREATE TABLE example_db.table_hash @@ -547,7 +665,7 @@ Syntax: PROPERTIES ("storage_type"="column"); ``` -11. Create a dynamic partitioning table (dynamic partitioning needs to be enabled in FE configuration), which creates partitions 3 days in advance every day. For example, if today is' 2020-01-08 ', partitions named 'p20200108', 'p20200109', 'p20200110', 'p20200111' will be created. +12. Create a dynamic partitioning table (dynamic partitioning needs to be enabled in FE configuration), which creates partitions 3 days in advance every day. For example, if today is' 2020-01-08 ', partitions named 'p20200108', 'p20200109', 'p20200110', 'p20200111' will be created. ``` [types: [DATE]; keys: [2020-01-08]; ‥types: [DATE]; keys: [2020-01-09]; ) @@ -582,7 +700,7 @@ Syntax: "dynamic_partition.buckets" = "32" ); ``` -12. Create a table with rollup index +13. Create a table with rollup index ``` CREATE TABLE example_db.rolup_index_table ( @@ -602,7 +720,7 @@ Syntax: PROPERTIES("replication_num" = "3"); ``` -13. Create a inmemory table: +14. Create a inmemory table: ``` CREATE TABLE example_db.table_hash @@ -620,7 +738,7 @@ Syntax: PROPERTIES ("in_memory"="true"); ``` -14. Create a hive external table +15. Create a hive external table ``` CREATE TABLE example_db.table_hive ( diff --git a/docs/en/sql-reference/sql-statements/Data Definition/RESTORE.md b/docs/en/sql-reference/sql-statements/Data Definition/RESTORE.md index 0a467d74ebfaf0..7df04c4cbbea8d 100644 --- a/docs/en/sql-reference/sql-statements/Data Definition/RESTORE.md +++ b/docs/en/sql-reference/sql-statements/Data Definition/RESTORE.md @@ -42,7 +42,7 @@ Explain: 2. The ON clause identifies the tables and partitions that need to be restored. If no partition is specified, all partitions of the table are restored by default. The specified tables and partitions must already exist in the warehouse backup. 3. The backup tables in the warehouse can be restored to new tables through AS statements. But the new table name cannot already exist in the database. Partition name cannot be changed. 4. The backup tables in the warehouse can be restored and replaced with the same-name tables in the database, but the table structure of the two tables must be completely consistent. Table structure includes: table name, column, partition, Rollup and so on. -5. Partitions of the recovery table can be specified, and the system checks whether the partition Range matches. +5. Partitions of the recovery table can be specified, and the system checks whether the partition Range or List matches. 6. PROPERTIES currently supports the following attributes: "Backup_timestamp" = "2018-05-04-16-45-08": specifies which version of the time to restore the corresponding backup must be filled in. This information can be obtained through the `SHOW SNAPSHOT ON repo;'statement. "Replication_num" = "3": Specifies the number of replicas of the restored table or partition. The default is 3. If an existing table or partition is restored, the number of copies must be the same as the number of copies of an existing table or partition. At the same time, there must be enough hosts to accommodate multiple copies. diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/DELETE.md b/docs/en/sql-reference/sql-statements/Data Manipulation/DELETE.md index a0ef240766580d..be4c606cb31ce5 100644 --- a/docs/en/sql-reference/sql-statements/Data Manipulation/DELETE.md +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/DELETE.md @@ -40,7 +40,7 @@ under the License. 2) When the selected key column does not exist in a rollup, delete cannot be performed. 3) The relationship between conditions can only be "and". If you want to achieve the "or" relationship, you need to divide the conditions into two DELETE statements. - 4) If it is a RANGE partitioned table, you can specify the partition. If not specified, and the session variable delete_without_partition is true, it will be applied to all partitions. If it is a single partition table, you do not need to specify it. + 4) If it is a partitioned table, you can specify the partition. If not specified, and the session variable delete_without_partition is true, it will be applied to all partitions. If it is a single partition table, you do not need to specify it. Notice: This statement may reduce query efficiency for a period of time after execution. diff --git a/docs/zh-CN/administrator-guide/alter-table/alter-table-temp-partition.md b/docs/zh-CN/administrator-guide/alter-table/alter-table-temp-partition.md index a9493bda211626..b46759642f2340 100644 --- a/docs/zh-CN/administrator-guide/alter-table/alter-table-temp-partition.md +++ b/docs/zh-CN/administrator-guide/alter-table/alter-table-temp-partition.md @@ -52,6 +52,15 @@ ALTER TABLE tbl2 ADD TEMPORARY PARTITION tp1 VALUES [("2020-01-01"), ("2020-02-0 ALTER TABLE tbl1 ADD TEMPORARY PARTITION tp1 VALUES LESS THAN("2020-02-01") ("in_memory" = "true", "replication_num" = "1") DISTRIBUTED BY HASH(k1) BUCKETS 5; + +ALTER TABLE tbl3 ADD TEMPORARY PARTITION tp1 VALUES IN ("Beijing", "Shanghai"); + +ALTER TABLE tbl4 ADD TEMPORARY PARTITION tp1 VALUES IN ((1, "Beijing"), (1, "Shanghai")); + +ALTER TABLE tbl3 ADD TEMPORARY PARTITION tp1 VALUES IN ("Beijing", "Shanghai") +("in_memory" = "true", "replication_num" = "1") +DISTRIBUTED BY HASH(k1) BUCKETS 5; + ``` 通过 `HELP ALTER TABLE;` 查看更多帮助和示例。 @@ -97,7 +106,13 @@ PROPERTIES ( 1. `strict_range` - 默认为 true。当该参数为 true 时,表示要被替换的所有正式分区的范围并集需要和替换的临时分区的范围并集完全相同。当置为 false 时,只需要保证替换后,新的正式分区间的范围不重叠即可。下面举例说明: + 默认为 true。 + + 对于 Range 分区,当该参数为 true 时,表示要被替换的所有正式分区的范围并集需要和替换的临时分区的范围并集完全相同。当置为 false 时,只需要保证替换后,新的正式分区间的范围不重叠即可。 + + 对于 List 分区,该参数恒为 true。要被替换的所有正式分区的枚举值必须和替换的临时分区枚举值完全相同。 + + 下面举例说明: * 示例1 @@ -131,6 +146,38 @@ PROPERTIES ( 范围并集不相同,如果 `strict_range` 为 true,则不可以使用 tp1 和 tp2 替换 p1。如果为 false,且替换后的两个分区范围 `[10, 30), [40, 50)` 和其他正式分区不重叠,则可以替换。 + * 示例3 + + 待替换的分区 p1, p2 的枚举值(=> 并集): + + ``` + (1, 2, 3), (4, 5, 6) => (1, 2, 3, 4, 5, 6) + ``` + + 替换分区 tp1, tp2, tp3 的枚举值(=> 并集): + + ``` + (1, 2, 3), (4), (5, 6) => (1, 2, 3, 4, 5, 6) + ``` + + 枚举值并集相同,可以使用 tp1,tp2,tp3 替换 p1,p2 + + * 示例4 + + 待替换的分区 p1, p2,p3 的枚举值(=> 并集): + + ``` + (("1","beijing"), ("1", "shanghai")), (("2","beijing"), ("2", "shanghai")), (("3","beijing"), ("3", "shanghai")) => (("1","beijing"), ("1", "shanghai"), ("2","beijing"), ("2", "shanghai"), ("3","beijing"), ("3", "shanghai")) + ``` + + 替换分区 tp1, tp2 的枚举值(=> 并集): + + ``` + (("1","beijing"), ("1", "shanghai")), (("2","beijing"), ("2", "shanghai"), ("3","beijing"), ("3", "shanghai")) => (("1","beijing"), ("1", "shanghai"), ("2","beijing"), ("2", "shanghai"), ("3","beijing"), ("3", "shanghai")) + ``` + + 枚举值并集相同,可以使用 tp1,tp2 替换 p1,p2,p3 + 2. `use_temp_partition_name` 默认为 false。当该参数为 false,并且待替换的分区和替换分区的个数相同时,则替换后的正式分区名称维持不变。如果为 true,则替换后,正式分区的名称为替换分区的名称。下面举例说明: diff --git a/docs/zh-CN/administrator-guide/dynamic-partition.md b/docs/zh-CN/administrator-guide/dynamic-partition.md index 2e84276b2daa0b..dfbc085ed35835 100644 --- a/docs/zh-CN/administrator-guide/dynamic-partition.md +++ b/docs/zh-CN/administrator-guide/dynamic-partition.md @@ -30,6 +30,8 @@ under the License. 目前实现了动态添加分区及动态删除分区的功能。 +动态分区只支持 Range 分区。 + ## 名词解释 * FE:Frontend,Doris 的前端节点。负责元数据管理和请求接入。 diff --git a/docs/zh-CN/administrator-guide/load-data/delete-manual.md b/docs/zh-CN/administrator-guide/load-data/delete-manual.md index ac48e87f1f4d67..0f460b9998b343 100644 --- a/docs/zh-CN/administrator-guide/load-data/delete-manual.md +++ b/docs/zh-CN/administrator-guide/load-data/delete-manual.md @@ -66,7 +66,7 @@ DELETE FROM my_table PARTITION p1 WHERE k1 < 3 AND k2 = "abc"; 2. `Where`语句中的列只能是`key`列 3. 当选定的`key`列不存在某个rollup表内时,无法进行delete 4. 条件语句中各个条件只能是`and`关系,如希望达成`or`可将条件分别写入两个delete语句中 -5. 如果指定表为RANGE分区表,则必须指定 `PARTITION`。如果是单分区表,可以不指定。 +5. 如果指定表为RANGE或者LIST分区表,则必须指定 `PARTITION`。如果是单分区表,可以不指定。 6. 不同于Insert into命令,delete不能手动指定`label`,有关label的概念可以查看[Insert Into文档](./insert-into-manual.md) ## 返回结果 diff --git a/docs/zh-CN/getting-started/best-practice.md b/docs/zh-CN/getting-started/best-practice.md index c4e2a4fce7a295..a51adfa88b230c 100644 --- a/docs/zh-CN/getting-started/best-practice.md +++ b/docs/zh-CN/getting-started/best-practice.md @@ -97,15 +97,23 @@ Doris 数据模型上目前分为三类: AGGREGATE KEY, UNIQUE KEY, DUPLICATE KE ### 1.3 分区和分桶 -Doris 支持两级分区存储, 第一层为 RANGE 分区(partition), 第二层为 HASH 分桶(bucket)。 +Doris 支持两级分区存储, 第一层为分区(partition),目前支持 RANGE 分区和 LIST 分区两种类型, 第二层为 HASH 分桶(bucket)。 -1.3.1. RANGE分区(partition) +1.3.1. 分区(partition) - RANGE分区用于将数据划分成不同区间, 逻辑上可以理解为将原始表划分成了多个子表。业务上,多数用户会选择采用按时间进行partition, 让时间进行partition有以下好处: + 分区用于将数据划分成不同区间, 逻辑上可以理解为将原始表划分成了多个子表。可以方便的按分区对数据进行管理,例如,删除数据时,更加迅速。 + +1.3.1.1. RANGE分区 + + 业务上,多数用户会选择采用按时间进行partition, 让时间进行partition有以下好处: * 可区分冷热数据 * 可用上Doris分级存储(SSD + SATA)的功能 - * 按分区删除数据时,更加迅速 + +1.3.1.2. LIST分区 + + 业务上,用户可以选择城市或者其他枚举值进行partition。 + 1.3.2. HASH分桶(bucket) diff --git a/docs/zh-CN/getting-started/data-partition.md b/docs/zh-CN/getting-started/data-partition.md index bce0ec550c5209..14f6073a1ac544 100644 --- a/docs/zh-CN/getting-started/data-partition.md +++ b/docs/zh-CN/getting-started/data-partition.md @@ -57,7 +57,9 @@ Doris 的建表是一个同步命令,命令返回成功,即表示建表成 本小节通过一个例子,来介绍 Doris 的建表方式。 ``` -CREATE TABLE IF NOT EXISTS example_db.expamle_tbl +-- Range Partition + +CREATE TABLE IF NOT EXISTS example_db.expamle_range_tbl ( `user_id` LARGEINT NOT NULL COMMENT "用户id", `date` DATE NOT NULL COMMENT "数据灌入日期时间", @@ -86,6 +88,38 @@ PROPERTIES "storage_cooldown_time" = "2018-01-01 12:00:00" ); + +-- List Partition + +CREATE TABLE IF NOT EXISTS example_db.expamle_list_tbl +( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `timestamp` DATETIME NOT NULL COMMENT "数据灌入的时间戳", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间" +) +ENGINE=olap +AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) +PARTITION BY LIST(`city`) +( + PARTITION `p_cn` VALUES IN ("Beijing", "Shanghai", "Hong Kong"), + PARTITION `p_usa` VALUES IN ("New York", "San Francisco"), + PARTITION `p_jp` VALUES IN ("Tokyo") +) +DISTRIBUTED BY HASH(`user_id`) BUCKETS 16 +PROPERTIES +( + "replication_num" = "3", + "storage_medium" = "SSD", + "storage_cooldown_time" = "2018-01-01 12:00:00" +); + ``` ### 列定义 @@ -106,7 +140,7 @@ AGGREGATE KEY 数据模型中,所有没有指定聚合方式(SUM、REPLACE ### 分区与分桶 -Doris 支持两层的数据划分。第一层是 Partition,仅支持 Range 的划分方式。第二层是 Bucket(Tablet),仅支持 Hash 的划分方式。 +Doris 支持两层的数据划分。第一层是 Partition,支持 Range 和 List 的划分方式。第二层是 Bucket(Tablet),仅支持 Hash 的划分方式。 也可以仅使用一层分区。使用一层分区时,只支持 Bucket 划分。 @@ -114,14 +148,17 @@ Doris 支持两层的数据划分。第一层是 Partition,仅支持 Range 的 * Partition 列可以指定一列或多列。分区类必须为 KEY 列。多列分区的使用方式在后面 **多列分区** 小结介绍。 * 不论分区列是什么类型,在写分区值时,都需要加双引号。 - * 分区列通常为时间列,以方便的管理新旧数据。 * 分区数量理论上没有上限。 * 当不使用 Partition 建表时,系统会自动生成一个和表名同名的,全值范围的 Partition。该 Partition 对用户不可见,并且不可删改。 + + #### Range 分区 + + * 分区列通常为时间列,以方便的管理新旧数据。 * Partition 支持通过 `VALUES LESS THAN (...)` 仅指定上界,系统会将前一个分区的上界作为该分区的下界,生成一个左闭右开的区间。通过,也支持通过 `VALUES [...)` 指定同时指定上下界,生成一个左闭右开的区间。 * 通过 `VALUES [...)` 同时指定上下界比较容易理解。这里举例说明,当使用 `VALUES LESS THAN (...)` 语句进行分区的增删操作时,分区范围的变化情况: - * 如上示例,当建表完成后,会自动生成如下3个分区: + * 如上 `expamle_range_tbl` 示例,当建表完成后,会自动生成如下3个分区: ``` p201701: [MIN_VALUE, 2017-02-01) @@ -180,6 +217,39 @@ Doris 支持两层的数据划分。第一层是 Partition,仅支持 Range 的 不可添加范围重叠的分区。 + #### List 分区 + + * 分区列支持 `BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, LARGEINT, DATE, DATETIME, CHAR, VARCHAR` 数据类型,分区值为枚举值。只有当数据为目标分区枚举值其中之一时,才可以命中分区。 + * Partition 支持通过 `VALUES IN (...)` 来指定每个分区包含的枚举值。 + * 下面通过示例说明,进行分区的增删操作时,分区的变化。 + + * 如上 `example_list_tbl` 示例,当建表完成后,会自动生成如下3个分区: + + ``` + p_cn: ("Beijing", "Shanghai", "Hong Kong") + p_usa: ("New York", "San Francisco") + p_jp: ("Tokyo") + ``` + + * 当我们增加一个分区 p_uk VALUES IN ("London"),分区结果如下: + + ``` + p_cn: ("Beijing", "Shanghai", "Hong Kong") + p_usa: ("New York", "San Francisco") + p_jp: ("Tokyo") + p_uk: ("London") + ``` + + * 当我们删除分区 p_jp,分区结果如下: + + ``` + p_cn: ("Beijing", "Shanghai", "Hong Kong") + p_usa: ("New York", "San Francisco") + p_uk: ("London") + ``` + + 不可添加范围重叠的分区。 + 2. Bucket * 如果使用了 Partition,则 `DISTRIBUTED ...` 语句描述的是数据在**各个分区内**的划分规则。如果不使用 Partition,则描述的是对整个表的数据的划分规则。 @@ -206,37 +276,70 @@ Doris 支持两层的数据划分。第一层是 Partition,仅支持 Range 的 Doris 支持指定多列作为分区列,示例如下: -``` -PARTITION BY RANGE(`date`, `id`) -( - PARTITION `p201701_1000` VALUES LESS THAN ("2017-02-01", "1000"), - PARTITION `p201702_2000` VALUES LESS THAN ("2017-03-01", "2000"), - PARTITION `p201703_all` VALUES LESS THAN ("2017-04-01") -) -``` - -在以上示例中,我们指定 `date`(DATE 类型) 和 `id`(INT 类型) 作为分区列。以上示例最终得到的分区如下: - -``` -* p201701_1000: [(MIN_VALUE, MIN_VALUE), ("2017-02-01", "1000") ) -* p201702_2000: [("2017-02-01", "1000"), ("2017-03-01", "2000") ) -* p201703_all: [("2017-03-01", "2000"), ("2017-04-01", MIN_VALUE)) -``` - -注意,最后一个分区用户缺省只指定了 `date` 列的分区值,所以 `id` 列的分区值会默认填充 `MIN_VALUE`。当用户插入数据时,分区列值会按照顺序依次比较,最终得到对应的分区。举例如下: - -``` -* 数据 --> 分区 -* 2017-01-01, 200 --> p201701_1000 -* 2017-01-01, 2000 --> p201701_1000 -* 2017-02-01, 100 --> p201701_1000 -* 2017-02-01, 2000 --> p201702_2000 -* 2017-02-15, 5000 --> p201702_2000 -* 2017-03-01, 2000 --> p201703_all -* 2017-03-10, 1 --> p201703_all -* 2017-04-01, 1000 --> 无法导入 -* 2017-05-01, 1000 --> 无法导入 -``` +##### Range 分区 + + ``` + PARTITION BY RANGE(`date`, `id`) + ( + PARTITION `p201701_1000` VALUES LESS THAN ("2017-02-01", "1000"), + PARTITION `p201702_2000` VALUES LESS THAN ("2017-03-01", "2000"), + PARTITION `p201703_all` VALUES LESS THAN ("2017-04-01") + ) + ``` + + 在以上示例中,我们指定 `date`(DATE 类型) 和 `id`(INT 类型) 作为分区列。以上示例最终得到的分区如下: + + ``` + * p201701_1000: [(MIN_VALUE, MIN_VALUE), ("2017-02-01", "1000") ) + * p201702_2000: [("2017-02-01", "1000"), ("2017-03-01", "2000") ) + * p201703_all: [("2017-03-01", "2000"), ("2017-04-01", MIN_VALUE)) + ``` + + 注意,最后一个分区用户缺省只指定了 `date` 列的分区值,所以 `id` 列的分区值会默认填充 `MIN_VALUE`。当用户插入数据时,分区列值会按照顺序依次比较,最终得到对应的分区。举例如下: + + ``` + * 数据 --> 分区 + * 2017-01-01, 200 --> p201701_1000 + * 2017-01-01, 2000 --> p201701_1000 + * 2017-02-01, 100 --> p201701_1000 + * 2017-02-01, 2000 --> p201702_2000 + * 2017-02-15, 5000 --> p201702_2000 + * 2017-03-01, 2000 --> p201703_all + * 2017-03-10, 1 --> p201703_all + * 2017-04-01, 1000 --> 无法导入 + * 2017-05-01, 1000 --> 无法导入 + ``` + +##### List 分区 + + ``` + PARTITION BY LIST(`id`, `city`) + ( + PARTITION `p1_city` VALUES IN (("1", "Beijing"), ("1", "Shanghai")), + PARTITION `p2_city` VALUES IN (("2", "Beijing"), ("2", "Shanghai")), + PARTITION `p3_city` VALUES IN (("3", "Beijing"), ("3", "Shanghai")) + ) + ``` + + 在以上示例中,我们指定 `id`(INT 类型) 和 `city`(VARCHAR 类型) 作为分区列。以上示例最终得到的分区如下: + + ``` + * p1_city: [("1", "Beijing"), ("1", "Shanghai")] + * p2_city: [("2", "Beijing"), ("2", "Shanghai")] + * p3_city: [("3", "Beijing"), ("3", "Shanghai")] + ``` + + 当用户插入数据时,分区列值会按照顺序依次比较,最终得到对应的分区。举例如下: + + ``` + * 数据 ---> 分区 + * 1, Beijing ---> p1_city + * 1, Shanghai ---> p1_city + * 2, Shanghai ---> p2_city + * 3, Beijing ---> p3_city + * 1, Tianjin ---> 无法导入 + * 4, Beijing ---> 无法导入 + ``` ### PROPERTIES diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md index 7103bc2be0883c..28d1e2fe250487 100644 --- a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md @@ -164,7 +164,7 @@ under the License. 其中 database 是 hive 表对应的库名字,table 是 hive 表的名字,hive.metastore.uris 是 hive metastore 服务地址。 注意:目前hive外部表仅用于Spark Load使用,不支持查询。 -1. key_desc +4. key_desc 语法: `key_type(k1[,k2 ...])` 说明: @@ -180,48 +180,90 @@ under the License. 注意: 除AGGREGATE KEY外,其他key_type在建表时,value列不需要指定聚合类型。 -2. partition_desc - partition描述有两种使用方式 - 1) LESS THAN - 语法: - - ``` - PARTITION BY RANGE (k1, k2, ...) - ( - PARTITION partition_name1 VALUES LESS THAN MAXVALUE|("value1", "value2", ...), - PARTITION partition_name2 VALUES LESS THAN MAXVALUE|("value1", "value2", ...) - ... - ) - ``` - - 说明: - 使用指定的 key 列和指定的数值范围进行分区。 - 1) 分区名称仅支持字母开头,字母、数字和下划线组成 - 2) 目前仅支持以下类型的列作为 Range 分区列,且只能指定一个分区列 - TINYINT, SMALLINT, INT, BIGINT, LARGEINT, DATE, DATETIME - 3) 分区为左闭右开区间,首个分区的左边界为做最小值 - 4) NULL 值只会存放在包含最小值的分区中。当包含最小值的分区被删除后,NULL 值将无法导入。 - 5) 可以指定一列或多列作为分区列。如果分区值缺省,则会默认填充最小值。 - - 注意: - 1) 分区一般用于时间维度的数据管理 - 2) 有数据回溯需求的,可以考虑首个分区为空分区,以便后续增加分区 - - 2)Fixed Range - 语法: - ``` - PARTITION BY RANGE (k1, k2, k3, ...) - ( - PARTITION partition_name1 VALUES [("k1-lower1", "k2-lower1", "k3-lower1",...), ("k1-upper1", "k2-upper1", "k3-upper1", ...)), - PARTITION partition_name2 VALUES [("k1-lower1-2", "k2-lower1-2", ...), ("k1-upper1-2", MAXVALUE, )) - "k3-upper1-2", ... - ) - ``` - 说明: - 1)Fixed Range比LESS THAN相对灵活些,左右区间完全由用户自己确定 - 2)其他与LESS THAN保持同步 - -3. distribution_desc +5. partition_desc + 目前支持 RANGE 和 LIST 两种分区方式。 + 5.1 RANGE 分区 + RANGE partition描述有两种使用方式 + 1) LESS THAN + 语法: + + ``` + PARTITION BY RANGE (k1, k2, ...) + ( + PARTITION partition_name1 VALUES LESS THAN MAXVALUE|("value1", "value2", ...), + PARTITION partition_name2 VALUES LESS THAN MAXVALUE|("value1", "value2", ...) + ... + ) + ``` + + 说明: + 使用指定的 key 列和指定的数值范围进行分区。 + 1) 分区名称仅支持字母开头,字母、数字和下划线组成 + 2) 目前仅支持以下类型的列作为 Range 分区列 + TINYINT, SMALLINT, INT, BIGINT, LARGEINT, DATE, DATETIME + 3) 分区为左闭右开区间,首个分区的左边界为做最小值 + 4) NULL 值只会存放在包含最小值的分区中。当包含最小值的分区被删除后,NULL 值将无法导入。 + 5) 可以指定一列或多列作为分区列。如果分区值缺省,则会默认填充最小值。 + + 注意: + 1) 分区一般用于时间维度的数据管理 + 2) 有数据回溯需求的,可以考虑首个分区为空分区,以便后续增加分区 + + 2)Fixed Range + 语法: + ``` + PARTITION BY RANGE (k1, k2, k3, ...) + ( + PARTITION partition_name1 VALUES [("k1-lower1", "k2-lower1", "k3-lower1",...), ("k1-upper1", "k2-upper1", "k3-upper1", ...)), + PARTITION partition_name2 VALUES [("k1-lower1-2", "k2-lower1-2", ...), ("k1-upper1-2", MAXVALUE, )) + "k3-upper1-2", ... + ) + ``` + 说明: + 1)Fixed Range比LESS THAN相对灵活些,左右区间完全由用户自己确定 + 2)其他与LESS THAN保持同步 + + 5.2 LIST 分区 + LIST partition分为单列分区和多列分区 + 1) 单列分区 + 语法: + + ``` + PARTITION BY LIST(k1) + ( + PARTITION partition_name1 VALUES IN ("value1", "value2", ...), + PARTITION partition_name2 VALUES IN ("value1", "value2", ...) + ... + ) + ``` + + 说明: + 使用指定的 key 列和制定的枚举值进行分区。 + 1) 分区名称仅支持字母开头,字母、数字和下划线组成 + 2) 目前仅支持以下类型的列作为 List 分区列 + BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, LARGEINT, DATE, DATETIME, CHAR, VARCHAR + 3) 分区为枚举值集合,各个分区之间分区值不能重复 + 4) 不可导入 NULL 值 + 5) 分区值不能缺省,必须指定至少一个 + + 2) 多列分区 + 语法: + + ``` + PARTITION BY LIST(k1, k2) + ( + PARTITION partition_name1 VALUES IN (("value1", "value2"), ("value1", "value2"), ...), + PARTITION partition_name2 VALUES IN (("value1", "value2"), ("value1", "value2"), ...) + ... + ) + ``` + + 说明: + 1) 多列分区的分区是元组枚举值的集合 + 2) 每个元组值的个数必须与分区列个数相等 + 3) 其他与单列分区保持同步 + +6. distribution_desc 1) Hash 分桶 语法: `DISTRIBUTED BY HASH (k1[,k2 ...]) [BUCKETS num]` @@ -230,7 +272,7 @@ under the License. 建议:建议使用Hash分桶方式 -4. PROPERTIES +7. PROPERTIES 1) 如果 ENGINE 类型为 olap 可以在 properties 设置该表数据的初始存储介质、存储到期时间和副本数。 @@ -271,7 +313,7 @@ under the License. ) ``` - 4) 如果希望使用动态分区特性,需要在properties 中指定 + 4) 如果希望使用动态分区特性,需要在properties 中指定。注意:动态分区只支持 RANGE 分区 ``` PROPERTIES ( @@ -418,9 +460,84 @@ under the License. ); ``` -4. 创建一个 mysql 表 +4. 创建一个 olap 表,使用 List 分区,使用Hash分桶,默认使用列存, + 相同key的记录同时存在,设置初始存储介质和冷却时间 + + 1)单列分区 + + ``` + CREATE TABLE example_db.table_list + ( + k1 INT, + k2 VARCHAR(128), + k3 SMALLINT, + v1 VARCHAR(2048), + v2 DATETIME DEFAULT "2014-02-04 15:36:00" + ) + ENGINE=olap + DUPLICATE KEY(k1, k2, k3) + PARTITION BY LIST (k1) + ( + PARTITION p1 VALUES IN ("1", "2", "3"), + PARTITION p2 VALUES IN ("4", "5", "6"), + PARTITION p3 VALUES IN ("7", "8", "9") + ) + DISTRIBUTED BY HASH(k2) BUCKETS 32 + PROPERTIES( + "storage_medium" = "SSD", "storage_cooldown_time" = "2022-06-04 00:00:00" + ); + ``` + + 说明: + 这个语句会将数据划分成如下3个分区: + + ``` + ("1", "2", "3") + ("4", "5", "6") + ("7", "8", "9") + ``` + + 不在这些分区枚举值内的数据将视为非法数据被过滤 + + 2) 多列分区 + + ``` + CREATE TABLE example_db.table_list + ( + k1 INT, + k2 VARCHAR(128), + k3 SMALLINT, + v1 VARCHAR(2048), + v2 DATETIME DEFAULT "2014-02-04 15:36:00" + ) + ENGINE=olap + DUPLICATE KEY(k1, k2, k3) + PARTITION BY LIST (k1, k2) + ( + PARTITION p1 VALUES IN (("1","beijing"), ("1", "shanghai")), + PARTITION p2 VALUES IN (("2","beijing"), ("2", "shanghai")), + PARTITION p3 VALUES IN (("3","beijing"), ("3", "shanghai")) + ) + DISTRIBUTED BY HASH(k2) BUCKETS 32 + PROPERTIES( + "storage_medium" = "SSD", "storage_cooldown_time" = "2022-06-04 00:00:00" + ); + ``` + + 说明: + 这个语句会将数据划分成如下3个分区: - 4.1 直接通过外表信息创建mysql表 + ``` + (("1","beijing"), ("1", "shanghai")) + (("2","beijing"), ("2", "shanghai")) + (("3","beijing"), ("3", "shanghai")) + ``` + + 不在这些分区枚举值内的数据将视为非法数据被过滤 + +5. 创建一个 mysql 表 + + 5.1 直接通过外表信息创建mysql表 ``` CREATE EXTERNAL TABLE example_db.table_mysql ( @@ -442,7 +559,7 @@ under the License. ) ``` - 4.2 通过External Catalog Resource创建mysql表 + 5.2 通过External Catalog Resource创建mysql表 ``` CREATE EXTERNAL RESOURCE "mysql_resource" PROPERTIES @@ -472,7 +589,7 @@ under the License. ) ``` -5. 创建一个数据文件存储在HDFS上的 broker 外部表, 数据使用 "|" 分割,"\n" 换行 +6. 创建一个数据文件存储在HDFS上的 broker 外部表, 数据使用 "|" 分割,"\n" 换行 ``` CREATE EXTERNAL TABLE example_db.table_broker ( @@ -495,7 +612,7 @@ under the License. ) ``` -6. 创建一张含有HLL列的表 +7. 创建一张含有HLL列的表 ``` CREATE TABLE example_db.example_table @@ -511,7 +628,7 @@ under the License. PROPERTIES ("storage_type"="column"); ``` -7. 创建一张含有BITMAP_UNION聚合类型的表(v1和v2列的原始数据类型必须是TINYINT,SMALLINT,INT) +8. 创建一张含有BITMAP_UNION聚合类型的表(v1和v2列的原始数据类型必须是TINYINT,SMALLINT,INT) ``` CREATE TABLE example_db.example_table @@ -527,7 +644,7 @@ under the License. PROPERTIES ("storage_type"="column"); ``` -8. 创建两张支持Colocate Join的表t1 和t2 +9. 创建两张支持Colocate Join的表t1 和t2 ``` CREATE TABLE `t1` ( @@ -551,7 +668,7 @@ under the License. ); ``` -9. 创建一个数据文件存储在BOS上的 broker 外部表 +10. 创建一个数据文件存储在BOS上的 broker 外部表 ``` CREATE EXTERNAL TABLE example_db.table_broker ( @@ -569,7 +686,7 @@ under the License. ) ``` -10. 创建一个带有bitmap 索引的表 +11. 创建一个带有bitmap 索引的表 ``` CREATE TABLE example_db.table_hash @@ -587,7 +704,7 @@ under the License. PROPERTIES ("storage_type"="column"); ``` -11. 创建一个动态分区表(需要在FE配置中开启动态分区功能),该表每天提前创建3天的分区,并删除3天前的分区。例如今天为`2020-01-08`,则会创建分区名为`p20200108`, `p20200109`, `p20200110`, `p20200111`的分区. 分区范围分别为: +12. 创建一个动态分区表(需要在FE配置中开启动态分区功能),该表每天提前创建3天的分区,并删除3天前的分区。例如今天为`2020-01-08`,则会创建分区名为`p20200108`, `p20200109`, `p20200110`, `p20200111`的分区. 分区范围分别为: ``` [types: [DATE]; keys: [2020-01-08]; ‥types: [DATE]; keys: [2020-01-09]; ) @@ -624,7 +741,7 @@ under the License. ); ``` -12. Create a table with rollup index +13. Create a table with rollup index ``` CREATE TABLE example_db.rolup_index_table ( @@ -643,7 +760,8 @@ under the License. ) PROPERTIES("replication_num" = "3"); -13. 创建一个内存表 +14. 创建一个内存表 + ``` CREATE TABLE example_db.table_hash ( @@ -660,7 +778,8 @@ under the License. PROPERTIES ("in_memory"="true"); ``` -14. 创建一个hive外部表 +15. 创建一个hive外部表 + ``` CREATE TABLE example_db.table_hive ( diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/RESTORE.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/RESTORE.md index 9871639e5ea2ea..ee400e30035570 100644 --- a/docs/zh-CN/sql-reference/sql-statements/Data Definition/RESTORE.md +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/RESTORE.md @@ -42,7 +42,7 @@ under the License. 2. ON 子句中标识需要恢复的表和分区。如果不指定分区,则默认恢复该表的所有分区。所指定的表和分区必须已存在于仓库备份中。 3. 可以通过 AS 语句将仓库中备份的表名恢复为新的表。但新表名不能已存在于数据库中。分区名称不能修改。 4. 可以将仓库中备份的表恢复替换数据库中已有的同名表,但须保证两张表的表结构完全一致。表结构包括:表名、列、分区、Rollup等等。 - 5. 可以指定恢复表的部分分区,系统会检查分区 Range 是否能够匹配。 + 5. 可以指定恢复表的部分分区,系统会检查分区 Range 或者 List 是否能够匹配。 6. PROPERTIES 目前支持以下属性: "backup_timestamp" = "2018-05-04-16-45-08":指定了恢复对应备份的哪个时间版本,必填。该信息可以通过 `SHOW SNAPSHOT ON repo;` 语句获得。 "replication_num" = "3":指定恢复的表或分区的副本数。默认为3。若恢复已存在的表或分区,则副本数必须和已存在表或分区的副本数相同。同时,必须有足够的 host 容纳多个副本。 diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/DELETE.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/DELETE.md index c1dcbeb8ec0bab..4f7de606ecc768 100644 --- a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/DELETE.md +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/DELETE.md @@ -40,7 +40,7 @@ under the License. 2) 当选定的 key 列不存在于某个 rollup 中时,无法进行 delete。 3) 条件之间只能是“与”的关系。 若希望达成“或”的关系,需要将条件分写在两个 DELETE 语句中。 - 4) 如果为RANGE分区表,可以指定分区,如不指定,且会话变量 delete_without_partition 为 true,则会应用到所有分区。如果是单分区表,可以不指定。 + 4) 如果为分区表,可以指定分区,如不指定,且会话变量 delete_without_partition 为 true,则会应用到所有分区。如果是单分区表,可以不指定。 注意: 该语句可能会降低执行后一段时间内的查询效率。 diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 50151fca753c04..bb2801ea1eac8a 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -244,7 +244,7 @@ terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALL, KW_ALTER, KW_A KW_INNER, KW_INSERT, KW_INT, KW_INTERMEDIATE, KW_INTERSECT, KW_INTERVAL, KW_INTO, KW_IS, KW_ISNULL, KW_ISOLATION, KW_JOIN, KW_KEY, KW_KEYS, KW_KILL, - KW_LABEL, KW_LARGEINT, KW_LAST, KW_LEFT, KW_LESS, KW_LEVEL, KW_LIKE, KW_LIMIT, KW_LINK, KW_LOAD, + KW_LABEL, KW_LARGEINT, KW_LAST, KW_LEFT, KW_LESS, KW_LEVEL, KW_LIKE, KW_LIMIT, KW_LINK, KW_LIST, KW_LOAD, KW_LOCAL, KW_LOCATION, KW_MATERIALIZED, KW_MAX, KW_MAX_VALUE, KW_MERGE, KW_MIN, KW_MINUTE, KW_MINUS, KW_MIGRATE, KW_MIGRATIONS, KW_MODIFY, KW_MONTH, KW_NAME, KW_NAMES, KW_NEGATIVE, KW_NO, KW_NOT, KW_NULL, KW_NULLS, @@ -416,11 +416,15 @@ nonterminal Long opt_field_length; nonterminal KeysDesc opt_keys; nonterminal PartitionKeyDesc partition_key_desc; +nonterminal PartitionKeyDesc list_partition_key_desc; nonterminal PartitionKeyDesc fixed_partition_key_desc; nonterminal List partition_key_list; -nonterminal SingleRangePartitionDesc single_range_partition_desc; -nonterminal List opt_single_range_partition_desc_list; -nonterminal List single_range_partition_desc_list; +nonterminal List partition_value_list; +nonterminal List partition_key_item_list; +nonterminal List> list_partition_values_list; +nonterminal SinglePartitionDesc single_partition_desc; +nonterminal List opt_single_partition_desc_list; +nonterminal List single_partition_desc_list; nonterminal List privilege_list; nonterminal List string_list; @@ -947,7 +951,7 @@ alter_table_clause ::= {: RESULT = new ModifyTablePropertiesClause(properties); :} - | KW_ADD opt_tmp:isTempPartition single_range_partition_desc:desc opt_distribution:distribution opt_properties:properties + | KW_ADD opt_tmp:isTempPartition single_partition_desc:desc opt_distribution:distribution opt_properties:properties {: RESULT = new AddPartitionClause(desc, distribution, properties, isTempPartition); :} @@ -1865,10 +1869,16 @@ opt_partition ::= :} /* Range partition */ | KW_PARTITION KW_BY KW_RANGE LPAREN ident_list:columns RPAREN - LPAREN opt_single_range_partition_desc_list:list RPAREN + LPAREN opt_single_partition_desc_list:list RPAREN {: RESULT = new RangePartitionDesc(columns, list); :} + /* List partition */ + | KW_PARTITION KW_BY KW_LIST LPAREN ident_list:columns RPAREN + LPAREN opt_single_partition_desc_list:list RPAREN + {: + RESULT = new ListPartitionDesc(columns, list); + :} ; opt_distribution ::= @@ -1933,39 +1943,45 @@ opt_keys ::= :} ; -opt_single_range_partition_desc_list ::= +opt_single_partition_desc_list ::= /* Empty */ {: RESULT = null; :} - | single_range_partition_desc_list:list + | single_partition_desc_list:list {: RESULT = list; :} ; -single_range_partition_desc_list ::= - single_range_partition_desc_list:list COMMA single_range_partition_desc:desc +single_partition_desc_list ::= + single_partition_desc_list:list COMMA single_partition_desc:desc {: list.add(desc); RESULT = list; :} - | single_range_partition_desc:desc + | single_partition_desc:desc {: RESULT = Lists.newArrayList(desc); :} ; -single_range_partition_desc ::= +single_partition_desc ::= KW_PARTITION opt_if_not_exists:ifNotExists ident:partName KW_VALUES KW_LESS KW_THAN partition_key_desc:desc opt_key_value_map:properties {: - RESULT = new SingleRangePartitionDesc(ifNotExists, partName, desc, properties); + RESULT = new SinglePartitionDesc(ifNotExists, partName, desc, properties); :} | KW_PARTITION opt_if_not_exists:ifNotExists ident:partName KW_VALUES fixed_partition_key_desc:desc opt_key_value_map:properties {: - RESULT = new SingleRangePartitionDesc(ifNotExists, partName, desc, properties); + RESULT = new SinglePartitionDesc(ifNotExists, partName, desc, properties); + :} + /* list partition */ + | KW_PARTITION opt_if_not_exists:ifNotExists ident:partName KW_VALUES KW_IN list_partition_key_desc:desc + opt_key_value_map:properties + {: + RESULT = new SinglePartitionDesc(ifNotExists, partName, desc, properties); :} ; @@ -1976,7 +1992,57 @@ partition_key_desc ::= :} | LPAREN partition_key_list:keys RPAREN {: - RESULT = new PartitionKeyDesc(keys); + RESULT = PartitionKeyDesc.createLessThan(keys); + :} + ; + +/* list partition PartitionKeyDesc */ +list_partition_key_desc ::= + LPAREN list_partition_values_list:keys RPAREN + {: + RESULT = PartitionKeyDesc.createIn(keys); + :} + ; + +/* List> inValues */ +list_partition_values_list ::= + partition_value_list:item + {: + ArrayList> l = new ArrayList(); + l.add(item); + RESULT = l; + :} + | list_partition_values_list:l COMMA partition_value_list:item + {: + l.add(item); + RESULT = l; + :} + ; + +/* List */ +partition_value_list ::= + /* single partition key */ + STRING_LITERAL:item + {: + RESULT = Lists.newArrayList(new PartitionValue(item)); + :} + /* multi partition keys : (1, "beijing") */ + | LPAREN partition_key_item_list:l RPAREN + {: + RESULT = l; + :} + ; + +/* List */ +partition_key_item_list ::= + STRING_LITERAL:item + {: + RESULT = Lists.newArrayList(new PartitionValue(item)); + :} + | partition_key_item_list:l COMMA STRING_LITERAL:item + {: + l.add(new PartitionValue(item)); + RESULT = l; :} ; @@ -2010,7 +2076,7 @@ fixed_partition_key_desc ::= /* format: [(lower), (upper))*/ LBRACKET LPAREN partition_key_list:lower RPAREN COMMA LPAREN partition_key_list:upper RPAREN RPAREN {: - RESULT = new PartitionKeyDesc(lower, upper); + RESULT = PartitionKeyDesc.createFixed(lower, upper); :} ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index ee9ae4bada1b27..3a00b62d3b2683 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -49,7 +49,6 @@ import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionType; -import org.apache.doris.catalog.RangePartitionInfo; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Replica.ReplicaState; import org.apache.doris.catalog.Table; @@ -1215,9 +1214,8 @@ private void createJob(long dbId, OlapTable olapTable, Map partitionColumns = rangePartitionInfo.getPartitionColumns(); + if (partitionInfo.getType() == PartitionType.RANGE || partitionInfo.getType() == PartitionType.LIST) { + List partitionColumns = partitionInfo.getPartitionColumns(); for (Column partitionCol : partitionColumns) { boolean found = false; for (Column alterColumn : alterSchema) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AddPartitionClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AddPartitionClause.java index 23f8a23eaae55c..ace4b4c3ab0fda 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AddPartitionClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AddPartitionClause.java @@ -25,13 +25,13 @@ // clause which is used to add partition public class AddPartitionClause extends AlterTableClause { - private SingleRangePartitionDesc partitionDesc; + private SinglePartitionDesc partitionDesc; private DistributionDesc distributionDesc; private Map properties; // true if this is to add a temporary partition private boolean isTempPartition; - public SingleRangePartitionDesc getSingeRangePartitionDesc() { + public SinglePartitionDesc getSingeRangePartitionDesc() { return partitionDesc; } @@ -43,7 +43,7 @@ public boolean isTempPartition() { return isTempPartition; } - public AddPartitionClause(SingleRangePartitionDesc partitionDesc, + public AddPartitionClause(SinglePartitionDesc partitionDesc, DistributionDesc distributionDesc, Map properties, boolean isTempPartition) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java index 55d7275e4d792e..5edefdd595344e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java @@ -68,6 +68,11 @@ public boolean getValue() { return value; } + @Override + public Object getRealValue() { + return getValue(); + } + @Override public boolean isMinValue() { return false; @@ -78,6 +83,9 @@ public int compareLiteral(LiteralExpr expr) { if (expr instanceof NullLiteral) { return 1; } + if (expr == MaxLiteral.MAX_VALUE) { + return -1; + } return Long.signum(getLongValue() - expr.getLongValue()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index 7641ac96e569da..d20c75a0b50fd8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -22,7 +22,6 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Index; import org.apache.doris.catalog.KeysType; -import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; @@ -382,12 +381,12 @@ public void analyze(Analyzer analyzer) throws UserException { if (engineName.equals("olap")) { // analyze partition if (partitionDesc != null) { - if (partitionDesc.getType() != PartitionType.RANGE) { - throw new AnalysisException("Currently only support range partition with engine type olap"); + if (partitionDesc instanceof ListPartitionDesc || partitionDesc instanceof RangePartitionDesc) { + partitionDesc.analyze(columnDefs, properties); + } else { + throw new AnalysisException("Currently only support range and list partition with engine type olap"); } - RangePartitionDesc rangePartitionDesc = (RangePartitionDesc) partitionDesc; - rangePartitionDesc.analyze(columnDefs, properties); } // analyze distribution diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ListPartitionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ListPartitionDesc.java new file mode 100644 index 00000000000000..31aec5fb21d5d9 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ListPartitionDesc.java @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.analysis.PartitionKeyDesc.PartitionKeyValueType; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ListPartitionInfo; +import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +// to describe the key list partition's information in create table stmt +public class ListPartitionDesc extends PartitionDesc { + + public ListPartitionDesc(List partitionColNames, + List singlePartitionDescs) { + super(partitionColNames, singlePartitionDescs); + type = PartitionType.LIST; + } + + @Override + public void checkPartitionKeyValueType(PartitionKeyDesc partitionKeyDesc) throws AnalysisException { + if (partitionKeyDesc.getPartitionType() != PartitionKeyValueType.IN) { + throw new AnalysisException("You can only use in values to create list partitions"); + } + } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder(); + sb.append("PARTITION BY LIST("); + int idx = 0; + for (String column : partitionColNames) { + if (idx != 0) { + sb.append(", "); + } + sb.append("`").append(column).append("`"); + idx++; + } + sb.append(")\n(\n"); + + for (int i = 0; i < singlePartitionDescs.size(); i++) { + if (i != 0) { + sb.append(",\n"); + } + sb.append(singlePartitionDescs.get(i).toSql()); + } + sb.append("\n)"); + return sb.toString(); + } + + @Override + public String toString() { + return toSql(); + } + + @Override + public PartitionInfo toPartitionInfo(List schema, Map partitionNameToId, boolean isTemp) + throws DdlException { + List partitionColumns = new ArrayList<>(); + + // check and get partition column + for (String colName : partitionColNames) { + boolean find = false; + for (Column column : schema) { + if (column.getName().equalsIgnoreCase(colName)) { + try { + ListPartitionInfo.checkPartitionColumn(column); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + + partitionColumns.add(column); + find = true; + break; + } + } + if (!find) { + throw new DdlException("Partition column[" + colName + "] does not found"); + } + } + + ListPartitionInfo listPartitionInfo = new ListPartitionInfo(partitionColumns); + for (SinglePartitionDesc desc : singlePartitionDescs) { + long partitionId = partitionNameToId.get(desc.getPartitionName()); + listPartitionInfo.handleNewSinglePartitionDesc(desc, partitionId, isTemp); + } + return listPartitionInfo; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java index 2823480922d057..5bed750709f060 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java @@ -117,7 +117,7 @@ protected void analyzeImpl(Analyzer analyzer) throws AnalysisException { * return real value */ public Object getRealValue() { - // implemented: TINYINT/SMALLINT/INT/BIGINT/LARGEINT/DATE/DATETIME + // implemented: TINYINT/SMALLINT/INT/BIGINT/LARGEINT/DATE/DATETIME/CHAR/VARCHAR/BOOLEAN Preconditions.checkState(false, "should implement this in derived class. " + this.type.toSql()); return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java index f2b46afe50951f..7566cdd6f2630a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java @@ -17,23 +17,104 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.apache.commons.lang.NotImplementedException; import java.util.List; import java.util.Map; +import java.util.Set; public class PartitionDesc { + protected List partitionColNames; + protected List singlePartitionDescs; protected PartitionType type; + public PartitionDesc(List partitionColNames, + List singlePartitionDescs) { + this.partitionColNames = partitionColNames; + this.singlePartitionDescs = singlePartitionDescs; + if (this.singlePartitionDescs == null) { + this.singlePartitionDescs = Lists.newArrayList(); + } + } + + public List getSinglePartitionDescs() { + return this.singlePartitionDescs; + } + + public List getPartitionColNames() { + return partitionColNames; + } + public void analyze(List columnDefs, Map otherProperties) throws AnalysisException { - throw new NotImplementedException(); + if (partitionColNames == null || partitionColNames.isEmpty()) { + throw new AnalysisException("No partition columns."); + } + + Set partColNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + for (String partitionCol : partitionColNames) { + if (!partColNames.add(partitionCol)) { + throw new AnalysisException("Duplicated partition column " + partitionCol); + } + + boolean found = false; + for (ColumnDef columnDef : columnDefs) { + if (columnDef.getName().equals(partitionCol)) { + if (!columnDef.isKey() && columnDef.getAggregateType() != AggregateType.NONE) { + throw new AnalysisException("The partition column could not be aggregated column"); + } + if (columnDef.getType().isFloatingPointType()) { + throw new AnalysisException("Floating point type column can not be partition column"); + } + if (!ConnectContext.get().getSessionVariable().isAllowPartitionColumnNullable() + && columnDef.isAllowNull()) { + throw new AnalysisException("The partition column must be NOT NULL"); + } + if (this instanceof ListPartitionDesc && columnDef.isAllowNull()) { + throw new AnalysisException("The list partition column must be NOT NULL"); + } + found = true; + break; + } + } + + if (!found) { + throw new AnalysisException("Partition column[" + partitionCol + "] does not exist in column list."); + } + } + + Set nameSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + for (SinglePartitionDesc desc : singlePartitionDescs) { + if (!nameSet.add(desc.getPartitionName())) { + throw new AnalysisException("Duplicated partition name: " + desc.getPartitionName()); + } + // in create table stmt, we use given properties + // copy one. because ProperAnalyzer will remove entry after analyze + Map givenProperties = null; + if (otherProperties != null) { + givenProperties = Maps.newHashMap(otherProperties); + } + // check partitionType + checkPartitionKeyValueType(desc.getPartitionKeyDesc()); + // analyze singlePartitionDesc + desc.analyze(columnDefs.size(), givenProperties); + } + } + + public void checkPartitionKeyValueType(PartitionKeyDesc partitionKeyDesc) throws AnalysisException { + } public PartitionType getType() { @@ -44,7 +125,7 @@ public String toSql() { throw new NotImplementedException(); } - public PartitionInfo toPartitionInfo(List columns, Map partitionNameToId, boolean isTemp) + public PartitionInfo toPartitionInfo(List schema, Map partitionNameToId, boolean isTemp) throws DdlException { throw new NotImplementedException(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java index 7fe7285cec007f..726c333c2b08fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java @@ -29,35 +29,45 @@ public class PartitionKeyDesc { public static final PartitionKeyDesc MAX_VALUE = new PartitionKeyDesc(); - public enum PartitionRangeType { + public enum PartitionKeyValueType { INVALID, LESS_THAN, - FIXED + FIXED, + IN } private List lowerValues; private List upperValues; - private PartitionRangeType partitionType; + private List> inValues; + private PartitionKeyValueType partitionKeyValueType; public static PartitionKeyDesc createMaxKeyDesc() { return MAX_VALUE; } private PartitionKeyDesc() { - partitionType = PartitionRangeType.LESS_THAN; // LESS_THAN is default type. + partitionKeyValueType = PartitionKeyValueType.LESS_THAN; // LESS_THAN is default type. } - // values less than - public PartitionKeyDesc(List upperValues) { - this.upperValues = upperValues; - partitionType = PartitionRangeType.LESS_THAN; + public static PartitionKeyDesc createLessThan(List upperValues) { + PartitionKeyDesc desc = new PartitionKeyDesc(); + desc.upperValues = upperValues; + desc.partitionKeyValueType = PartitionKeyValueType.LESS_THAN; + return desc; } - // fixed range - public PartitionKeyDesc(List lowerValues, List upperValues) { - this.lowerValues = lowerValues; - this.upperValues = upperValues; - partitionType = PartitionRangeType.FIXED; + public static PartitionKeyDesc createIn(List> inValues) { + PartitionKeyDesc desc = new PartitionKeyDesc(); + desc.inValues = inValues; + desc.partitionKeyValueType = PartitionKeyValueType.IN; + return desc; + } + + public static PartitionKeyDesc createFixed(List lowerValues, List upperValues) { + PartitionKeyDesc desc = new PartitionKeyDesc(); + desc.lowerValues = lowerValues; + desc.upperValues = upperValues; + return desc; } public List getLowerValues() { @@ -68,6 +78,10 @@ public List getUpperValues() { return upperValues; } + public List> getInValues() { + return inValues; + } + public boolean isMax() { return this == MAX_VALUE; } @@ -80,13 +94,13 @@ public boolean hasUpperValues() { return upperValues != null; } - public PartitionRangeType getPartitionType() { - return partitionType; + public PartitionKeyValueType getPartitionType() { + return partitionKeyValueType; } public void analyze(int partColNum) throws AnalysisException { if (!isMax()) { - if (upperValues.isEmpty() || upperValues.size() > partColNum) { + if ((upperValues != null && (upperValues.isEmpty() || upperValues.size() > partColNum))) { throw new AnalysisException("Partition values number is more than partition column number: " + toSql()); } } @@ -119,13 +133,27 @@ public String toSql() { return "MAXVALUE"; } - if (partitionType == PartitionRangeType.LESS_THAN) { + if (partitionKeyValueType == PartitionKeyValueType.LESS_THAN) { return getPartitionValuesStr(upperValues); - } else if (partitionType == PartitionRangeType.FIXED) { + } else if (partitionKeyValueType == PartitionKeyValueType.FIXED) { StringBuilder sb = new StringBuilder("["); sb.append(getPartitionValuesStr(lowerValues)).append(", ").append(getPartitionValuesStr(upperValues)); sb.append(")"); return sb.toString(); + } else if (partitionKeyValueType == PartitionKeyValueType.IN) { + StringBuilder sb = new StringBuilder("("); + for (int i = 0; i < inValues.size(); i++) { + String valueStr = getPartitionValuesStr(inValues.get(i)); + if (inValues.get(i).size() == 1) { + valueStr = valueStr.substring(1, valueStr.length() - 1); + } + sb.append(valueStr); + if (i < inValues.size() -1) { + sb.append(","); + } + } + sb.append(")"); + return sb.toString(); } else { return "INVALID"; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RangePartitionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RangePartitionDesc.java index f0c345c6cfb3da..eb08ad2780edc5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RangePartitionDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RangePartitionDesc.java @@ -17,101 +17,31 @@ package org.apache.doris.analysis; -import org.apache.doris.analysis.PartitionKeyDesc.PartitionRangeType; -import org.apache.doris.catalog.AggregateType; +import org.apache.doris.analysis.PartitionKeyDesc.PartitionKeyValueType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.PartitionInfo; -import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.RangePartitionInfo; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import org.apache.doris.qe.ConnectContext; - +import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Set; // to describe the key range partition's information in create table stmt public class RangePartitionDesc extends PartitionDesc { - private List partitionColNames; - private List singleRangePartitionDescs; public RangePartitionDesc(List partitionColNames, - List singlePartitionDescs) { - type = PartitionType.RANGE; - this.partitionColNames = partitionColNames; - this.singleRangePartitionDescs = singlePartitionDescs; - if (singleRangePartitionDescs == null) { - singleRangePartitionDescs = Lists.newArrayList(); - } - } - - public List getSingleRangePartitionDescs() { - return this.singleRangePartitionDescs; - } - - public List getPartitionColNames() { - return partitionColNames; + List singlePartitionDescs) { + super(partitionColNames, singlePartitionDescs); + type = org.apache.doris.catalog.PartitionType.RANGE; } @Override - public void analyze(List columnDefs, Map otherProperties) throws AnalysisException { - if (partitionColNames == null || partitionColNames.isEmpty()) { - throw new AnalysisException("No partition columns."); - } - - Set partColNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); - for (String partitionCol : partitionColNames) { - if (!partColNames.add(partitionCol)) { - throw new AnalysisException("Duplicated partition column " + partitionCol); - } - - boolean found = false; - for (ColumnDef columnDef : columnDefs) { - if (columnDef.getName().equals(partitionCol)) { - if (!columnDef.isKey() && columnDef.getAggregateType() != AggregateType.NONE) { - throw new AnalysisException("The partition column could not be aggregated column"); - } - if (columnDef.getType().isFloatingPointType()) { - throw new AnalysisException("Floating point type column can not be partition column"); - } - if (!ConnectContext.get().getSessionVariable().isAllowPartitionColumnNullable() - && columnDef.isAllowNull()) { - throw new AnalysisException("The partition column must be NOT NULL"); - } - found = true; - break; - } - } - - if (!found) { - throw new AnalysisException("Partition column[" + partitionCol + "] does not exist in column list."); - } - } - - Set nameSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); - PartitionRangeType partitionType = PartitionRangeType.INVALID; - for (SingleRangePartitionDesc desc : singleRangePartitionDescs) { - if (!nameSet.add(desc.getPartitionName())) { - throw new AnalysisException("Duplicated partition name: " + desc.getPartitionName()); - } - // in create table stmt, we use given properties - // copy one. because ProperAnalyzer will remove entry after analyze - Map givenProperties = null; - if (otherProperties != null) { - givenProperties = Maps.newHashMap(otherProperties); - } - // check partitionType - if (partitionType == PartitionRangeType.INVALID) { - partitionType = desc.getPartitionKeyDesc().getPartitionType(); - } else if (partitionType != desc.getPartitionKeyDesc().getPartitionType()) { - throw new AnalysisException("You can only use one of these methods to create partitions"); - } - desc.analyze(columnDefs.size(), givenProperties); + public void checkPartitionKeyValueType(PartitionKeyDesc partitionKeyDesc) throws AnalysisException { + if (partitionKeyDesc.getPartitionType() != PartitionKeyValueType.FIXED && + partitionKeyDesc.getPartitionType() != PartitionKeyValueType.LESS_THAN) { + throw new AnalysisException("You can only use fixed or less than values to create range partitions"); } } @@ -129,11 +59,11 @@ public String toSql() { } sb.append(")\n(\n"); - for (int i = 0; i < singleRangePartitionDescs.size(); i++) { + for (int i = 0; i < singlePartitionDescs.size(); i++) { if (i != 0) { sb.append(",\n"); } - sb.append(singleRangePartitionDescs.get(i).toSql()); + sb.append(singlePartitionDescs.get(i).toSql()); } sb.append("\n)"); return sb.toString(); @@ -147,28 +77,15 @@ public String toString() { @Override public PartitionInfo toPartitionInfo(List schema, Map partitionNameToId, boolean isTemp) throws DdlException { - List partitionColumns = Lists.newArrayList(); + List partitionColumns = new ArrayList<>(); // check and get partition column for (String colName : partitionColNames) { boolean find = false; for (Column column : schema) { if (column.getName().equalsIgnoreCase(colName)) { - if (!column.isKey() && column.getAggregationType() != AggregateType.NONE) { - throw new DdlException("The partition column could not be aggregated column"); - } - - if (column.getType().isFloatingPointType()) { - throw new DdlException("Floating point type column can not be partition column"); - } - - if (!ConnectContext.get().getSessionVariable().isAllowPartitionColumnNullable() - && column.isAllowNull()) { - throw new DdlException("The partition column must be NOT NULL"); - } - try { - RangePartitionInfo.checkRangeColumnType(column); + RangePartitionInfo.checkPartitionColumn(column); } catch (AnalysisException e) { throw new DdlException(e.getMessage()); } @@ -176,6 +93,7 @@ public PartitionInfo toPartitionInfo(List schema, Map part partitionColumns.add(column); find = true; break; + } } if (!find) { @@ -186,17 +104,17 @@ public PartitionInfo toPartitionInfo(List schema, Map part /* * validate key range * eg. - * VALUE LESS THEN (10, 100, 1000) - * VALUE LESS THEN (50, 500) - * VALUE LESS THEN (80) - * + * VALUE LESS THAN (10, 100, 1000) + * VALUE LESS THAN (50, 500) + * VALUE LESS THAN (80) + * * key range is: * ( {MIN, MIN, MIN}, {10, 100, 1000} ) * [ {10, 100, 1000}, {50, 500, MIN } ) * [ {50, 500, MIN }, {80, MIN, MIN } ) */ RangePartitionInfo rangePartitionInfo = new RangePartitionInfo(partitionColumns); - for (SingleRangePartitionDesc desc : singleRangePartitionDescs) { + for (SinglePartitionDesc desc : singlePartitionDescs) { long partitionId = partitionNameToId.get(desc.getPartitionName()); rangePartitionInfo.handleNewSinglePartitionDesc(desc, partitionId, isTemp); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SingleRangePartitionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java similarity index 92% rename from fe/fe-core/src/main/java/org/apache/doris/analysis/SingleRangePartitionDesc.java rename to fe/fe-core/src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java index 027df42bf8613b..102d4b875665d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SingleRangePartitionDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java @@ -17,7 +17,7 @@ package org.apache.doris.analysis; -import org.apache.doris.analysis.PartitionKeyDesc.PartitionRangeType; +import org.apache.doris.analysis.PartitionKeyDesc.PartitionKeyValueType; import org.apache.doris.catalog.DataProperty; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; @@ -33,7 +33,7 @@ import java.util.Map; -public class SingleRangePartitionDesc { +public class SinglePartitionDesc { private boolean isAnalyzed; private boolean ifNotExists; @@ -48,10 +48,10 @@ public class SingleRangePartitionDesc { private TTabletType tabletType = TTabletType.TABLET_TYPE_DISK; private Pair versionInfo; - public SingleRangePartitionDesc(boolean ifNotExists, String partName, PartitionKeyDesc partitionKeyDesc, - Map properties) { + public SinglePartitionDesc(boolean ifNotExists, String partName, PartitionKeyDesc partitionKeyDesc, + Map properties) { this.ifNotExists = ifNotExists; - + this.isAnalyzed = false; this.partName = partName; @@ -146,8 +146,8 @@ public boolean isAnalyzed() { public String toSql() { StringBuilder sb = new StringBuilder(); sb.append("PARTITION ").append(partName); - if (partitionKeyDesc.getPartitionType() == PartitionRangeType.LESS_THAN) { - sb.append(" VALUES LESS THEN "); + if (partitionKeyDesc.getPartitionType() == PartitionKeyValueType.LESS_THAN) { + sb.append(" VALUES LESS THAN "); } else { sb.append(" VALUES "); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java index c7ef065af98cbd..9f0b60ccd14022 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java @@ -87,7 +87,9 @@ public int compareLiteral(LiteralExpr expr) { if (expr instanceof NullLiteral) { return 1; } - + if (expr == MaxLiteral.MAX_VALUE) { + return -1; + } // compare string with utf-8 byte array, same with DM,BE,StorageEngine byte[] thisBytes = null; byte[] otherBytes = null; @@ -166,6 +168,11 @@ public double getDoubleValue() { return Double.valueOf(value); } + @Override + public String getRealValue() { + return getStringValue(); + } + /** * Convert a string literal to a date literal * diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 8c246fd82ebd6c..7f96fe71587490 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -37,9 +37,8 @@ import org.apache.doris.catalog.OlapTable.OlapTableState; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionInfo; -import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionType; -import org.apache.doris.catalog.RangePartitionInfo; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Replica.ReplicaState; import org.apache.doris.catalog.Resource; @@ -82,7 +81,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; -import com.google.common.collect.Range; import com.google.common.collect.Table.Cell; import java.io.DataInput; @@ -554,14 +552,11 @@ private void checkAndPrepareMeta() { if (localPartition != null) { // Partition already exist. PartitionInfo localPartInfo = localOlapTbl.getPartitionInfo(); - if (localPartInfo.getType() == PartitionType.RANGE) { - // If this is a range partition, check range - RangePartitionInfo localRangePartInfo = (RangePartitionInfo) localPartInfo; - RangePartitionInfo remoteRangePartInfo - = (RangePartitionInfo) remoteOlapTbl.getPartitionInfo(); - Range localRange = localRangePartInfo.getRange(localPartition.getId()); - Range remoteRange = remoteRangePartInfo.getRange(backupPartInfo.id); - if (localRange.equals(remoteRange)) { + if (localPartInfo.getType() == PartitionType.RANGE + || localPartInfo.getType() == PartitionType.LIST) { + PartitionItem localItem = localPartInfo.getItem(localPartition.getId()); + PartitionItem remoteItem = remoteOlapTbl.getPartitionInfo().getItem(backupPartInfo.id); + if (localItem.equals(remoteItem)) { // Same partition, same range if (genFileMappingWhenBackupReplicasEqual(localPartInfo, localPartition, localTbl, backupPartInfo, partitionName, tblInfo)) { return; @@ -570,7 +565,7 @@ private void checkAndPrepareMeta() { // Same partition name, different range status = new Status(ErrCode.COMMON_ERROR, "Partition " + partitionName + " in table " + localTbl.getName() - + " has different range with partition in repository"); + + " has different partition item with partition in repository"); return; } } else { @@ -582,16 +577,13 @@ private void checkAndPrepareMeta() { } else { // partitions does not exist PartitionInfo localPartitionInfo = localOlapTbl.getPartitionInfo(); - if (localPartitionInfo.getType() == PartitionType.RANGE) { - // Check if the partition range can be added to the table - RangePartitionInfo localRangePartitionInfo = (RangePartitionInfo) localPartitionInfo; - RangePartitionInfo remoteRangePartitionInfo - = (RangePartitionInfo) remoteOlapTbl.getPartitionInfo(); - Range remoteRange = remoteRangePartitionInfo.getRange(backupPartInfo.id); - if (localRangePartitionInfo.getAnyIntersectRange(remoteRange, false) != null) { + if (localPartitionInfo.getType() == PartitionType.RANGE + || localPartitionInfo.getType() == PartitionType.LIST) { + PartitionItem remoteItem = remoteOlapTbl.getPartitionInfo().getItem(backupPartInfo.id); + if (localPartitionInfo.getAnyIntersectItem(remoteItem, false) != null) { status = new Status(ErrCode.COMMON_ERROR, "Partition " + partitionName + " in table " + localTbl.getName() - + " has conflict range with existing ranges"); + + " has conflict partition item with existing items"); return; } else { // this partition can be added to this table, set ids @@ -754,7 +746,7 @@ private void checkAndPrepareMeta() { } else { ok = true; } - + if (ok) { LOG.debug("finished to create all restored replcias. {}", this); // add restored partitions. @@ -766,16 +758,20 @@ private void checkAndPrepareMeta() { try { Partition restoredPart = entry.second; OlapTable remoteTbl = (OlapTable) backupMeta.getTable(entry.first); - RangePartitionInfo localPartitionInfo = (RangePartitionInfo) localTbl.getPartitionInfo(); - RangePartitionInfo remotePartitionInfo = (RangePartitionInfo) remoteTbl.getPartitionInfo(); - BackupPartitionInfo backupPartitionInfo - = jobInfo.getOlapTableInfo(entry.first).getPartInfo(restoredPart.getName()); - long remotePartId = backupPartitionInfo.id; - Range remoteRange = remotePartitionInfo.getRange(remotePartId); - DataProperty remoteDataProperty = remotePartitionInfo.getDataProperty(remotePartId); - localPartitionInfo.addPartition(restoredPart.getId(), false, remoteRange, - remoteDataProperty, (short) restoreReplicationNum, - remotePartitionInfo.getIsInMemory(remotePartId)); + if (localTbl.getPartitionInfo().getType() == PartitionType.RANGE + || localTbl.getPartitionInfo().getType() == PartitionType.LIST) { + + PartitionInfo remotePartitionInfo = remoteTbl.getPartitionInfo(); + PartitionInfo localPartitionInfo = localTbl.getPartitionInfo(); + BackupPartitionInfo backupPartitionInfo + = jobInfo.getOlapTableInfo(entry.first).getPartInfo(restoredPart.getName()); + long remotePartId = backupPartitionInfo.id; + PartitionItem remoteItem = remoteTbl.getPartitionInfo().getItem(remotePartId); + DataProperty remoteDataProperty = remotePartitionInfo.getDataProperty(remotePartId); + localPartitionInfo.addPartition(restoredPart.getId(), false, remoteItem, + remoteDataProperty, (short) restoreReplicationNum, + remotePartitionInfo.getIsInMemory(remotePartId)); + } localTbl.addPartition(restoredPart); } finally { localTbl.writeUnlock(); @@ -958,7 +954,8 @@ private Partition resetPartitionForRestore(OlapTable localTbl, OlapTable remoteT Partition remotePart = remoteTbl.getPartition(partName); Preconditions.checkNotNull(remotePart); PartitionInfo localPartitionInfo = localTbl.getPartitionInfo(); - Preconditions.checkState(localPartitionInfo.getType() == PartitionType.RANGE); + Preconditions.checkState(localPartitionInfo.getType() == PartitionType.RANGE + || localPartitionInfo.getType() == PartitionType.LIST); // generate new partition id long newPartId = catalog.getNextId(); @@ -1074,13 +1071,12 @@ private void replayCheckAndPrepareMeta() { OlapTable localTbl = (OlapTable) db.getTable(entry.first); Partition restorePart = entry.second; OlapTable remoteTbl = (OlapTable) backupMeta.getTable(entry.first); - RangePartitionInfo localPartitionInfo = (RangePartitionInfo) localTbl.getPartitionInfo(); - RangePartitionInfo remotePartitionInfo = (RangePartitionInfo) remoteTbl.getPartitionInfo(); + PartitionInfo localPartitionInfo = localTbl.getPartitionInfo(); + PartitionInfo remotePartitionInfo = remoteTbl.getPartitionInfo(); BackupPartitionInfo backupPartitionInfo = jobInfo.getOlapTableInfo(entry.first).getPartInfo(restorePart.getName()); long remotePartId = backupPartitionInfo.id; - Range remoteRange = remotePartitionInfo.getRange(remotePartId); DataProperty remoteDataProperty = remotePartitionInfo.getDataProperty(remotePartId); - localPartitionInfo.addPartition(restorePart.getId(), false, remoteRange, + localPartitionInfo.addPartition(restorePart.getId(), false, remotePartitionInfo.getItem(remotePartId), remoteDataProperty, (short) restoreReplicationNum, remotePartitionInfo.getIsInMemory(remotePartId)); localTbl.addPartition(restorePart); @@ -1098,6 +1094,7 @@ private void replayCheckAndPrepareMeta() { } } } + // restored tables for (Table restoreTbl : restoredTbls) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index 16f24ebd3de8e5..3ae81163a9c19c 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -67,7 +67,6 @@ import org.apache.doris.analysis.MigrateDbStmt; import org.apache.doris.analysis.PartitionDesc; import org.apache.doris.analysis.PartitionRenameClause; -import org.apache.doris.analysis.RangePartitionDesc; import org.apache.doris.analysis.RecoverDbStmt; import org.apache.doris.analysis.RecoverPartitionStmt; import org.apache.doris.analysis.RecoverTableStmt; @@ -75,7 +74,7 @@ import org.apache.doris.analysis.RestoreStmt; import org.apache.doris.analysis.RollupRenameClause; import org.apache.doris.analysis.ShowAlterStmt.AlterType; -import org.apache.doris.analysis.SingleRangePartitionDesc; +import org.apache.doris.analysis.SinglePartitionDesc; import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.TableRef; import org.apache.doris.analysis.TableRenameClause; @@ -225,7 +224,6 @@ import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.collect.Queues; -import com.google.common.collect.Range; import com.google.common.collect.Sets; import com.sleepycat.je.rep.InsufficientLogException; import com.sleepycat.je.rep.NetworkRestore; @@ -3048,7 +3046,7 @@ public void createTableLike(CreateTableLikeStmt stmt) throws DdlException { } public void addPartition(Database db, String tableName, AddPartitionClause addPartitionClause) throws DdlException { - SingleRangePartitionDesc singlePartitionDesc = addPartitionClause.getSingeRangePartitionDesc(); + SinglePartitionDesc singlePartitionDesc = addPartitionClause.getSingeRangePartitionDesc(); DistributionDesc distributionDesc = addPartitionClause.getDistributionDesc(); boolean isTempPartition = addPartitionClause.isTempPartition(); @@ -3081,8 +3079,8 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa // check partition type PartitionInfo partitionInfo = olapTable.getPartitionInfo(); - if (partitionInfo.getType() != PartitionType.RANGE) { - throw new DdlException("Only support adding partition to range partitioned table"); + if (partitionInfo.getType() != PartitionType.RANGE && partitionInfo.getType() != PartitionType.LIST) { + throw new DdlException("Only support adding partition to range and list partitioned table"); } // check partition name @@ -3105,9 +3103,8 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa properties.put(PropertyAnalyzer.PROPERTIES_INMEMORY, olapTable.isInMemory().toString()); } - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; - singlePartitionDesc.analyze(rangePartitionInfo.getPartitionColumns().size(), properties); - rangePartitionInfo.checkAndCreateRange(singlePartitionDesc, isTempPartition); + singlePartitionDesc.analyze(partitionInfo.getPartitionColumns().size(), properties); + partitionInfo.createAndCheckPartitionItem(singlePartitionDesc, isTempPartition); // get distributionInfo List baseSchema = olapTable.getBaseSchema(); @@ -3235,13 +3232,12 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa // check partition type PartitionInfo partitionInfo = olapTable.getPartitionInfo(); - if (partitionInfo.getType() != PartitionType.RANGE) { - throw new DdlException("Only support adding partition to range partitioned table"); + if (partitionInfo.getType() != PartitionType.RANGE && partitionInfo.getType() != PartitionType.LIST) { + throw new DdlException("Only support adding partition to range and list partitioned table"); } // update partition info - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; - rangePartitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, partitionId, isTempPartition); + partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, partitionId, isTempPartition); if (isTempPartition) { olapTable.addTempPartition(partition); @@ -3250,11 +3246,23 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa } // log - PartitionPersistInfo info = new PartitionPersistInfo(db.getId(), olapTable.getId(), partition, - rangePartitionInfo.getRange(partitionId), dataProperty, - rangePartitionInfo.getReplicationNum(partitionId), - rangePartitionInfo.getIsInMemory(partitionId), - isTempPartition); + PartitionPersistInfo info = null; + if (partitionInfo.getType() == PartitionType.RANGE) { + info = new PartitionPersistInfo(db.getId(), olapTable.getId(), partition, + partitionInfo.getItem(partitionId).getItems(), + ListPartitionItem.DUMMY_ITEM, + dataProperty, + partitionInfo.getReplicationNum(partitionId), + partitionInfo.getIsInMemory(partitionId), + isTempPartition); + } else if (partitionInfo.getType() == PartitionType.LIST) { + info = new PartitionPersistInfo(db.getId(), olapTable.getId(), partition, + RangePartitionItem.DUMMY_ITEM, + partitionInfo.getItem(partitionId), dataProperty, + partitionInfo.getReplicationNum(partitionId), + partitionInfo.getIsInMemory(partitionId), + isTempPartition); + } editLog.logAddPartition(info); LOG.info("succeed in creating partition[{}], temp: {}", partitionId, isTempPartition); @@ -3282,9 +3290,15 @@ public void replayAddPartition(PartitionPersistInfo info) throws UserException { olapTable.addPartition(partition); } - ((RangePartitionInfo) partitionInfo).unprotectHandleNewSinglePartitionDesc(partition.getId(), - info.isTempPartition(), info.getRange(), info.getDataProperty(), info.getReplicationNum(), - info.isInMemory()); + PartitionItem partitionItem = null; + if (partitionInfo.getType() == PartitionType.RANGE) { + partitionItem = new RangePartitionItem(info.getRange()); + } else if (partitionInfo.getType() == PartitionType.LIST) { + partitionItem = info.getListPartitionItem(); + } + + partitionInfo.unprotectHandleNewSinglePartitionDesc(partition.getId(), info.isTempPartition(), + partitionItem, info.getDataProperty(), info.getReplicationNum(), info.isInMemory()); if (!isCheckpointThread()) { // add to inverted index @@ -3328,7 +3342,7 @@ public void dropPartition(Database db, OlapTable olapTable, DropPartitionClause } PartitionInfo partitionInfo = olapTable.getPartitionInfo(); - if (partitionInfo.getType() != PartitionType.RANGE) { + if (partitionInfo.getType() != PartitionType.RANGE && partitionInfo.getType() != PartitionType.LIST) { throw new DdlException("Alter table [" + olapTable.getName() + "] failed. Not a partitioned table"); } @@ -3528,12 +3542,10 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws DdlExcept Map partitionNameToId = Maps.newHashMap(); if (partitionDesc != null) { // gen partition id first - if (partitionDesc instanceof RangePartitionDesc) { - RangePartitionDesc rangeDesc = (RangePartitionDesc) partitionDesc; - for (SingleRangePartitionDesc desc : rangeDesc.getSingleRangePartitionDescs()) { - long partitionId = getNextId(); - partitionNameToId.put(desc.getPartitionName(), partitionId); - } + PartitionDesc partDesc = partitionDesc; + for (SinglePartitionDesc desc : partDesc.getSinglePartitionDescs()) { + long partitionId = getNextId(); + partitionNameToId.put(desc.getPartitionName(), partitionId); } partitionInfo = partitionDesc.toPartitionInfo(baseSchema, partitionNameToId, false); } else { @@ -3754,12 +3766,20 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws DdlExcept tabletIdSet, olapTable.getCopiedIndexes(), isInMemory, storageFormat, tabletType); olapTable.addPartition(partition); - } else if (partitionInfo.getType() == PartitionType.RANGE) { + } else if (partitionInfo.getType() == PartitionType.RANGE || partitionInfo.getType() == PartitionType.LIST) { try { // just for remove entries in stmt.getProperties(), // and then check if there still has unknown properties PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(), DataProperty.DEFAULT_DATA_PROPERTY); - DynamicPartitionUtil.checkAndSetDynamicPartitionProperty(olapTable, properties); + if (partitionInfo.getType() == PartitionType.RANGE) { + DynamicPartitionUtil.checkAndSetDynamicPartitionProperty(olapTable, properties); + + } else if (partitionInfo.getType() == PartitionType.LIST) { + if (DynamicPartitionUtil.checkDynamicPartitionPropertiesExist(properties)) { + throw new DdlException("Only support dynamic partition properties on range partition table"); + + } + } if (properties != null && !properties.isEmpty()) { // here, all properties should be checked @@ -3770,9 +3790,8 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws DdlExcept } // this is a 2-level partitioned tables - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; for (Map.Entry entry : partitionNameToId.entrySet()) { - DataProperty dataProperty = rangePartitionInfo.getDataProperty(entry.getValue()); + DataProperty dataProperty = partitionInfo.getDataProperty(entry.getValue()); Partition partition = createPartitionWithIndices(db.getClusterName(), db.getId(), olapTable.getId(), olapTable.getBaseIndexId(), entry.getValue(), entry.getKey(), olapTable.getIndexIdToMeta(), distributionInfo, @@ -3781,7 +3800,7 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws DdlExcept versionInfo, bfColumns, bfFpp, tabletIdSet, olapTable.getCopiedIndexes(), isInMemory, storageFormat, - rangePartitionInfo.getTabletType(entry.getValue())); + partitionInfo.getTabletType(entry.getValue())); olapTable.addPartition(partition); } } else { @@ -4000,7 +4019,7 @@ public static void getDdlStmt(String dbName, Table table, List createTab if (separatePartition) { partitionId = Lists.newArrayList(); } - if (partitionInfo.getType() == PartitionType.RANGE) { + if (partitionInfo.getType() == PartitionType.RANGE || partitionInfo.getType() == PartitionType.LIST) { sb.append("\n").append(partitionInfo.toSql(olapTable, partitionId)); } @@ -4167,28 +4186,38 @@ public static void getDdlStmt(String dbName, Table table, List createTab createTableStmt.add(sb.toString()); // 2. add partition - if (separatePartition && (table instanceof OlapTable) - && ((OlapTable) table).getPartitionInfo().getType() == PartitionType.RANGE - && ((OlapTable) table).getPartitions().size() > 1) { - OlapTable olapTable = (OlapTable) table; - RangePartitionInfo partitionInfo = (RangePartitionInfo) olapTable.getPartitionInfo(); - boolean first = true; - for (Map.Entry> entry : partitionInfo.getSortedRangeMap(false)) { - if (first) { - first = false; - continue; + if (separatePartition && (table instanceof OlapTable) && ((OlapTable) table).getPartitions().size() > 1) { + if (((OlapTable) table).getPartitionInfo().getType() == PartitionType.RANGE || + ((OlapTable) table).getPartitionInfo().getType() == PartitionType.LIST) { + OlapTable olapTable = (OlapTable) table; + PartitionInfo partitionInfo = olapTable.getPartitionInfo(); + boolean first = true; + for (Map.Entry entry : partitionInfo.getSortedItemMap(false)) { + if (first) { + first = false; + continue; + } + sb = new StringBuilder(); + Partition partition = olapTable.getPartition(entry.getKey()); + sb.append("ALTER TABLE ").append(table.getName()); + sb.append(" ADD PARTITION ").append(partition.getName()).append(" VALUES "); + if (partitionInfo.getType() == PartitionType.RANGE) { + sb.append("["); + sb.append(((RangePartitionItem) entry.getValue()).getItems().lowerEndpoint().toSql()); + sb.append(", "); + sb.append(((RangePartitionItem) entry.getValue()).getItems().upperEndpoint().toSql()); + sb.append(")"); + } else if (partitionInfo.getType() == PartitionType.LIST) { + sb.append("IN ("); + sb.append(((ListPartitionItem) entry.getValue()).toSql()); + sb.append(")"); + } + sb.append("(\"version_info\" = \""); + sb.append(Joiner.on(",").join(partition.getVisibleVersion(), partition.getVisibleVersionHash())) + .append("\""); + sb.append(");"); + addPartitionStmt.add(sb.toString()); } - sb = new StringBuilder(); - Partition partition = olapTable.getPartition(entry.getKey()); - sb.append("ALTER TABLE ").append(table.getName()); - sb.append(" ADD PARTITION ").append(partition.getName()).append(" VALUES ["); - sb.append(entry.getValue().lowerEndpoint().toSql()); - sb.append(", ").append(entry.getValue().upperEndpoint().toSql()).append(")"); - sb.append("(\"version_info\" = \""); - sb.append(Joiner.on(",").join(partition.getVisibleVersion(), partition.getVisibleVersionHash())) - .append("\""); - sb.append(");"); - addPartitionStmt.add(sb.toString()); } } @@ -5128,7 +5157,7 @@ public void modifyTableColocate(Database db, OlapTable table, String colocateGro // user set a new colocate group, // check if all partitions all this table has same buckets num and same replication number PartitionInfo partitionInfo = table.getPartitionInfo(); - if (partitionInfo.getType() == PartitionType.RANGE) { + if (partitionInfo.getType() == PartitionType.RANGE || partitionInfo.getType() == PartitionType.LIST) { int bucketsNum = -1; short replicationNum = -1; for (Partition partition : table.getPartitions()) { @@ -5279,7 +5308,7 @@ public void renamePartition(Database db, OlapTable table, PartitionRenameClause throw new DdlException("Table[" + table.getName() + "] is under " + table.getState()); } - if (table.getPartitionInfo().getType() != PartitionType.RANGE) { + if (table.getPartitionInfo().getType() != PartitionType.RANGE && table.getPartitionInfo().getType() != PartitionType.LIST) { throw new DdlException("Table[" + table.getName() + "] is single partitioned. " + "no need to rename partition name."); } @@ -5370,8 +5399,8 @@ public void modifyTableReplicationNum(Database db, OlapTable table, Map range, DataProperty dataProperty, + Range range, PartitionItem listPartitionItem, + DataProperty dataProperty, short replicationNum, boolean isInMemory) { if (idToPartition.containsKey(partition.getId())) { @@ -119,7 +120,7 @@ public synchronized boolean recyclePartition(long dbId, long tableId, Partition // recycle partition RecyclePartitionInfo partitionInfo = new RecyclePartitionInfo(dbId, tableId, partition, - range, dataProperty, replicationNum, + range, listPartitionItem, dataProperty, replicationNum, isInMemory); idToRecycleTime.put(partition.getId(), System.currentTimeMillis()); idToPartition.put(partition.getId(), partitionInfo); @@ -426,22 +427,28 @@ public synchronized void recoverPartition(long dbId, OlapTable table, String par if (recoverPartitionInfo == null) { throw new DdlException("No partition named " + partitionName + " in table " + table.getName()); } - - // check if range is invalid + + PartitionInfo partitionInfo = table.getPartitionInfo(); Range recoverRange = recoverPartitionInfo.getRange(); - RangePartitionInfo partitionInfo = (RangePartitionInfo) table.getPartitionInfo(); - if (partitionInfo.getAnyIntersectRange(recoverRange, false) != null) { - throw new DdlException("Can not recover partition[" + partitionName + "]. Range conflict."); + PartitionItem recoverItem = null; + if (partitionInfo.getType() == PartitionType.RANGE) { + recoverItem = new RangePartitionItem(recoverRange); + } else if (partitionInfo.getType() == PartitionType.LIST) { + recoverItem = recoverPartitionInfo.getListPartitionItem();; + } + // check if partition item is invalid + if (partitionInfo.getAnyIntersectItem(recoverItem, false) != null) { + throw new DdlException("Can not recover partition[" + partitionName + "]. Partition item conflict."); } // recover partition Partition recoverPartition = recoverPartitionInfo.getPartition(); Preconditions.checkState(recoverPartition.getName().equalsIgnoreCase(partitionName)); table.addPartition(recoverPartition); - + // recover partition info long partitionId = recoverPartition.getId(); - partitionInfo.setRange(partitionId, false, recoverRange); + partitionInfo.setItem(partitionId, false, recoverItem); partitionInfo.setDataProperty(partitionId, recoverPartitionInfo.getDataProperty()); partitionInfo.setReplicationNum(partitionId, recoverPartitionInfo.getReplicationNum()); partitionInfo.setIsInMemory(partitionId, recoverPartitionInfo.isInMemory()); @@ -461,19 +468,25 @@ public synchronized void replayRecoverPartition(OlapTable table, long partitionI Iterator> iterator = idToPartition.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry entry = iterator.next(); - RecyclePartitionInfo partitionInfo = entry.getValue(); - if (partitionInfo.getPartition().getId() != partitionId) { + RecyclePartitionInfo recyclePartitionInfo = entry.getValue(); + if (recyclePartitionInfo.getPartition().getId() != partitionId) { continue; } - Preconditions.checkState(partitionInfo.getTableId() == table.getId()); + Preconditions.checkState(recyclePartitionInfo.getTableId() == table.getId()); - table.addPartition(partitionInfo.getPartition()); - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) table.getPartitionInfo(); - rangePartitionInfo.setRange(partitionId, false, partitionInfo.getRange()); - rangePartitionInfo.setDataProperty(partitionId, partitionInfo.getDataProperty()); - rangePartitionInfo.setReplicationNum(partitionId, partitionInfo.getReplicationNum()); - rangePartitionInfo.setIsInMemory(partitionId, partitionInfo.isInMemory()); + table.addPartition(recyclePartitionInfo.getPartition()); + PartitionInfo partitionInfo = table.getPartitionInfo(); + PartitionItem recoverItem = null; + if (partitionInfo.getType() == PartitionType.RANGE) { + recoverItem = new RangePartitionItem(recyclePartitionInfo.getRange()); + } else if (partitionInfo.getType() == PartitionType.LIST) { + recoverItem = recyclePartitionInfo.getListPartitionItem(); + } + partitionInfo.setItem(partitionId, false, recoverItem); + partitionInfo.setDataProperty(partitionId, recyclePartitionInfo.getDataProperty()); + partitionInfo.setReplicationNum(partitionId, recyclePartitionInfo.getReplicationNum()); + partitionInfo.setIsInMemory(partitionId, recyclePartitionInfo.isInMemory()); iterator.remove(); idToRecycleTime.remove(partitionId); @@ -725,6 +738,7 @@ public class RecyclePartitionInfo implements Writable { private long tableId; private Partition partition; private Range range; + private PartitionItem listPartitionItem; private DataProperty dataProperty; private short replicationNum; private boolean isInMemory; @@ -734,12 +748,14 @@ public RecyclePartitionInfo() { } public RecyclePartitionInfo(long dbId, long tableId, Partition partition, - Range range, DataProperty dataProperty, short replicationNum, + Range range, PartitionItem listPartitionItem, + DataProperty dataProperty, short replicationNum, boolean isInMemory) { this.dbId = dbId; this.tableId = tableId; this.partition = partition; this.range = range; + this.listPartitionItem = listPartitionItem; this.dataProperty = dataProperty; this.replicationNum = replicationNum; this.isInMemory = isInMemory; @@ -761,6 +777,10 @@ public Range getRange() { return range; } + public PartitionItem getListPartitionItem() { + return listPartitionItem; + } + public DataProperty getDataProperty() { return dataProperty; } @@ -779,6 +799,7 @@ public void write(DataOutput out) throws IOException { out.writeLong(tableId); partition.write(out); RangeUtils.writeRange(out, range); + listPartitionItem.write(out); dataProperty.write(out); out.writeShort(replicationNum); out.writeBoolean(isInMemory); @@ -789,6 +810,12 @@ public void readFields(DataInput in) throws IOException { tableId = in.readLong(); partition = Partition.read(in); range = RangeUtils.readRange(in); + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_98) { + listPartitionItem = ListPartitionItem.read(in); + } else { + listPartitionItem = ListPartitionItem.DUMMY_ITEM; + } + dataProperty = DataProperty.read(in); replicationNum = in.readShort(); if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_72) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java new file mode 100644 index 00000000000000..87141450c0c5fe --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.analysis.PartitionKeyDesc; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.analysis.SinglePartitionDesc; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.util.ListUtil; + +import com.google.common.base.Preconditions; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class ListPartitionInfo extends PartitionInfo{ + + public ListPartitionInfo() { + // for persist + super(); + } + + public ListPartitionInfo(List partitionColumns) { + super(PartitionType.LIST); + this.partitionColumns = partitionColumns; + this.isMultiColumnPartition = partitionColumns.size() > 1; + } + + public static PartitionInfo read(DataInput in) throws IOException { + PartitionInfo partitionInfo = new ListPartitionInfo(); + partitionInfo.readFields(in); + return partitionInfo; + } + + public PartitionItem createAndCheckPartitionItem(SinglePartitionDesc desc, boolean isTemp) throws DdlException { + // get partition key + PartitionKeyDesc partitionKeyDesc = desc.getPartitionKeyDesc(); + + for (List values : partitionKeyDesc.getInValues()) { + Preconditions.checkArgument(values.size() == partitionColumns.size(), + "partition key desc list size[" + values.size() + "] is not equal to " + + "partition column size[" + partitionColumns.size() + "]"); + } + List partitionKeys = new ArrayList<>(); + try { + for (List values : partitionKeyDesc.getInValues()) { + PartitionKey partitionKey = PartitionKey.createListPartitionKey(values, partitionColumns); + checkNewPartitionKey(partitionKey, partitionKeyDesc, isTemp); + partitionKeys.add(partitionKey); + } + } catch (AnalysisException e) { + throw new DdlException("Invalid list value format: " + e.getMessage()); + } + return new ListPartitionItem(partitionKeys); + } + + private void checkNewPartitionKey(PartitionKey newKey, PartitionKeyDesc keyDesc, boolean isTemp) throws AnalysisException { + Map id2Item = idToItem; + if (isTemp) { + id2Item = idToTempItem; + } + // check new partition key not exists. + for (Map.Entry entry : id2Item.entrySet()) { + if (((ListPartitionItem)entry.getValue()).getItems().contains(newKey)) { + throw new AnalysisException("The partition key[" + keyDesc.toSql() + "] is overlap with current " + + entry.getValue().toString()); + } + } + } + + @Override + public void checkPartitionItemListsMatch(List list1, List list2) throws DdlException { + ListUtil.checkPartitionKeyListsMatch(list1, list2); + } + + @Override + public void checkPartitionItemListsConflict(List list1, List list2) throws DdlException { + ListUtil.checkListsConflict(list1, list2); + } + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + // partition columns + out.writeInt(partitionColumns.size()); + for (Column column : partitionColumns) { + column.write(out); + } + + out.writeInt(idToItem.size()); + for (Map.Entry entry : idToItem.entrySet()) { + out.writeLong(entry.getKey()); + entry.getValue().write(out); + } + + out.writeInt(idToTempItem.size()); + for (Map.Entry entry : idToTempItem.entrySet()) { + out.writeLong(entry.getKey()); + entry.getValue().write(out); + } + + } + + public void readFields(DataInput in) throws IOException { + super.readFields(in); + + int counter = in.readInt(); + for (int i = 0; i < counter; i++) { + Column column = Column.read(in); + partitionColumns.add(column); + } + + this.isMultiColumnPartition = partitionColumns.size() > 1; + + counter = in.readInt(); + for (int i = 0; i < counter; i++) { + long partitionId = in.readLong(); + ListPartitionItem partitionItem = ListPartitionItem.read(in); + idToItem.put(partitionId, partitionItem); + } + + counter = in.readInt(); + for (int i = 0; i < counter; i++) { + long partitionId = in.readLong(); + ListPartitionItem partitionItem = ListPartitionItem.read(in); + idToTempItem.put(partitionId, partitionItem); + } + } + + public static void checkPartitionColumn(Column column) throws AnalysisException { + PrimitiveType type = column.getDataType(); + if (!type.isFixedPointType() && !type.isDateType() + && !type.isCharFamily() && type != PrimitiveType.BOOLEAN) { + throw new AnalysisException("Column[" + column.getName() + "] type[" + type + + "] cannot be a list partition key."); + } + } + + @Override + public String toSql(OlapTable table, List partitionId) { + StringBuilder sb = new StringBuilder(); + sb.append("PARTITION BY LIST("); + int idx = 0; + for (Column column : partitionColumns) { + if (idx != 0) { + sb.append(", "); + } + sb.append("`").append(column.getName()).append("`"); + idx++; + } + sb.append(")\n("); + + // sort list + List> entries = new ArrayList<>(this.idToItem.entrySet()); + Collections.sort(entries, ListUtil.LIST_MAP_ENTRY_COMPARATOR); + idx = 0; + for (Map.Entry entry : entries) { + Partition partition = table.getPartition(entry.getKey()); + String partitionName = partition.getName(); + List partitionKeys = entry.getValue().getItems(); + + sb.append("PARTITION ").append(partitionName).append(" VALUES IN "); + sb.append("("); + int idxInternal = 0; + for (PartitionKey partitionKey : partitionKeys) { + String partitionKeyStr = partitionKey.toSql(); + if (!isMultiColumnPartition) { + partitionKeyStr = partitionKeyStr.substring(1, partitionKeyStr.length() - 1); + } + sb.append(partitionKeyStr); + if (partitionKeys.size() > 1 && idxInternal != partitionKeys.size() - 1) { + sb.append(","); + } + idxInternal++; + } + sb.append(")"); + + if (partitionId != null) { + partitionId.add(entry.getKey()); + break; + } + + if (idx != entries.size() - 1) { + sb.append(",\n"); + } + idx++; + } + sb.append(")"); + return sb.toString(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java new file mode 100644 index 00000000000000..d74d76306bd9f3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import com.clearspring.analytics.util.Lists; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class ListPartitionItem extends PartitionItem { + private List partitionKeys; + + public static ListPartitionItem DUMMY_ITEM = new ListPartitionItem(Lists.newArrayList()); + + public ListPartitionItem(List partitionKeys) { + this.partitionKeys = partitionKeys; + } + + public List getItems() { + return partitionKeys; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(partitionKeys.size()); + for (PartitionKey partitionKey : partitionKeys) { + partitionKey.write(out); + } + } + + public static ListPartitionItem read(DataInput input) throws IOException { + int counter = input.readInt(); + List partitionKeys = new ArrayList<>(); + for (int i = 0; i < counter; i++) { + PartitionKey partitionKey = PartitionKey.read(input); + partitionKeys.add(partitionKey); + } + return new ListPartitionItem(partitionKeys); + } + + @Override + public int compareTo(PartitionItem other) { + int thisKeyLen = this.partitionKeys.size(); + int otherKeyLen = ((ListPartitionItem) other).getItems().size(); + int minLen = Math.min(thisKeyLen, otherKeyLen); + for (int i = 0; i < minLen; i++) { + int ret = this.getItems().get(i).compareTo(((ListPartitionItem) other).getItems().get(i)); + if (0 != ret) { + return ret; + } + } + return Integer.compare(thisKeyLen, otherKeyLen); + } + + @Override + public PartitionItem getIntersect(PartitionItem newItem) { + List newKeys = newItem.getItems(); + for (PartitionKey newKey : newKeys) { + if (partitionKeys.contains(newKey)) { + return newItem; + } + } + return null; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (!(obj instanceof ListPartitionItem)) { + return false; + } + + ListPartitionItem partitionItem = (ListPartitionItem) obj; + // check keys + if (partitionKeys != partitionItem.getItems()) { + if (partitionKeys.size() != partitionItem.getItems().size()) { + return false; + } + for (int i = 0; i < partitionKeys.size(); i++) { + if (!partitionKeys.get(i).equals(partitionItem.getItems().get(i))) { + return false; + } + } + } + + return true; + } + + @Override + public int hashCode() { + int result = 17; + result = 31 * result + partitionKeys.size(); + for (PartitionKey partitionKey : partitionKeys) { + result = 31 * result + partitionKey.hashCode(); + } + return result; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("partitionKeys: ["); + for (PartitionKey partitionKey : partitionKeys) { + builder.append(partitionKey.toString()); + } + builder.append("]; "); + return builder.toString(); + } + + public String toSql() { + StringBuilder sb = new StringBuilder(); + int size = partitionKeys.size(); + if (size > 1) { + sb.append("("); + } + + for (PartitionKey partitionKey : partitionKeys) { + sb.append(partitionKey.toSql()); + } + + if (size > 1) { + sb.append(")"); + } + + return sb.toString(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 0e9a2da040ba89..72b0d9cb0ae8fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -34,6 +34,7 @@ import org.apache.doris.catalog.Tablet.TabletStatus; import org.apache.doris.clone.TabletSchedCtx; import org.apache.doris.clone.TabletScheduler; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeMetaVersion; @@ -41,7 +42,6 @@ import org.apache.doris.common.io.DeepCopy; import org.apache.doris.common.io.Text; import org.apache.doris.common.util.PropertyAnalyzer; -import org.apache.doris.common.util.RangeUtils; import org.apache.doris.common.util.Util; import org.apache.doris.qe.OriginStatement; import org.apache.doris.system.SystemInfoService; @@ -52,16 +52,16 @@ import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Range; import com.google.common.collect.Sets; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -428,19 +428,16 @@ public Status resetIdsForRestore(Catalog catalog, Database db, int restoreReplic } // reset partition info and idToPartition map - if (partitionInfo.getType() == PartitionType.RANGE) { - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; + if (partitionInfo.getType() == PartitionType.RANGE || partitionInfo.getType() == PartitionType.LIST) { for (Map.Entry entry : origPartNameToId.entrySet()) { long newPartId = catalog.getNextId(); - rangePartitionInfo.idToDataProperty.put(newPartId, - rangePartitionInfo.idToDataProperty.remove(entry.getValue())); - rangePartitionInfo.idToReplicationNum.remove(entry.getValue()); - rangePartitionInfo.idToReplicationNum.put(newPartId, - (short) restoreReplicationNum); - rangePartitionInfo.getIdToRange(false).put(newPartId, - rangePartitionInfo.getIdToRange(false).remove(entry.getValue())); - - rangePartitionInfo.idToInMemory.put(newPartId, rangePartitionInfo.idToInMemory.remove(entry.getValue())); + partitionInfo.idToDataProperty.put(newPartId, + partitionInfo.idToDataProperty.remove(entry.getValue())); + partitionInfo.idToReplicationNum.remove(entry.getValue()); + partitionInfo.idToReplicationNum.put(newPartId, (short) restoreReplicationNum); + partitionInfo.getIdToItem(false).put(newPartId, + partitionInfo.getIdToItem(false).remove(entry.getValue())); + partitionInfo.idToInMemory.put(newPartId, partitionInfo.idToInMemory.remove(entry.getValue())); idToPartition.put(newPartId, idToPartition.remove(entry.getValue())); } } else { @@ -681,22 +678,44 @@ private Partition dropPartition(long dbId, String partitionName, boolean isForce idToPartition.remove(partition.getId()); nameToPartition.remove(partitionName); - Preconditions.checkState(partitionInfo.getType() == PartitionType.RANGE); - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; + Preconditions.checkState(partitionInfo.getType() == PartitionType.RANGE || + partitionInfo.getType() == PartitionType.LIST); if (!isForceDrop) { // recycle partition - Catalog.getCurrentRecycleBin().recyclePartition(dbId, id, partition, - rangePartitionInfo.getRange(partition.getId()), - rangePartitionInfo.getDataProperty(partition.getId()), - rangePartitionInfo.getReplicationNum(partition.getId()), - rangePartitionInfo.getIsInMemory(partition.getId())); + if (partitionInfo.getType() == PartitionType.RANGE) { + Catalog.getCurrentRecycleBin().recyclePartition(dbId, id, partition, + partitionInfo.getItem(partition.getId()).getItems(), + new ListPartitionItem(Lists.newArrayList(new PartitionKey())), + partitionInfo.getDataProperty(partition.getId()), + partitionInfo.getReplicationNum(partition.getId()), + partitionInfo.getIsInMemory(partition.getId())); + + } else if (partitionInfo.getType() == PartitionType.LIST) { + // construct a dummy range + List dummyColumns = new ArrayList<>(); + dummyColumns.add(new Column("dummy", PrimitiveType.INT)); + PartitionKey dummyKey = null; + try { + dummyKey = PartitionKey.createInfinityPartitionKey(dummyColumns, false); + } catch (AnalysisException e) { + e.printStackTrace(); + } + Range dummyRange = Range.open(new PartitionKey(), dummyKey); + + Catalog.getCurrentRecycleBin().recyclePartition(dbId, id, partition, + dummyRange, + partitionInfo.getItem(partition.getId()), + partitionInfo.getDataProperty(partition.getId()), + partitionInfo.getReplicationNum(partition.getId()), + partitionInfo.getIsInMemory(partition.getId())); + } } else if (!reserveTablets) { Catalog.getCurrentCatalog().onErasePartition(partition); } // drop partition info - rangePartitionInfo.dropPartition(partition.getId()); + partitionInfo.dropPartition(partition.getId()); } return partition; } @@ -1133,7 +1152,9 @@ public void readFields(DataInput in) throws IOException { partitionInfo = SinglePartitionInfo.read(in); } else if (partType == PartitionType.RANGE) { partitionInfo = RangePartitionInfo.read(in); - } else { + } else if (partType == PartitionType.LIST) { + partitionInfo = ListPartitionInfo.read(in); + }else { throw new IOException("invalid partition type: " + partType); } @@ -1197,9 +1218,9 @@ public void readFields(DataInput in) throws IOException { if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_77) { RangePartitionInfo tempRangeInfo = tempPartitions.getPartitionInfo(); if (tempRangeInfo != null) { - for (long partitionId : tempRangeInfo.getIdToRange(false).keySet()) { - ((RangePartitionInfo) this.partitionInfo).addPartition(partitionId, true, - tempRangeInfo.getRange(partitionId), tempRangeInfo.getDataProperty(partitionId), + for (long partitionId : tempRangeInfo.getIdToItem(false).keySet()) { + this.partitionInfo.addPartition(partitionId, true, + tempRangeInfo.getItem(partitionId), tempRangeInfo.getDataProperty(partitionId), tempRangeInfo.getReplicationNum(partitionId), tempRangeInfo.getIsInMemory(partitionId)); } } @@ -1288,11 +1309,11 @@ public Partition replacePartition(Partition newPartition) { short replicationNum = partitionInfo.getReplicationNum(oldPartition.getId()); boolean isInMemory = partitionInfo.getIsInMemory(oldPartition.getId()); - if (partitionInfo.getType() == PartitionType.RANGE) { - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; - Range range = rangePartitionInfo.getRange(oldPartition.getId()); - rangePartitionInfo.dropPartition(oldPartition.getId()); - rangePartitionInfo.addPartition(newPartition.getId(), false, range, dataProperty, + if (partitionInfo.getType() == PartitionType.RANGE + || partitionInfo.getType() == PartitionType.LIST) { + PartitionItem item = partitionInfo.getItem(oldPartition.getId()); + partitionInfo.dropPartition(oldPartition.getId()); + partitionInfo.addPartition(newPartition.getId(), false, item, dataProperty, replicationNum, isInMemory); } else { partitionInfo.dropPartition(oldPartition.getId()); @@ -1533,41 +1554,8 @@ public void dropTempPartition(String partitionName, boolean needDropTablet) { */ public void replaceTempPartitions(List partitionNames, List tempPartitionNames, boolean strictRange, boolean useTempPartitionName) throws DdlException { - RangePartitionInfo rangeInfo = (RangePartitionInfo) partitionInfo; - - if (strictRange) { - // check if range of partitions and temp partitions are exactly same - List> rangeList = Lists.newArrayList(); - List> tempRangeList = Lists.newArrayList(); - for (String partName : partitionNames) { - Partition partition = nameToPartition.get(partName); - Preconditions.checkNotNull(partition); - rangeList.add(rangeInfo.getRange(partition.getId())); - } - - for (String partName : tempPartitionNames) { - Partition partition = tempPartitions.getPartition(partName); - Preconditions.checkNotNull(partition); - tempRangeList.add(rangeInfo.getRange(partition.getId())); - } - RangeUtils.checkRangeListsMatch(rangeList, tempRangeList); - } else { - // check after replacing, whether the range will conflict - Set replacePartitionIds = Sets.newHashSet(); - for (String partName : partitionNames) { - Partition partition = nameToPartition.get(partName); - Preconditions.checkNotNull(partition); - replacePartitionIds.add(partition.getId()); - } - List> replacePartitionRanges = Lists.newArrayList(); - for (String partName : tempPartitionNames) { - Partition partition = tempPartitions.getPartition(partName); - Preconditions.checkNotNull(partition); - replacePartitionRanges.add(rangeInfo.getRange(partition.getId())); - } - List> sortedRangeList = rangeInfo.getRangeList(replacePartitionIds, false); - RangeUtils.checkRangeConflict(sortedRangeList, replacePartitionRanges); - } + // check partition items + checkPartition(partitionNames, tempPartitionNames, strictRange); // begin to replace // 1. drop old partitions @@ -1584,7 +1572,7 @@ public void replaceTempPartitions(List partitionNames, List temp // drop tempPartitions.dropPartition(partitionName, false); // move the range from idToTempRange to idToRange - rangeInfo.moveRangeFromTempToFormal(partition.getId()); + partitionInfo.moveFromTempToFormal(partition.getId()); } // change the name so that after replacing, the partition name remain unchanged @@ -1595,6 +1583,44 @@ public void replaceTempPartitions(List partitionNames, List temp } } + private void checkPartition(List partitionNames, List tempPartitionNames, + boolean strictRange) throws DdlException { + if (strictRange) { + List list = Lists.newArrayList(); + List tempList = Lists.newArrayList(); + for (String partName : partitionNames) { + Partition partition = nameToPartition.get(partName); + Preconditions.checkNotNull(partition); + list.add(partitionInfo.getItem(partition.getId())); + } + for (String partName : tempPartitionNames) { + Partition partition = tempPartitions.getPartition(partName); + Preconditions.checkNotNull(partition); + tempList.add(partitionInfo.getItem(partition.getId())); + } + partitionInfo.checkPartitionItemListsMatch(list, tempList); + } else { + // check after replacing, whether the range will conflict + Set replacePartitionIds = Sets.newHashSet(); + for (String partName : partitionNames) { + Partition partition = nameToPartition.get(partName); + Preconditions.checkNotNull(partition); + replacePartitionIds.add(partition.getId()); + } + // get all items except for partitions in "replacePartitionIds" + List currentItemList = partitionInfo.getItemList(replacePartitionIds, false); + + List replacePartitionItems = Lists.newArrayList(); + for (String partName : tempPartitionNames) { + Partition partition = tempPartitions.getPartition(partName); + Preconditions.checkNotNull(partition); + replacePartitionItems.add(partitionInfo.getItem(partition.getId())); + } + + partitionInfo.checkPartitionItemListsConflict(currentItemList, replacePartitionItems); + } + } + public void addTempPartition(Partition partition) { tempPartitions.addPartition(partition); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java index 575525b91805c4..b64b341cb75cd8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java @@ -17,23 +17,29 @@ package org.apache.doris.catalog; +import org.apache.doris.analysis.SinglePartitionDesc; +import org.apache.doris.common.DdlException; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.thrift.TStorageMedium; +import org.apache.doris.thrift.TTabletType; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; -import org.apache.doris.thrift.TStorageMedium; -import org.apache.doris.thrift.TTabletType; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; /* * Repository of a partition's related infos @@ -42,6 +48,12 @@ public class PartitionInfo implements Writable { private static final Logger LOG = LogManager.getLogger(PartitionInfo.class); protected PartitionType type; + // partition columns for list and range partitions + protected List partitionColumns = Lists.newArrayList(); + // formal partition id -> partition item + protected Map idToItem = Maps.newHashMap(); + // temp partition id -> partition item + protected Map idToTempItem = Maps.newHashMap(); // partition id -> data property protected Map idToDataProperty; // partition id -> replication num @@ -71,10 +83,122 @@ public PartitionInfo(PartitionType type) { this.idToTabletType = new HashMap<>(); } + public PartitionInfo(PartitionType type, List partitionColumns) { + this(type); + this.partitionColumns = partitionColumns; + this.isMultiColumnPartition = partitionColumns.size() > 1; + } + public PartitionType getType() { return type; } + public List getPartitionColumns(){ + return partitionColumns; + } + + public Map getIdToItem(boolean isTemp) { + if (isTemp) { + return idToTempItem; + } else { + return idToItem; + } + } + + public PartitionItem getItem(long partitionId) { + PartitionItem item = idToItem.get(partitionId); + if (item == null) { + item = idToTempItem.get(partitionId); + } + return item; + } + + public void setItem(long partitionId, boolean isTemp, PartitionItem item) { + setItemInternal(partitionId, isTemp, item); + } + + private void setItemInternal(long partitionId, boolean isTemp, PartitionItem item) { + if (isTemp) { + idToTempItem.put(partitionId, item); + } else { + idToItem.put(partitionId, item); + } + } + + public PartitionItem handleNewSinglePartitionDesc(SinglePartitionDesc desc, + long partitionId, boolean isTemp) throws DdlException { + Preconditions.checkArgument(desc.isAnalyzed()); + PartitionItem partitionItem = createAndCheckPartitionItem(desc, isTemp); + setItemInternal(partitionId, isTemp, partitionItem); + + idToDataProperty.put(partitionId, desc.getPartitionDataProperty()); + idToReplicationNum.put(partitionId, desc.getReplicationNum()); + idToInMemory.put(partitionId, desc.isInMemory()); + + return partitionItem; + } + + public PartitionItem createAndCheckPartitionItem(SinglePartitionDesc desc, boolean isTemp) throws DdlException { + return null; + } + + public void unprotectHandleNewSinglePartitionDesc(long partitionId, boolean isTemp, PartitionItem partitionItem, + DataProperty dataProperty, short replicationNum, + boolean isInMemory) { + setItemInternal(partitionId, isTemp, partitionItem); + idToDataProperty.put(partitionId, dataProperty); + idToReplicationNum.put(partitionId, replicationNum); + idToInMemory.put(partitionId, isInMemory); + } + + public List> getSortedItemMap(boolean isTemp) { + Map tmpMap = idToItem; + if (isTemp) { + tmpMap = idToTempItem; + } + List> sortedList = Lists.newArrayList(tmpMap.entrySet()); + Collections.sort(sortedList, PartitionItem.ITEM_MAP_ENTRY_COMPARATOR); + return sortedList; + } + + // get sorted item list, exclude partitions which ids are in 'excludePartitionIds' + public List getItemList(Set excludePartitionIds, boolean isTemp) { + Map tempMap = idToItem; + if (isTemp) { + tempMap = idToTempItem; + } + List resultList = Lists.newArrayList(); + for (Map.Entry entry : tempMap.entrySet()) { + if (!excludePartitionIds.contains(entry.getKey())) { + resultList.add(entry.getValue()); + } + } + return resultList; + } + + // return any item intersect with the newItem. + // return null if no item intersect. + public PartitionItem getAnyIntersectItem(PartitionItem newItem, boolean isTemp) { + Map tmpMap = idToItem; + if (isTemp) { + tmpMap = idToTempItem; + } + PartitionItem retItem; + for (PartitionItem item : tmpMap.values()) { + retItem = item.getIntersect(newItem); + if (null != retItem) { + return retItem; + } + } + return null; + } + + public void checkPartitionItemListsMatch(List list1, List list2) throws DdlException { + } + + public void checkPartitionItemListsConflict(List list1, List list2) throws DdlException { + } + public DataProperty getDataProperty(long partitionId) { return idToDataProperty.get(partitionId); } @@ -117,6 +241,14 @@ public void dropPartition(long partitionId) { idToDataProperty.remove(partitionId); idToReplicationNum.remove(partitionId); idToInMemory.remove(partitionId); + idToItem.remove(partitionId); + idToTempItem.remove(partitionId); + } + + public void addPartition(long partitionId, boolean isTemp, PartitionItem item, DataProperty dataProperty, + short replicationNum, boolean isInMemory){ + addPartition(partitionId, dataProperty, replicationNum, isInMemory); + setItemInternal(partitionId, isTemp, item); } public void addPartition(long partitionId, DataProperty dataProperty, @@ -141,6 +273,13 @@ public String toSql(OlapTable table, List partitionId) { return ""; } + public void moveFromTempToFormal(long tempPartitionId) { + PartitionItem item = idToTempItem.remove(tempPartitionId); + if (item != null) { + idToItem.put(tempPartitionId, item); + } + } + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, type.name()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionItem.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionItem.java new file mode 100644 index 00000000000000..f71b8c60b0dd1a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionItem.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.common.io.Writable; + +import java.util.Comparator; +import java.util.Map; + +public abstract class PartitionItem implements Comparable,Writable { + public static final Comparator> ITEM_MAP_ENTRY_COMPARATOR = + Comparator.comparing(o -> ((ListPartitionItem) o.getValue()).getItems().iterator().next()); + + public abstract T getItems(); + + public abstract PartitionItem getIntersect(PartitionItem newItem); + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java index 3fd0732a91ebfd..e0aec6166db520 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java @@ -17,12 +17,14 @@ package org.apache.doris.catalog; +import org.apache.doris.analysis.BoolLiteral; import org.apache.doris.analysis.DateLiteral; import org.apache.doris.analysis.IntLiteral; import org.apache.doris.analysis.LargeIntLiteral; import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.analysis.MaxLiteral; import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.analysis.StringLiteral; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; @@ -85,6 +87,40 @@ public static PartitionKey createPartitionKey(List keys, List values, List columns) + throws AnalysisException { + // for multi list partition: + // + // PARTITION BY LIST(k1, k2) + // ( + // PARTITION p1 VALUES IN (("1","beijing"), ("1", "shanghai")), + // PARTITION p2 VALUES IN (("2","shanghai")) + // ) + // + // for single list partition: + // + // PARTITION BY LIST(`k1`) + // ( + // PARTITION p1 VALUES IN ("1", "2", "3", "4", "5"), + // PARTITION p2 VALUES IN ("6", "7", "8", "9", "10"), + // PARTITION p3 VALUES IN ("11", "12", "13", "14", "15"), + // PARTITION p4 VALUES IN ("16", "17", "18", "19", "20"), + // PARTITION p5 VALUES IN ("21", "22", "23", "24", "25"), + // PARTITION p6 VALUES IN ("26") + // ) + // + Preconditions.checkArgument(values.size() == columns.size(), + "in value size[" + values.size() + "] is not equal to partition column size[" + columns.size() + "]."); + + PartitionKey partitionKey = new PartitionKey(); + for (int i = 0; i < values.size(); i++) { + partitionKey.keys.add(values.get(i).getValue(Type.fromPrimitiveType(columns.get(i).getDataType()))); + partitionKey.types.add(columns.get(i).getDataType()); + } + + return partitionKey; + } + public void pushColumn(LiteralExpr keyValue, PrimitiveType keyType) { keys.add(keyValue); types.add(keyType); @@ -271,6 +307,13 @@ public void readFields(DataInput in) throws IOException { case DATETIME: literal = DateLiteral.read(in); break; + case CHAR: + case VARCHAR: + literal = StringLiteral.read(in); + break; + case BOOLEAN: + literal = BoolLiteral.read(in); + break; default: throw new IOException("type[" + type.name() + "] not supported: "); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionType.java index f35e200c3f8df1..f148f0f2591423 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionType.java @@ -21,7 +21,8 @@ public enum PartitionType { UNPARTITIONED("UNPARTITIONED"), - RANGE("RANGE"); + RANGE("RANGE"), + LIST("LIST"); public String typeString; @@ -35,6 +36,8 @@ public static PartitionType fromThrift(TPartitionType tType) { return UNPARTITIONED; case RANGE_PARTITIONED: return RANGE; + case LIST_PARTITIONED: + return LIST; default: return UNPARTITIONED; } @@ -46,6 +49,8 @@ public TPartitionType toThrift() { return TPartitionType.UNPARTITIONED; case RANGE: return TPartitionType.RANGE_PARTITIONED; + case LIST: + return TPartitionType.LIST_PARTITIONED; default: return TPartitionType.UNPARTITIONED; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java index c3ad7b82037efd..71f1c7a14354bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java @@ -18,7 +18,7 @@ package org.apache.doris.catalog; import org.apache.doris.analysis.PartitionKeyDesc; -import org.apache.doris.analysis.SingleRangePartitionDesc; +import org.apache.doris.analysis.SinglePartitionDesc; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeMetaVersion; @@ -26,12 +26,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.collect.Range; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -39,16 +35,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Set; public class RangePartitionInfo extends PartitionInfo { - private static final Logger LOG = LogManager.getLogger(RangePartitionInfo.class); - - private List partitionColumns = Lists.newArrayList(); - // formal partition id -> partition range - private Map> idToRange = Maps.newHashMap(); - // temp partition id -> partition range - private Map> idToTempRange = Maps.newHashMap(); public RangePartitionInfo() { // for persist @@ -61,24 +49,7 @@ public RangePartitionInfo(List partitionColumns) { this.isMultiColumnPartition = partitionColumns.size() > 1; } - public List getPartitionColumns() { - return partitionColumns; - } - - @Override - public void dropPartition(long partitionId) { - super.dropPartition(partitionId); - idToRange.remove(partitionId); - idToTempRange.remove(partitionId); - } - - public void addPartition(long partitionId, boolean isTemp, Range range, DataProperty dataProperty, - short replicationNum, boolean isInMemory) { - addPartition(partitionId, dataProperty, replicationNum, isInMemory); - setRangeInternal(partitionId, isTemp, range); - } - - public Range checkAndCreateRange(SingleRangePartitionDesc desc, boolean isTemp) throws DdlException { + public PartitionItem createAndCheckPartitionItem(SinglePartitionDesc desc, boolean isTemp) throws DdlException { Range newRange = null; PartitionKeyDesc partitionKeyDesc = desc.getPartitionKeyDesc(); // check range @@ -89,7 +60,7 @@ public Range checkAndCreateRange(SingleRangePartitionDesc desc, bo } Preconditions.checkNotNull(newRange); - return newRange; + return new RangePartitionItem(newRange); } // create a new range and check it. @@ -97,7 +68,7 @@ private Range createAndCheckNewRange(PartitionKeyDesc partKeyDesc, throws AnalysisException, DdlException { Range newRange = null; // generate and sort the existing ranges - List>> sortedRanges = getSortedRangeMap(isTemp); + List> sortedRanges = getSortedItemMap(isTemp); // create upper values for new range PartitionKey newRangeUpper = null; @@ -112,8 +83,8 @@ private Range createAndCheckNewRange(PartitionKeyDesc partKeyDesc, Range lastRange = null; Range currentRange = null; - for (Map.Entry> entry : sortedRanges) { - currentRange = entry.getValue(); + for (Map.Entry entry : sortedRanges) { + currentRange = entry.getValue().getItems(); // check if equals to upper bound PartitionKey upperKey = currentRange.upperEndpoint(); if (upperKey.compareTo(newRangeUpper) >= 0) { @@ -156,53 +127,7 @@ private Range checkNewRange(PartitionKeyDesc partKeyDesc, Partitio return newRange; } - public Range handleNewSinglePartitionDesc(SingleRangePartitionDesc desc, - long partitionId, boolean isTemp) throws DdlException { - Preconditions.checkArgument(desc.isAnalyzed()); - Range range = null; - try { - range = checkAndCreateRange(desc, isTemp); - setRangeInternal(partitionId, isTemp, range); - } catch (IllegalArgumentException e) { - // Range.closedOpen may throw this if (lower > upper) - throw new DdlException("Invalid key range: " + e.getMessage()); - } - idToDataProperty.put(partitionId, desc.getPartitionDataProperty()); - idToReplicationNum.put(partitionId, desc.getReplicationNum()); - idToInMemory.put(partitionId, desc.isInMemory()); - return range; - } - - public void unprotectHandleNewSinglePartitionDesc(long partitionId, boolean isTemp, Range range, - DataProperty dataProperty, short replicationNum, - boolean isInMemory) { - setRangeInternal(partitionId, isTemp, range); - idToDataProperty.put(partitionId, dataProperty); - idToReplicationNum.put(partitionId, replicationNum); - idToInMemory.put(partitionId, isInMemory); - } - - public void setRange(long partitionId, boolean isTemp, Range range) { - setRangeInternal(partitionId, isTemp, range); - } - - public Map> getIdToRange(boolean isTemp) { - if (isTemp) { - return idToTempRange; - } else { - return idToRange; - } - } - - public Range getRange(long partitionId) { - Range range = idToRange.get(partitionId); - if (range == null) { - range = idToTempRange.get(partitionId); - } - return range; - } - - public static void checkRangeColumnType(Column column) throws AnalysisException { + public static void checkPartitionColumn(Column column) throws AnalysisException { PrimitiveType type = column.getDataType(); if (!type.isFixedPointType() && !type.isDateType()) { throw new AnalysisException("Column[" + column.getName() + "] type[" + type @@ -210,62 +135,26 @@ public static void checkRangeColumnType(Column column) throws AnalysisException } } - public List>> getSortedRangeMap(boolean isTemp) { - Map> tmpMap = idToRange; + @Override + public List> getSortedItemMap(boolean isTemp) { + Map tmpMap = idToItem; if (isTemp) { - tmpMap = idToTempRange; + tmpMap = idToTempItem; } - List>> sortedList = Lists.newArrayList(tmpMap.entrySet()); + List> sortedList = Lists.newArrayList(tmpMap.entrySet()); Collections.sort(sortedList, RangeUtils.RANGE_MAP_ENTRY_COMPARATOR); return sortedList; } - // get a sorted range list, exclude partitions which ids are in 'excludePartitionIds' - public List> getRangeList(Set excludePartitionIds, boolean isTemp) { - Map> tmpMap = idToRange; - if (isTemp) { - tmpMap = idToTempRange; - } - List> resultList = Lists.newArrayList(); - for (Map.Entry> entry : tmpMap.entrySet()) { - if (!excludePartitionIds.contains(entry.getKey())) { - resultList.add(entry.getValue()); - } - } - return resultList; - } - - // return any range intersect with the newRange. - // return null if no range intersect. - public Range getAnyIntersectRange(Range newRange, boolean isTemp) { - Map> tmpMap = idToRange; - if (isTemp) { - tmpMap = idToTempRange; - } - for (Range range : tmpMap.values()) { - if (range.isConnected(newRange)) { - Range intersection = range.intersection(newRange); - if (!intersection.isEmpty()) { - return range; - } - } - } - return null; - } - - private void setRangeInternal(long partitionId, boolean isTemp, Range range) { - if (isTemp) { - idToTempRange.put(partitionId, range); - } else { - idToRange.put(partitionId, range); - } + @Override + public void checkPartitionItemListsMatch(List list1, List list2) + throws DdlException { + RangeUtils.checkPartitionItemListsMatch(list1, list2); } - public void moveRangeFromTempToFormal(long tempPartitionId) { - Range range = idToTempRange.remove(tempPartitionId); - if (range != null) { - idToRange.put(tempPartitionId, range); - } + @Override + public void checkPartitionItemListsConflict(List list1, List list2) throws DdlException { + RangeUtils.checkRangeConflict(list1, list2); } public static PartitionInfo read(DataInput in) throws IOException { @@ -284,16 +173,16 @@ public void write(DataOutput out) throws IOException { column.write(out); } - out.writeInt(idToRange.size()); - for (Map.Entry> entry : idToRange.entrySet()) { + out.writeInt(idToItem.size()); + for (Map.Entry entry : idToItem.entrySet()) { out.writeLong(entry.getKey()); - RangeUtils.writeRange(out, entry.getValue()); + entry.getValue().write(out); } - out.writeInt(idToTempRange.size()); - for (Map.Entry> entry : idToTempRange.entrySet()) { + out.writeInt(idToTempItem.size()); + for (Map.Entry entry : idToTempItem.entrySet()) { out.writeLong(entry.getKey()); - RangeUtils.writeRange(out, entry.getValue()); + entry.getValue().write(out); } } @@ -312,7 +201,7 @@ public void readFields(DataInput in) throws IOException { for (int i = 0; i < counter; i++) { long partitionId = in.readLong(); Range range = RangeUtils.readRange(in); - idToRange.put(partitionId, range); + idToItem.put(partitionId, new RangePartitionItem(range)); } if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_77) { @@ -320,7 +209,7 @@ public void readFields(DataInput in) throws IOException { for (int i = 0; i < counter; i++) { long partitionId = in.readLong(); Range range = RangeUtils.readRange(in); - idToTempRange.put(partitionId, range); + idToTempItem.put(partitionId, new RangePartitionItem(range)); } } } @@ -340,15 +229,14 @@ public String toSql(OlapTable table, List partitionId) { sb.append(")\n("); // sort range - List>> entries = - new ArrayList>>(this.idToRange.entrySet()); + List> entries = new ArrayList<>(this.idToItem.entrySet()); Collections.sort(entries, RangeUtils.RANGE_MAP_ENTRY_COMPARATOR); idx = 0; - for (Map.Entry> entry : entries) { + for (Map.Entry entry : entries) { Partition partition = table.getPartition(entry.getKey()); String partitionName = partition.getName(); - Range range = entry.getValue(); + Range range = entry.getValue().getItems(); // print all partitions' range is fixed range, even if some of them is created by less than range sb.append("PARTITION ").append(partitionName).append(" VALUES ["); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionItem.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionItem.java new file mode 100644 index 00000000000000..db3fe21edc5b34 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionItem.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.common.util.RangeUtils; + +import com.google.common.collect.Range; + +import java.io.DataOutput; +import java.io.IOException; + +public class RangePartitionItem extends PartitionItem { + private Range partitionKeyRange; + public static final Range DUMMY_ITEM; + + static { + DUMMY_ITEM = Range.closed(new PartitionKey(), new PartitionKey()); + } + + public RangePartitionItem(Range range) { + this.partitionKeyRange = range; + } + + public Range getItems() { + return partitionKeyRange; + } + + @Override + public void write(DataOutput out) throws IOException { + RangeUtils.writeRange(out, partitionKeyRange); + } + + @Override + public int compareTo(PartitionItem other) { + if (partitionKeyRange.contains(other.getItems())) { + return 1; + } + if (partitionKeyRange.equals(((RangePartitionItem) other).getItems())) { + return 0; + } + return -1; + } + + @Override + public PartitionItem getIntersect(PartitionItem newItem) { + Range newRange = newItem.getItems(); + if (partitionKeyRange.isConnected(newRange)) { + Range intersection = partitionKeyRange.intersection(newRange); + if (!intersection.isEmpty()) { + return new RangePartitionItem(intersection); + } + } + return null; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof RangePartitionItem) { + return partitionKeyRange.equals(((RangePartitionItem) obj).getItems()); + } + return false; + } + + @Override + public String toString() { + return partitionKeyRange.toString(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index 32b18726da5fe4..4855554e58304e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -23,15 +23,17 @@ import org.apache.doris.analysis.HashDistributionDesc; import org.apache.doris.analysis.PartitionKeyDesc; import org.apache.doris.analysis.PartitionValue; -import org.apache.doris.analysis.SingleRangePartitionDesc; +import org.apache.doris.analysis.SinglePartitionDesc; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.DynamicPartitionProperty; import org.apache.doris.catalog.HashDistributionInfo; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.RangePartitionInfo; +import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; @@ -159,13 +161,13 @@ private ArrayList getAddPartitionClause(Database db, OlapTab db.getFullName(), olapTable.getName()); continue; } - for (Range partitionKeyRange : rangePartitionInfo.getIdToRange(false).values()) { + for (PartitionItem partitionItem : rangePartitionInfo.getIdToItem(false).values()) { // only support single column partition now try { - RangeUtils.checkRangeIntersect(partitionKeyRange, addPartitionKeyRange); + RangeUtils.checkRangeIntersect(partitionItem.getItems(), addPartitionKeyRange); } catch (DdlException e) { isPartitionExists = true; - if (addPartitionKeyRange.equals(partitionKeyRange)) { + if (addPartitionKeyRange.equals(partitionItem.getItems())) { clearCreatePartitionFailedMsg(olapTable.getName()); } else { recordCreatePartitionFailedMsg(db.getFullName(), olapTable.getName(), e.getMessage()); @@ -178,7 +180,7 @@ private ArrayList getAddPartitionClause(Database db, OlapTab } // construct partition desc - PartitionKeyDesc partitionKeyDesc = new PartitionKeyDesc(Collections.singletonList(lowerValue), Collections.singletonList(upperValue)); + PartitionKeyDesc partitionKeyDesc = PartitionKeyDesc.createFixed(Collections.singletonList(lowerValue), Collections.singletonList(upperValue)); HashMap partitionProperties = new HashMap<>(1); if (dynamicPartitionProperty.getReplicationNum() == DynamicPartitionProperty.NOT_SET_REPLICATION_NUM) { partitionProperties.put("replication_num", String.valueOf(olapTable.getDefaultReplicationNum())); @@ -187,7 +189,7 @@ private ArrayList getAddPartitionClause(Database db, OlapTab } String partitionName = dynamicPartitionProperty.getPrefix() + DynamicPartitionUtil.getFormattedPartitionName( dynamicPartitionProperty.getTimeZone(), prevBorder, dynamicPartitionProperty.getTimeUnit()); - SingleRangePartitionDesc rangePartitionDesc = new SingleRangePartitionDesc(true, partitionName, + SinglePartitionDesc rangePartitionDesc = new SinglePartitionDesc(true, partitionName, partitionKeyDesc, partitionProperties); // construct distribution desc @@ -237,12 +239,12 @@ private ArrayList getDropPartitionClause(Database db, OlapT } RangePartitionInfo info = (RangePartitionInfo) (olapTable.getPartitionInfo()); - List>> idToRanges = new ArrayList<>(info.getIdToRange(false).entrySet()); - idToRanges.sort(Comparator.comparing(o -> o.getValue().upperEndpoint())); - for (Map.Entry> idToRange : idToRanges) { + List> idToItems = new ArrayList<>(info.getIdToItem(false).entrySet()); + idToItems.sort(Comparator.comparing(o -> ((RangePartitionItem) o.getValue()).getItems().upperEndpoint())); + for (Map.Entry idToItem : idToItems) { try { - Long checkDropPartitionId = idToRange.getKey(); - Range checkDropPartitionKey = idToRange.getValue(); + Long checkDropPartitionId = idToItem.getKey(); + Range checkDropPartitionKey = idToItem.getValue().getItems(); RangeUtils.checkRangeIntersect(reservePartitionKeyRange, checkDropPartitionKey); if (checkDropPartitionKey.upperEndpoint().compareTo(reservePartitionKeyRange.lowerEndpoint()) <= 0) { String dropPartitionName = olapTable.getPartition(checkDropPartitionId).getName(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java index ad480e02721880..5f4d443fe41145 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -206,6 +206,8 @@ public final class FeMetaVersion { public static final int VERSION_96 = 96; // persist orig stmt of export job public static final int VERSION_97 = 97; + // add list partition + public static final int VERSION_98 = 98; // note: when increment meta version, should assign the latest version to VERSION_CURRENT - public static final int VERSION_CURRENT = VERSION_97; + public static final int VERSION_CURRENT = VERSION_98; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java index 18c3fb8e757d13..b08444bd7deb41 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java @@ -90,7 +90,7 @@ public ProcResult fetchResult() throws AnalysisException { colNames.add(column.getName()); } partitionInfo.add(joiner.join(colNames)); // partition key - partitionInfo.add(rangePartitionInfo.getRange(esShardPartitions.getPartitionId()).toString()); // range + partitionInfo.add(rangePartitionInfo.getItem(esShardPartitions.getPartitionId()).getItems().toString());// range partitionInfo.add("-"); // dis partitionInfo.add(esShardPartitions.getShardRoutings().size()); // shards partitionInfo.add(1); // replica num diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java index 5bd62279fed1a1..f56d7c18244ee3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java @@ -33,7 +33,6 @@ import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionType; -import org.apache.doris.catalog.RangePartitionInfo; import org.apache.doris.catalog.Table.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; @@ -214,9 +213,8 @@ private List> getPartitionInfos() { // for range partitions, we return partitions in ascending range order by default. // this is to be consistent with the behaviour before 0.12 - if (tblPartitionInfo.getType() == PartitionType.RANGE) { - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) tblPartitionInfo; - partitionIds = rangePartitionInfo.getSortedRangeMap(isTempPartition).stream() + if (tblPartitionInfo.getType() == PartitionType.RANGE || tblPartitionInfo.getType() == PartitionType.LIST) { + partitionIds = tblPartitionInfo.getSortedItemMap(isTempPartition).stream() .map(Map.Entry::getKey).collect(Collectors.toList()); } else { Collection partitions = isTempPartition ? olapTable.getTempPartitions() : olapTable.getPartitions(); @@ -236,15 +234,15 @@ private List> getPartitionInfos() { partitionInfo.add(partition.getVisibleVersionHash()); partitionInfo.add(partition.getState()); - if (tblPartitionInfo.getType() == PartitionType.RANGE) { - // partition range info - List partitionColumns = ((RangePartitionInfo) tblPartitionInfo).getPartitionColumns(); - List colNames = new ArrayList(); + if (tblPartitionInfo.getType() == PartitionType.RANGE + || tblPartitionInfo.getType() == PartitionType.LIST) { + List partitionColumns = tblPartitionInfo.getPartitionColumns(); + List colNames = new ArrayList<>(); for (Column column : partitionColumns) { colNames.add(column.getName()); } partitionInfo.add(joiner.join(colNames)); - partitionInfo.add(((RangePartitionInfo) tblPartitionInfo).getRange(partitionId).toString()); + partitionInfo.add(tblPartitionInfo.getItem(partitionId).getItems().toString()); } else { partitionInfo.add(""); partitionInfo.add(""); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/ListUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/ListUtil.java index d5dc38cc8be9d7..cfe5a293bd5dd1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/ListUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/ListUtil.java @@ -17,12 +17,26 @@ package org.apache.doris.common.util; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.common.DdlException; + import com.google.common.base.Preconditions; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.List; +import java.util.Map; public class ListUtil { + + public static final Comparator> LIST_MAP_ENTRY_COMPARATOR = + Comparator.comparing(o -> ((ListPartitionItem) o.getValue()).getItems().iterator().next()); + public static final Comparator PARTITION_KEY_COMPARATOR = + Comparator.comparing(o -> ((ListPartitionItem) o).getItems().iterator().next()); + /** * split a list to multi expected number sublist * for example: @@ -55,4 +69,74 @@ public static List> splitBySize(List list, int expectedSize) return result; } + + public static void checkPartitionKeyListsMatch(List list1, List list2) throws DdlException { + Collections.sort(list1, PARTITION_KEY_COMPARATOR); + Collections.sort(list2, PARTITION_KEY_COMPARATOR); + + int idx1 = 0; + int idx2 = 0; + List keys1 = new ArrayList<>(); + List keys2 = new ArrayList<>(); + keys1.addAll(list1.get(idx1).getItems()); + keys2.addAll(list2.get(idx2).getItems()); + + while (true) { + int size = Math.min(keys1.size(), keys2.size()); + for (int i = 0; i < size; i++) { + int res = keys1.get(i).compareTo(keys2.get(i)); + if (res != 0) { + throw new DdlException("2 partition key lists are not matched. " + + keys1 + " vs. " + keys2); + } + } + + int res = keys1.size() - keys2.size(); + if (res == 0) { + ++idx1; + ++idx2; + if (idx1 == list1.size() || idx2 == list2.size()) { + break; + } + keys1.clear(); + keys2.clear(); + keys1.addAll(list1.get(idx1).getItems()); + keys2.addAll(list2.get(idx2).getItems()); + } else if (res > 0) { + if (++idx2 == list2.size()) { + break; + } + keys1.removeAll(keys2); + keys2.clear(); + keys2.addAll(list2.get(idx2).getItems()); + } else { + if (++idx1 == list1.size()) { + break; + } + keys2.removeAll(keys1); + keys1.clear(); + keys1.addAll(list1.get(idx1).getItems()); + } + } + + if (idx1 < list1.size() || idx2 < list2.size()) { + throw new DdlException("2 partition key lists are not matched. " + + list1 + " vs. " + list2); + } + } + + // Check if items in list2 conflicts with items in list1 + public static void checkListsConflict(List list1, List list2) throws DdlException { + for (PartitionItem checkItem : list2) { + List checkKeys = checkItem.getItems(); + for (PartitionKey checkKey : checkKeys) { + for (PartitionItem currentItem : list1) { + if (((ListPartitionItem) currentItem).getItems().contains(checkKey)) { + throw new DdlException("The partition key[" + checkKey.toSql() + "] is overlap with current " + + currentItem.toString()); + } + } + } + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/RangeUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/RangeUtils.java index e91b94a59c0530..eeef22eac4e919 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/RangeUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/RangeUtils.java @@ -17,7 +17,9 @@ package org.apache.doris.common.util; +import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.common.DdlException; import com.google.common.collect.BoundType; @@ -35,11 +37,11 @@ public class RangeUtils { - public static final Comparator>> RANGE_MAP_ENTRY_COMPARATOR = - Comparator.comparing(o -> o.getValue().lowerEndpoint()); + public static final Comparator> RANGE_MAP_ENTRY_COMPARATOR = + Comparator.comparing(o -> (((RangePartitionItem)o.getValue()).getItems()).lowerEndpoint()); - public static final Comparator> RANGE_COMPARATOR = - Comparator.comparing(o -> o.lowerEndpoint()); + public static final Comparator RANGE_COMPARATOR = + Comparator.comparing(o -> ((RangePartitionItem) o).getItems().lowerEndpoint()); public static void checkRangeIntersect(Range range1, Range range2) throws DdlException { if (range2.isConnected(range1)) { @@ -74,14 +76,14 @@ public static void checkRangeIntersect(Range range1, Range> list1, List> list2) throws DdlException { + public static void checkPartitionItemListsMatch(List list1, List list2) throws DdlException { Collections.sort(list1, RangeUtils.RANGE_COMPARATOR); Collections.sort(list2, RangeUtils.RANGE_COMPARATOR); int idx1 = 0; int idx2 = 0; - Range range1 = list1.get(idx1); - Range range2 = list2.get(idx2); + Range range1 = list1.get(idx1).getItems(); + Range range2 = list2.get(idx2).getItems(); while (true) { if (range1.lowerEndpoint().compareTo(range2.lowerEndpoint()) != 0) { throw new DdlException("2 range lists are not stricly matched. " @@ -95,21 +97,21 @@ public static void checkRangeListsMatch(List> list1, List 0) { if (++idx2 == list2.size()) { break; } range1 = Range.closedOpen(range2.upperEndpoint(), range1.upperEndpoint()); - range2 = list2.get(idx2); + range2 = list2.get(idx2).getItems(); } else { if (++idx1 == list1.size()) { break; } range2 = Range.closedOpen(range1.upperEndpoint(), range2.upperEndpoint()); - range1 = list1.get(idx1); + range1 = list1.get(idx1).getItems(); } } @@ -192,18 +194,18 @@ public static Range readRange(DataInput in) throws IOException { } // check if any ranges in "rangesToBeChecked" conflict with ranges in "baseRanges". - public static void checkRangeConflict(List> baseRanges, - List> rangesToBeChecked) throws DdlException { + public static void checkRangeConflict(List baseRanges, + List rangesToBeChecked) throws DdlException { RangeMap baseRangeMap = TreeRangeMap.create(); long idx = 0; - for (Range baseRange : baseRanges) { - baseRangeMap.put(baseRange, idx++); + for (PartitionItem item : baseRanges) { + baseRangeMap.put(item.getItems(), idx++); } - for (Range range : rangesToBeChecked) { - if (!baseRangeMap.subRangeMap(range).asMapOfRanges().isEmpty()) { - throw new DdlException("Range: " + range + " conflicts with existing range"); + for (PartitionItem item : rangesToBeChecked) { + if (!baseRangeMap.subRangeMap(item.getItems()).asMapOfRanges().isEmpty()) { + throw new DdlException("Range: " + item.getItems() + " conflicts with existing range"); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardPartitions.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardPartitions.java index b81deade16cf05..a7ceff2e5f4720 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardPartitions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardPartitions.java @@ -17,7 +17,7 @@ package org.apache.doris.external.elasticsearch; -import org.apache.doris.analysis.SingleRangePartitionDesc; +import org.apache.doris.analysis.SinglePartitionDesc; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.thrift.TNetworkAddress; @@ -40,7 +40,7 @@ public class EsShardPartitions { private final String indexName; // shardid -> host1, host2, host3 private Map> shardRoutings; - private SingleRangePartitionDesc partitionDesc; + private SinglePartitionDesc partitionDesc; private PartitionKey partitionKey; private long partitionId = -1; @@ -127,11 +127,11 @@ public Map> getShardRoutings() { return shardRoutings; } - public SingleRangePartitionDesc getPartitionDesc() { + public SinglePartitionDesc getPartitionDesc() { return partitionDesc; } - public void setPartitionDesc(SingleRangePartitionDesc partitionDesc) { + public void setPartitionDesc(SinglePartitionDesc partitionDesc) { this.partitionDesc = partitionDesc; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsTablePartitions.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsTablePartitions.java index d6bc5e3d1ca527..43f70130ada9f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsTablePartitions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsTablePartitions.java @@ -20,7 +20,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.EsTable; import org.apache.doris.catalog.PartitionInfo; -import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.RangePartitionInfo; import org.apache.doris.catalog.SinglePartitionInfo; import org.apache.doris.common.DdlException; @@ -29,7 +29,6 @@ import org.apache.logging.log4j.Logger; import com.google.common.collect.Maps; -import com.google.common.collect.Range; import java.util.ArrayList; import java.util.Comparator; @@ -96,13 +95,13 @@ public static EsTablePartitions fromShardPartitions(EsTable esTable, EsShardPart esShardPartitionsList.sort(Comparator.comparing(EsShardPartitions::getPartitionKey)); long partitionId = 0; for (EsShardPartitions esShardPartitions : esShardPartitionsList) { - Range range = partitionInfo.handleNewSinglePartitionDesc( + PartitionItem item = partitionInfo.handleNewSinglePartitionDesc( esShardPartitions.getPartitionDesc(), partitionId, false); esTablePartitions.addPartition(esShardPartitions.getIndexName(), partitionId); esShardPartitions.setPartitionId(partitionId); ++partitionId; LOG.debug("add partition to es table [{}] with range [{}]", esTable.getName(), - range); + item.getItems()); } } return esTablePartitions; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java index deb612dfd8f04d..5f4bd824843ff4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java @@ -33,6 +33,7 @@ import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.PrimitiveType; @@ -339,7 +340,7 @@ private EtlPartitionInfo createEtlPartitionInfo(OlapTable table, Set parti partitionColumnRefs.add(column.getName()); } - for (Map.Entry> entry : rangePartitionInfo.getSortedRangeMap(false)) { + for (Map.Entry entry : rangePartitionInfo.getSortedItemMap(false)) { long partitionId = entry.getKey(); if (!partitionIds.contains(partitionId)) { continue; @@ -354,7 +355,7 @@ private EtlPartitionInfo createEtlPartitionInfo(OlapTable table, Set parti int bucketNum = partition.getDistributionInfo().getBucketNum(); // is max partition - Range range = entry.getValue(); + Range range = entry.getValue().getItems(); boolean isMaxPartition = range.upperEndpoint().isMaxValue(); // start keys @@ -380,8 +381,7 @@ private EtlPartitionInfo createEtlPartitionInfo(OlapTable table, Set parti etlPartitions.add(new EtlPartition(partitionId, startKeys, endKeys, isMaxPartition, bucketNum)); } - } else { - Preconditions.checkState(type == PartitionType.UNPARTITIONED); + } else if (type == PartitionType.UNPARTITIONED) { Preconditions.checkState(partitionIds.size() == 1); for (Long partitionId : partitionIds) { @@ -394,8 +394,10 @@ private EtlPartitionInfo createEtlPartitionInfo(OlapTable table, Set parti int bucketNum = partition.getDistributionInfo().getBucketNum(); etlPartitions.add(new EtlPartition(partitionId, Lists.newArrayList(), Lists.newArrayList(), - true, bucketNum)); + true, bucketNum)); } + } else { + throw new LoadException("Spark Load does not support list partition yet"); } // distribution column refs diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/PartitionPersistInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/PartitionPersistInfo.java index 5b4fd3fcab42a2..d482e793641d97 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/PartitionPersistInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/PartitionPersistInfo.java @@ -19,7 +19,9 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.DataProperty; +import org.apache.doris.catalog.ListPartitionItem; import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.io.Writable; @@ -37,6 +39,7 @@ public class PartitionPersistInfo implements Writable { private Partition partition; private Range range; + private PartitionItem listPartitionItem; private DataProperty dataProperty; private short replicationNum; private boolean isInMemory = false; @@ -46,13 +49,14 @@ public PartitionPersistInfo() { } public PartitionPersistInfo(long dbId, long tableId, Partition partition, Range range, - DataProperty dataProperty, short replicationNum, + PartitionItem listPartitionItem, DataProperty dataProperty, short replicationNum, boolean isInMemory, boolean isTempPartition) { this.dbId = dbId; this.tableId = tableId; this.partition = partition; this.range = range; + this.listPartitionItem = listPartitionItem; this.dataProperty = dataProperty; this.replicationNum = replicationNum; @@ -76,6 +80,10 @@ public Range getRange() { return range; } + public PartitionItem getListPartitionItem() { + return listPartitionItem; + } + public DataProperty getDataProperty() { return dataProperty; } @@ -98,6 +106,7 @@ public void write(DataOutput out) throws IOException { partition.write(out); RangeUtils.writeRange(out, range); + listPartitionItem.write(out); dataProperty.write(out); out.writeShort(replicationNum); out.writeBoolean(isInMemory); @@ -110,6 +119,12 @@ public void readFields(DataInput in) throws IOException { partition = Partition.read(in); range = RangeUtils.readRange(in); + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_98) { + listPartitionItem = ListPartitionItem.read(in); + } else { + listPartitionItem = ListPartitionItem.DUMMY_ITEM; + } + dataProperty = DataProperty.read(in); replicationNum = in.readShort(); if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_72) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DataSplitSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DataSplitSink.java index 5da1f092e039d3..217f53dc972036 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/DataSplitSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DataSplitSink.java @@ -334,7 +334,7 @@ private static EtlPartitionRange createPartitionRange( Column partitionColumn = rangePartitionInfo.getPartitionColumns().get(0); PrimitiveType columnType = partitionColumn.getDataType(); - Range range = rangePartitionInfo.getRange(partitionId); + Range range = rangePartitionInfo.getItem(partitionId).getItems(); // start key EtlPartitionKey startPartitionKey = null; PartitionKey startKey = range.lowerEndpoint(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java index e662de08931f1c..6853459ab2200c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java @@ -23,7 +23,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.EsTable; import org.apache.doris.catalog.PartitionInfo; -import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.RangePartitionInfo; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; @@ -48,7 +48,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; -import com.google.common.collect.Range; import com.google.common.collect.Sets; import java.util.ArrayList; @@ -286,7 +285,7 @@ private Collection partitionPrune(PartitionInfo partitionInfo) throws Anal switch (partitionInfo.getType()) { case RANGE: { RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; - Map> keyRangeById = rangePartitionInfo.getIdToRange(false); + Map keyRangeById = rangePartitionInfo.getIdToItem(false); partitionPruner = new RangePartitionPruner(keyRangeById, rangePartitionInfo.getPartitionColumns(), columnFilters); return partitionPruner.prune(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ListPartitionPruner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ListPartitionPruner.java index 9c840a8fcc9c6d..a9eba852141605 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ListPartitionPruner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ListPartitionPruner.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.InPredicate; import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; @@ -28,27 +29,21 @@ import com.google.common.collect.Range; import com.google.common.collect.Sets; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; -/* - * (hujie, cmy) - * ATTN: do not delete it before considering useless for certain +/** + * list partition pruner */ -@Deprecated public class ListPartitionPruner implements PartitionPruner { - private static final Logger LOG = LoggerFactory.getLogger(ListPartitionPruner.class); - private Map> partitionListMap; + private Map partitionListMap; private List partitionColumns; private Map partitionColumnFilters; - public ListPartitionPruner(Map> listMap, + public ListPartitionPruner(Map listMap, List columns, Map filters) { partitionListMap = listMap; @@ -56,31 +51,99 @@ public ListPartitionPruner(Map> listMap, partitionColumnFilters = filters; } - private Collection pruneListMap( - Map> listMap, - Range range) { + private Collection pruneListMap(Map listMap, + Range range, + int columnId) { Set resultSet = Sets.newHashSet(); - for (Map.Entry> entry : listMap.entrySet()) { - for (PartitionKey key : entry.getValue()) { - if (range.contains(key)) { + for (Map.Entry entry : listMap.entrySet()) { + List partitionKeys = entry.getValue().getItems(); + for (PartitionKey partitionKey : partitionKeys) { + LiteralExpr expr = partitionKey.getKeys().get(columnId); + if (contain(range, expr, columnId)) { resultSet.add(entry.getKey()); - break; } } } return resultSet; } + /** + * check literal expr exist in partition range + * @param range the partition key range + * @param literalExpr expr to be checked + * @param columnId expr column index in partition key + * @return + */ + private boolean contain(Range range, LiteralExpr literalExpr, int columnId) { + LiteralExpr lowerExpr = range.lowerEndpoint().getKeys().get(columnId); + LiteralExpr upperExpr = range.upperEndpoint().getKeys().get(columnId); + BoundType lType = range.lowerBoundType(); + BoundType uType = range.upperBoundType(); + int ret1 = PartitionKey.compareLiteralExpr(literalExpr, lowerExpr); + int ret2 = PartitionKey.compareLiteralExpr(literalExpr, upperExpr); + + if (lType == BoundType.CLOSED && uType == BoundType.CLOSED) { + if (ret1 >= 0 && ret2 <= 0) { + return true; + } + } else if (lType == BoundType.CLOSED && uType == BoundType.OPEN) { + if (ret1 >= 0 && ret2 < 0) { + return true; + } + } else if (lType == BoundType.OPEN && uType == BoundType.CLOSED) { + if (ret1 > 0 && ret2 <= 0) { + return true; + } + } else if (lType == BoundType.OPEN && uType == BoundType.OPEN) { + if (ret1 > 0 && ret2 < 0) { + return true; + } + } + return false; + } + + /** + * get min literal expr from partition key list map by partition key column id. + * @param columnId + * @return + */ + private LiteralExpr getMinLiteral(int columnId) { + LiteralExpr minLiteral = null; + for (Map.Entry entry : partitionListMap.entrySet()) { + List partitionKeys = entry.getValue().getItems(); + for (PartitionKey partitionKey : partitionKeys) { + minLiteral = getMinExpr(partitionKey.getKeys().get(columnId), minLiteral); + } + } + return minLiteral; + } + + private LiteralExpr getMinExpr(LiteralExpr expr, LiteralExpr minLiteral) { + if (minLiteral == null) { + minLiteral = expr; + return minLiteral; + } + if (expr.compareLiteral(minLiteral) < 0) { + minLiteral = expr; + } + return minLiteral; + } + private Collection prune( - Map> listMap, + Map listMap, int columnId, PartitionKey minKey, PartitionKey maxKey, int complex) throws AnalysisException { + // if partition item map is empty, no need to prune. + if (listMap.size() == 0) { + return Lists.newArrayList(); + } + if (columnId == partitionColumns.size()) { try { - return pruneListMap(listMap, Range.closed(minKey, maxKey)); + return pruneListMap(listMap, Range.closed(minKey, maxKey), columnId - 1); } catch (IllegalArgumentException e) { return Lists.newArrayList(); } @@ -89,13 +152,13 @@ private Collection prune( PartitionColumnFilter filter = partitionColumnFilters.get(keyColumn.getName()); // no filter in this column if (null == filter) { - minKey.pushColumn(LiteralExpr.createInfinity(Type.fromPrimitiveType(keyColumn.getDataType()), false), - keyColumn.getDataType()); + minKey.pushColumn(getMinLiteral(columnId), keyColumn.getDataType()); maxKey.pushColumn(LiteralExpr.createInfinity(Type.fromPrimitiveType(keyColumn.getDataType()), true), - keyColumn.getDataType()); + keyColumn.getDataType()); Collection result = null; try { - return pruneListMap(listMap, Range.closed(minKey, maxKey)); + // prune next partition column + result = prune(listMap, columnId + 1, minKey, maxKey, complex); } catch (IllegalArgumentException e) { result = Lists.newArrayList(); } @@ -105,13 +168,18 @@ private Collection prune( } InPredicate inPredicate = filter.getInPredicate(); if (null == inPredicate || inPredicate.getChildren().size() * complex > 100) { + // case: where k1 = 1; if (filter.lowerBoundInclusive && filter.upperBoundInclusive && filter.lowerBound != null && filter.upperBound != null && 0 == filter.lowerBound.compareLiteral(filter.upperBound)) { minKey.pushColumn(filter.lowerBound, keyColumn.getDataType()); maxKey.pushColumn(filter.upperBound, keyColumn.getDataType()); - Collection result = - prune(listMap, columnId + 1, minKey, maxKey, complex); + // handle like in predicate + Collection result = pruneListMap(listMap, Range.closed(minKey, maxKey), columnId); + // prune next partition column + if (partitionColumns.size() > 1) { + result.retainAll(prune(listMap, columnId + 1, minKey, maxKey, complex)); + } minKey.popColumn(); maxKey.popColumn(); return result; @@ -124,25 +192,13 @@ private Collection prune( int lastColumnId = partitionColumns.size() - 1; if (filter.lowerBound != null) { minKey.pushColumn(filter.lowerBound, keyColumn.getDataType()); - if (filter.lowerBoundInclusive && columnId != lastColumnId) { - Column column = partitionColumns.get(columnId + 1); - minKey.pushColumn(LiteralExpr.createInfinity(Type.fromPrimitiveType(column.getDataType()), false), - column.getDataType()); - isPushMin = true; - } } else { - minKey.pushColumn(LiteralExpr.createInfinity(Type.fromPrimitiveType(keyColumn.getDataType()), false), + minKey.pushColumn(getMinLiteral(columnId), keyColumn.getDataType()); isPushMin = true; } if (filter.upperBound != null) { maxKey.pushColumn(filter.upperBound, keyColumn.getDataType()); - if (filter.upperBoundInclusive && columnId != lastColumnId) { - Column column = partitionColumns.get(columnId + 1); - maxKey.pushColumn(LiteralExpr.createInfinity(Type.fromPrimitiveType(column.getDataType()), true), - column.getDataType()); - isPushMax = true; - } } else { maxKey.pushColumn(LiteralExpr.createInfinity(Type.fromPrimitiveType(keyColumn.getDataType()), true), keyColumn.getDataType()); @@ -151,7 +207,12 @@ private Collection prune( Collection result = null; try { - return pruneListMap(listMap, Range.range(minKey, lowerType, maxKey, upperType)); + result = pruneListMap(listMap, Range.range(minKey, lowerType, maxKey, upperType), columnId); + // prune next partition column + if (partitionColumns.size() > 1) { + result.retainAll(prune(listMap, columnId + 1, minKey, maxKey, complex)); + } + return result; } catch (IllegalArgumentException e) { result = Lists.newArrayList(); } @@ -170,7 +231,13 @@ private Collection prune( LiteralExpr expr = (LiteralExpr) inPredicate.getChild(i); minKey.pushColumn(expr, keyColumn.getDataType()); maxKey.pushColumn(expr, keyColumn.getDataType()); - resultSet.addAll(prune(listMap, columnId + 1, minKey, maxKey, complex)); + Collection result = pruneListMap(listMap, Range.closed(minKey, maxKey), columnId); + // prune next partition column + if (partitionColumns.size() > 1) { + // Take the intersection + result.retainAll(prune(listMap, columnId + 1, minKey, maxKey, complex)); + } + resultSet.addAll(result); minKey.popColumn(); maxKey.popColumn(); } @@ -183,5 +250,3 @@ public Collection prune() throws AnalysisException { return prune(partitionListMap, 0, minKey, maxKey, 1); } } - -/* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 3ec9866e1126be..5628e8a93bedac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -39,9 +39,8 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionInfo; -import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionType; -import org.apache.doris.catalog.RangePartitionInfo; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Tablet; import org.apache.doris.common.AnalysisException; @@ -71,7 +70,6 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Range; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -333,22 +331,28 @@ public void computeStats(Analyzer analyzer) { cardinality = cardinality == -1 ? 0 : cardinality; } - private Collection partitionPrune(RangePartitionInfo partitionInfo, PartitionNames partitionNames) throws AnalysisException { - Map> keyRangeById = null; + private Collection partitionPrune(PartitionInfo partitionInfo, PartitionNames partitionNames) throws AnalysisException { + PartitionPruner partitionPruner = null; + Map keyItemMap; if (partitionNames != null) { - keyRangeById = Maps.newHashMap(); + keyItemMap = Maps.newHashMap(); for (String partName : partitionNames.getPartitionNames()) { - Partition part = olapTable.getPartition(partName, partitionNames.isTemp()); - if (part == null) { + Partition partition = olapTable.getPartition(partName, partitionNames.isTemp()); + if (partition == null) { ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_SUCH_PARTITION, partName); } - keyRangeById.put(part.getId(), partitionInfo.getRange(part.getId())); + keyItemMap.put(partition.getId(), partitionInfo.getItem(partition.getId())); } } else { - keyRangeById = partitionInfo.getIdToRange(false); + keyItemMap = partitionInfo.getIdToItem(false); + } + if (partitionInfo.getType() == PartitionType.RANGE) { + partitionPruner = new RangePartitionPruner(keyItemMap, + partitionInfo.getPartitionColumns(), columnFilters); + } else if (partitionInfo.getType() == PartitionType.LIST) { + partitionPruner = new ListPartitionPruner(keyItemMap, + partitionInfo.getPartitionColumns(), columnFilters); } - PartitionPruner partitionPruner = new RangePartitionPruner(keyRangeById, - partitionInfo.getPartitionColumns(), columnFilters); return partitionPruner.prune(); } @@ -464,8 +468,8 @@ private void computePartitionInfo() throws AnalysisException { // Step1: compute partition ids PartitionNames partitionNames = ((BaseTableRef) desc.getRef()).getPartitionNames(); PartitionInfo partitionInfo = olapTable.getPartitionInfo(); - if (partitionInfo.getType() == PartitionType.RANGE) { - selectedPartitionIds = partitionPrune((RangePartitionInfo) partitionInfo, partitionNames); + if (partitionInfo.getType() == PartitionType.RANGE || partitionInfo.getType() == PartitionType.LIST) { + selectedPartitionIds = partitionPrune(partitionInfo, partitionNames); } else { selectedPartitionIds = null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index 0d9d4d12173148..8ca0cca730a2a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -24,14 +24,17 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DistributionInfo; import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.ListPartitionItem; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.PartitionType; -import org.apache.doris.catalog.RangePartitionInfo; +import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.catalog.Tablet; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; @@ -45,6 +48,7 @@ import org.apache.doris.thrift.TDataSink; import org.apache.doris.thrift.TDataSinkType; import org.apache.doris.thrift.TExplainLevel; +import org.apache.doris.thrift.TExprNode; import org.apache.doris.thrift.TNodeInfo; import org.apache.doris.thrift.TOlapTableIndexSchema; import org.apache.doris.thrift.TOlapTableIndexTablets; @@ -67,6 +71,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -213,32 +218,22 @@ private TOlapTablePartitionParam createPartition(long dbId, OlapTable table) thr PartitionType partType = table.getPartitionInfo().getType(); switch (partType) { + case LIST: case RANGE: { - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) table.getPartitionInfo(); - for (Column partCol : rangePartitionInfo.getPartitionColumns()) { + PartitionInfo partitionInfo = table.getPartitionInfo(); + for (Column partCol : partitionInfo.getPartitionColumns()) { partitionParam.addToPartitionColumns(partCol.getName()); } - int partColNum = rangePartitionInfo.getPartitionColumns().size(); + int partColNum = partitionInfo.getPartitionColumns().size(); DistributionInfo selectedDistInfo = null; for (Long partitionId : partitionIds) { Partition partition = table.getPartition(partitionId); TOlapTablePartition tPartition = new TOlapTablePartition(); tPartition.setId(partition.getId()); - Range range = rangePartitionInfo.getRange(partition.getId()); - // set start keys - if (range.hasLowerBound() && !range.lowerEndpoint().isMinValue()) { - for (int i = 0; i < partColNum; i++) { - tPartition.addToStartKeys(range.lowerEndpoint().getKeys().get(i).treeToThrift().getNodes().get(0)); - } - } - // set end keys - if (range.hasUpperBound() && !range.upperEndpoint().isMaxValue()) { - for (int i = 0; i < partColNum; i++) { - tPartition.addToEndKeys(range.upperEndpoint().getKeys().get(i).treeToThrift().getNodes().get(0)); - } - } + // set partition keys + setPartitionKeys(tPartition, partitionInfo.getItem(partition.getId()), partColNum); for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { tPartition.addToIndexes(new TOlapTableIndexTablets(index.getId(), Lists.newArrayList( @@ -287,6 +282,34 @@ private TOlapTablePartitionParam createPartition(long dbId, OlapTable table) thr return partitionParam; } + private void setPartitionKeys(TOlapTablePartition tPartition, PartitionItem partitionItem, int partColNum) { + if (partitionItem instanceof RangePartitionItem) { + Range range = partitionItem.getItems(); + // set start keys + if (range.hasLowerBound() && !range.lowerEndpoint().isMinValue()) { + for (int i = 0; i < partColNum; i++) { + tPartition.addToStartKeys(range.lowerEndpoint().getKeys().get(i).treeToThrift().getNodes().get(0)); + } + } + // set end keys + if (range.hasUpperBound() && !range.upperEndpoint().isMaxValue()) { + for (int i = 0; i < partColNum; i++) { + tPartition.addToEndKeys(range.upperEndpoint().getKeys().get(i).treeToThrift().getNodes().get(0)); + } + } + } else if (partitionItem instanceof ListPartitionItem){ + List partitionKeys = partitionItem.getItems(); + // set in keys + for (PartitionKey partitionKey : partitionKeys) { + List tExprNodes = new ArrayList<>(); + for (int i = 0; i < partColNum; i++) { + tExprNodes.add(partitionKey.getKeys().get(i).treeToThrift().getNodes().get(0)); + } + tPartition.addToInKeys(tExprNodes); + } + } + } + private TOlapTableLocationParam createLocation(OlapTable table) throws UserException { TOlapTableLocationParam locationParam = new TOlapTableLocationParam(); // BE id -> path hash diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RangePartitionPruner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RangePartitionPruner.java index d91ae0886860d6..8568599732bf8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/RangePartitionPruner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RangePartitionPruner.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.analysis.NullLiteral; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; @@ -43,11 +44,11 @@ public class RangePartitionPruner implements PartitionPruner { private static final Logger LOG = LogManager.getLogger(RangePartitionPruner.class); - private Map> partitionRangeMap; + private Map partitionRangeMap; private List partitionColumns; private Map partitionColumnFilters; - public RangePartitionPruner(Map> rangeMap, + public RangePartitionPruner(Map rangeMap, List columns, Map filters) { partitionRangeMap = rangeMap; @@ -186,8 +187,8 @@ public Collection prune() throws AnalysisException { PartitionKey maxKey = new PartitionKey(); // Map to RangeMapTree RangeMap rangeMap = TreeRangeMap.create(); - for (Map.Entry> entry : partitionRangeMap.entrySet()) { - rangeMap.put(entry.getValue(), entry.getKey()); + for (Map.Entry entry : partitionRangeMap.entrySet()) { + rangeMap.put(entry.getValue().getItems(), entry.getKey()); } return prune(rangeMap, 0, minKey, maxKey, 1); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 8a30f1e375981a..f3c41c1aae9d5b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -50,11 +50,11 @@ import org.apache.doris.thrift.TMasterOpResult; import org.apache.doris.thrift.TUniqueId; +import com.google.common.base.Strings; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.google.common.base.Strings; - import java.io.IOException; import java.io.StringReader; import java.io.UnsupportedEncodingException; @@ -242,7 +242,7 @@ private void handleQuery() { } // analyze the origin stmt and return multi-statements - private List analyze(String originStmt) throws AnalysisException { + private List analyze(String originStmt) throws AnalysisException, DdlException { LOG.debug("the originStmts are: {}", originStmt); // Parse statement with parser generated by CUP&FLEX SqlScanner input = new SqlScanner(new StringReader(originStmt), ctx.getSessionVariable().getSqlMode()); @@ -251,9 +251,9 @@ private List analyze(String originStmt) throws AnalysisException return SqlParserUtils.getMultiStmts(parser); } catch (Error e) { throw new AnalysisException("Please check your sql, we meet an error when parsing.", e); - } catch (AnalysisException e) { - LOG.warn("origin_stmt: " + originStmt + "; Analyze error message: " + parser.getErrorMsg(originStmt), e); + } catch (AnalysisException | DdlException e) { String errorMessage = parser.getErrorMsg(originStmt); + LOG.debug("origin stmt: {}; Analyze error message: {}", originStmt, parser.getErrorMsg(originStmt), e); if (errorMessage == null) { throw e; } else { @@ -262,7 +262,7 @@ private List analyze(String originStmt) throws AnalysisException } catch (Exception e) { // TODO(lingbin): we catch 'Exception' to prevent unexpected error, // should be removed this try-catch clause future. - throw new AnalysisException("Internal Error, maybe syntax error or this is a bug, please contact with Palo RD."); + throw new AnalysisException("Internal Error, maybe syntax error or this is a bug"); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java index aa09ad822ee096..15a2c8f501b069 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java @@ -26,11 +26,13 @@ import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.analysis.IntLiteral; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.RangePartitionInfo; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.catalog.Type; import org.apache.doris.common.Config; import org.apache.doris.planner.PartitionColumnFilter; @@ -38,7 +40,6 @@ import org.apache.doris.common.AnalysisException; import com.google.common.collect.Lists; -import com.google.common.collect.Range; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -479,11 +480,11 @@ public boolean rewritePredicate(CompoundPredicate predicate, List> range = rangePartitionInfo.getIdToRange(false); - for (Map.Entry> entry : range.entrySet()) { + Map range = rangePartitionInfo.getIdToItem(false); + for (Map.Entry entry : range.entrySet()) { Long partId = entry.getKey(); for (PartitionSingle single : partitionSingleList) { - if (entry.getValue().contains(single.getPartitionKey())) { + if (((RangePartitionItem) entry.getValue()).getItems().contains(single.getPartitionKey())) { if (single.getPartitionId() == 0) { single.setPartitionId(partId); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/HadoopLoadPendingTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/HadoopLoadPendingTask.java index 09140e960954d6..309815ae8ac55e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/HadoopLoadPendingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/HadoopLoadPendingTask.java @@ -305,7 +305,7 @@ private EtlPartitionInfo createPartitionInfo(OlapTable table, long partitionId) // begin keys // is max partition - Range range = rangePartitionInfo.getRange(partitionId); + Range range = rangePartitionInfo.getItem(partitionId).getItems(); boolean isMaxPartition = range.upperEndpoint().isMaxValue(); // start keys diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex index c3d918cfe2c431..e4a1e459c8bd43 100644 --- a/fe/fe-core/src/main/jflex/sql_scanner.flex +++ b/fe/fe-core/src/main/jflex/sql_scanner.flex @@ -245,6 +245,7 @@ import org.apache.doris.qe.SqlModeHelper; keywordMap.put("like", new Integer(SqlParserSymbols.KW_LIKE)); keywordMap.put("limit", new Integer(SqlParserSymbols.KW_LIMIT)); keywordMap.put("link", new Integer(SqlParserSymbols.KW_LINK)); + keywordMap.put("list", new Integer(SqlParserSymbols.KW_LIST)); keywordMap.put("load", new Integer(SqlParserSymbols.KW_LOAD)); keywordMap.put("local", new Integer(SqlParserSymbols.KW_LOCAL)); keywordMap.put("location", new Integer(SqlParserSymbols.KW_LOCATION)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ListPartitionPrunerTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ListPartitionPrunerTest.java new file mode 100644 index 00000000000000..e1cc5772f6f52b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ListPartitionPrunerTest.java @@ -0,0 +1,170 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.utframe.DorisAssert; +import org.apache.doris.utframe.UtFrameUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.UUID; + +public class ListPartitionPrunerTest { + private static String runningDir = "fe/mocked/DemoTest/" + UUID.randomUUID().toString() + "/"; + private static DorisAssert dorisAssert; + + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + + @AfterClass + public static void tearDown() throws Exception { + UtFrameUtils.cleanDorisFeDir(runningDir); + } + + @BeforeClass + public static void setUp() throws Exception { + Config.enable_batch_delete_by_default = true; + FeConstants.runningUnitTest = true; + UtFrameUtils.createMinDorisCluster(runningDir); + + String createSinglePartColWithSinglePartKey = "create table test.t1\n" + + "(k1 int, k2 varchar(128), k3 int, v1 int, v2 int)\n" + +"partition by list(k1)\n" + + "(\n" + + "partition p1 values in (\"1\"),\n" + + "partition p2 values in (\"2\")\n" + + ")\n" + + "distributed by hash(k2) buckets 1\n" + + "properties('replication_num' = '1');"; + String createSinglePartColWithMultiPartKey = "create table test.t2\n" + + "(k1 int, k2 varchar(128), k3 int, v1 int, v2 int)\n" + +"partition by list(k1)\n" + + "(\n" + + "partition p1 values in (\"1\", \"3\", \"5\"),\n" + + "partition p2 values in (\"2\", \"4\", \"6\"),\n" + + "partition p3 values in (\"7\", \"8\")\n" + + ")\n" + + "distributed by hash(k2) buckets 1\n" + + "properties('replication_num' = '1');"; + String createMultiPartColWithSinglePartKey = "create table test.t3\n" + + "(k1 int, k2 varchar(128), k3 int, v1 int, v2 int)\n" + +"partition by list(k1, k2)\n" + + "(\n" + + "partition p1 values in ((\"1\", \"beijing\")),\n" + + "partition p2 values in ((\"2\", \"beijing\"))\n" + + ")\n" + + "distributed by hash(k2) buckets 1\n" + + "properties('replication_num' = '1');"; + String createMultiPartColWithMultiPartKey = "create table test.t4\n" + + "(k1 int, k2 varchar(128), k3 int, v1 int, v2 int)\n" + +"partition by list(k1, k2)\n" + + "(\n" + + "partition p1 values in ((\"1\", \"beijing\"), (\"2\", \"shanghai\")),\n" + + "partition p2 values in ((\"2\", \"beijing\")),\n" + + "partition p3 values in ((\"3\", \"tianjin\"), (\"1\", \"shanghai\"))\n" + + ")\n" + + "distributed by hash(k2) buckets 1\n" + + "properties('replication_num' = '1');"; + dorisAssert = new DorisAssert(); + dorisAssert.withDatabase("test").useDatabase("test"); + dorisAssert.withTable(createSinglePartColWithSinglePartKey) + .withTable(createSinglePartColWithMultiPartKey) + .withTable(createMultiPartColWithSinglePartKey) + .withTable(createMultiPartColWithMultiPartKey); + } + + @Test + public void testSelectWithPartition() throws Exception { + String sql = "select * from t1 partition p1;"; + dorisAssert.query(sql).explainContains("partitions=1/2"); + + sql = "select * from t2 partition (p2, p3);"; + dorisAssert.query(sql).explainContains("partitions=2/3"); + + sql = "select * from t3 partition (p1, p2);"; + dorisAssert.query(sql).explainContains("partitions=2/2"); + + sql = "select * from t4 partition p2;"; + dorisAssert.query(sql).explainContains("partitions=1/3"); + } + + @Test + public void testPartitionPrune() throws Exception { + // single partition column + String sql = "select * from t2 where k1 < 7"; + dorisAssert.query(sql).explainContains("partitions=2/3"); + + sql = "select * from t2 where k1 = 1;"; + dorisAssert.query(sql).explainContains("partitions=1/3"); + + sql = "select * from t2 where k1 in (1, 2);"; + dorisAssert.query(sql).explainContains("partitions=2/3"); + + sql = "select * from t2 where k1 >= 6;"; + dorisAssert.query(sql).explainContains("partitions=2/3"); + + sql = "select * from t2 where k1 < 8 and k1 > 6;"; + dorisAssert.query(sql).explainContains("partitions=1/3"); + + sql = "select * from t2 where k2 = \"beijing\";"; + dorisAssert.query(sql).explainContains("partitions=3/3"); + + // multi partition columns + sql = "select * from t4 where k1 = 2;"; + dorisAssert.query(sql).explainContains("partitions=2/3"); + + sql = "select * from t4 where k2 = \"tianjin\";"; + dorisAssert.query(sql).explainContains("partitions=1/3"); + + sql = "select * from t4 where k1 = 1 and k2 = \"shanghai\";"; + dorisAssert.query(sql).explainContains("partitions=2/3"); + + sql = "select * from t4 where k1 in (1, 3) and k2 in (\"tianjin\", \"shanghai\");"; + dorisAssert.query(sql).explainContains("partitions=2/3"); + + sql = "select * from t4 where k1 in (1, 3);"; + dorisAssert.query(sql).explainContains("partitions=2/3"); + + sql = "select * from t4 where k2 in (\"tianjin\", \"shanghai\");"; + dorisAssert.query(sql).explainContains("partitions=2/3"); + + sql = "select * from t4 where k1 < 3;"; + dorisAssert.query(sql).explainContains("partitions=3/3"); + + sql = "select * from t4 where k1 > 2;"; + dorisAssert.query(sql).explainContains("partitions=1/3"); + + sql = "select * from t4 where k2 <\"shanghai\";"; + dorisAssert.query(sql).explainContains("partitions=2/3"); + + sql = "select * from t4 where k2 >=\"shanghai\";"; + dorisAssert.query(sql).explainContains("partitions=2/3"); + + sql = "select * from t4 where k1 > 1 and k2 < \"shanghai\";"; + dorisAssert.query(sql).explainContains("partitions=2/3"); + + sql = "select * from t4 where k1 >= 2 and k2 = \"shanghai\";"; + dorisAssert.query(sql).explainContains("partitions=2/3"); + } + +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionKeyDescTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionKeyDescTest.java index b694de8a488cbe..ffc8eeffb2dc5d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionKeyDescTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionKeyDescTest.java @@ -17,11 +17,15 @@ package org.apache.doris.analysis; +import org.apache.doris.common.DdlException; + import com.google.common.collect.Lists; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.util.ArrayList; import java.util.List; public class PartitionKeyDescTest { @@ -34,8 +38,8 @@ public void setUp() { } @Test - public void testNormal() { - PartitionKeyDesc desc = new PartitionKeyDesc(values); + public void testNormal() throws DdlException { + PartitionKeyDesc desc = PartitionKeyDesc.createLessThan(values); Assert.assertEquals(values, desc.getUpperValues()); Assert.assertEquals("('1', 'abc')", desc.toSql()); @@ -48,4 +52,24 @@ public void testMax() { Assert.assertNull(desc.getUpperValues()); Assert.assertEquals("MAXVALUE", desc.toSql()); } + + @Test + public void testListMulti() { + List> list = new ArrayList<>(); + list.add(values); + list.add(Lists.newArrayList(new PartitionValue("2"), new PartitionValue("cde"))); + PartitionKeyDesc desc = PartitionKeyDesc.createIn(list); + Assert.assertEquals(list, desc.getInValues()); + Assert.assertEquals("(('1', 'abc'),('2', 'cde'))", desc.toSql()); + } + + @Test + public void testListSingle() { + List> list = new ArrayList<>(); + list.add(Lists.newArrayList(new PartitionValue("1"))); + list.add(Lists.newArrayList(new PartitionValue("2"))); + PartitionKeyDesc desc = PartitionKeyDesc.createIn(list); + Assert.assertEquals(list, desc.getInValues()); + Assert.assertEquals("('1','2')", desc.toSql()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/CatalogMocker.java b/fe/fe-core/src/test/java/org/apache/doris/backup/CatalogMocker.java index 20537be12a6f15..d184f22ac84ca4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/CatalogMocker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/CatalogMocker.java @@ -33,10 +33,12 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.RandomDistributionInfo; import org.apache.doris.catalog.RangePartitionInfo; +import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Replica.ReplicaState; import org.apache.doris.catalog.ScalarType; @@ -294,7 +296,8 @@ public static Database mockDb() throws AnalysisException { PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("10")), Lists.newArrayList(TEST_TBL_BASE_SCHEMA.get(0))); Range rangeP1 = Range.closedOpen(rangeP1Lower, rangeP1Upper); - rangePartitionInfo.setRange(TEST_PARTITION1_ID, false, rangeP1); + PartitionItem item1 = new RangePartitionItem(rangeP1); + rangePartitionInfo.setItem(TEST_PARTITION1_ID, false, item1); PartitionKey rangeP2Lower = PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("10")), @@ -303,7 +306,8 @@ public static Database mockDb() throws AnalysisException { PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("20")), Lists.newArrayList(TEST_TBL_BASE_SCHEMA.get(0))); Range rangeP2 = Range.closedOpen(rangeP2Lower, rangeP2Upper); - rangePartitionInfo.setRange(TEST_PARTITION2_ID, false, rangeP2); + PartitionItem item2 = new RangePartitionItem(rangeP2); + rangePartitionInfo.setItem(TEST_PARTITION1_ID, false, item2); rangePartitionInfo.setReplicationNum(TEST_PARTITION1_ID, (short) 3); rangePartitionInfo.setReplicationNum(TEST_PARTITION2_ID, (short) 3); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java index eb68f6b3cdd55c..0ec5a1b352bef3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java @@ -19,7 +19,7 @@ import org.apache.doris.analysis.PartitionKeyDesc; import org.apache.doris.analysis.PartitionValue; -import org.apache.doris.analysis.SingleRangePartitionDesc; +import org.apache.doris.analysis.SinglePartitionDesc; import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MaterializedIndex.IndexState; import org.apache.doris.catalog.Replica.ReplicaState; @@ -245,13 +245,12 @@ public static void createEsTable(Database db) throws DdlException { // table List partitionColumns = Lists.newArrayList(); - List singleRangePartitionDescs = Lists.newArrayList(); + List singlePartitionDescs = Lists.newArrayList(); partitionColumns.add(userId); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", - new PartitionKeyDesc(Lists - .newArrayList(new PartitionValue("100"))), - null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", + PartitionKeyDesc.createLessThan(Lists.newArrayList(new PartitionValue("100"))), + null)); RangePartitionInfo partitionInfo = new RangePartitionInfo(partitionColumns); Map properties = Maps.newHashMap(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java index 0dabc1817c3807..4aec3be70d8c7d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java @@ -143,6 +143,59 @@ public void testNormal() throws DdlException { + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1',\n" + "'function_column.sequence_type' = 'int');")); + /** + * create table with list partition + */ + // single partition column with single key + ExceptionChecker + .expectThrowsNoException(() -> createTable("create table test.tbl9\n" + + "(k1 int, k2 varchar(128), k3 int, v1 int, v2 int)\n" + +"partition by list(k1)\n" + + "(\n" + + "partition p1 values in (\"1\"),\n" + + "partition p2 values in (\"2\")\n" + + ")\n" + + "distributed by hash(k2) buckets 1\n" + + "properties('replication_num' = '1');")); + + // single partition column with multi keys + ExceptionChecker + .expectThrowsNoException(() -> createTable("create table test.tbl10\n" + + "(k1 int, k2 varchar(128), k3 int, v1 int, v2 int)\n" + +"partition by list(k1)\n" + + "(\n" + + "partition p1 values in (\"1\", \"3\", \"5\"),\n" + + "partition p2 values in (\"2\", \"4\", \"6\"),\n" + + "partition p3 values in (\"7\", \"8\")\n" + + ")\n" + + "distributed by hash(k2) buckets 1\n" + + "properties('replication_num' = '1');")); + + // multi partition columns with single key + ExceptionChecker + .expectThrowsNoException(() -> createTable("create table test.tbl11\n" + + "(k1 int, k2 varchar(128), k3 int, v1 int, v2 int)\n" + +"partition by list(k1, k2)\n" + + "(\n" + + "partition p1 values in ((\"1\", \"beijing\")),\n" + + "partition p2 values in ((\"2\", \"beijing\"))\n" + + ")\n" + + "distributed by hash(k2) buckets 1\n" + + "properties('replication_num' = '1');")); + + // multi partition columns with multi keys + ExceptionChecker + .expectThrowsNoException(() -> createTable("create table test.tbl12\n" + + "(k1 int, k2 varchar(128), k3 int, v1 int, v2 int)\n" + +"partition by list(k1, k2)\n" + + "(\n" + + "partition p1 values in ((\"1\", \"beijing\"), (\"1\", \"shanghai\")),\n" + + "partition p2 values in ((\"2\", \"beijing\"), (\"2\", \"shanghai\")),\n" + + "partition p3 values in ((\"3\", \"tianjin\"))\n" + + ")\n" + + "distributed by hash(k2) buckets 1\n" + + "properties('replication_num' = '1');")); + Database db = Catalog.getCurrentCatalog().getDb("default_cluster:test"); OlapTable tbl6 = (OlapTable) db.getTable("tbl6"); Assert.assertTrue(tbl6.getColumn("k1").isKey()); @@ -222,5 +275,203 @@ public void testAbnormal() throws DdlException { + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1',\n" + "'function_column.sequence_type' = 'double');")); + + /** + * create table with list partition + */ + // single partition column with single key + ExceptionChecker + .expectThrowsWithMsg(AnalysisException.class, "Syntax error", () -> createTable("create table test.tbl9\n" + + "(k1 int, k2 varchar(128), k3 int, v1 int, v2 int)\n" + +"partition by list(k1)\n" + + "(\n" + + "partition p1 values in (\"1\"),\n" + + "partition p2 values in ()\n" + + ")\n" + + "distributed by hash(k2) buckets 1\n" + + "properties('replication_num' = '1');")); + + // single partition column with multi keys + ExceptionChecker + .expectThrowsWithMsg(IllegalArgumentException.class, "partition key desc list size[2] is not equal to partition column size[1]", + () -> createTable("create table test.tbl10\n" + + "(k1 int, k2 varchar(128), k3 int, v1 int, v2 int)\n" + +"partition by list(k1)\n" + + "(\n" + + "partition p1 values in (\"1\", \"3\", \"5\"),\n" + + "partition p2 values in (\"2\", \"4\", \"6\"),\n" + + "partition p3 values in ((\"7\", \"8\"))\n" + + ")\n" + + "distributed by hash(k2) buckets 1\n" + + "properties('replication_num' = '1');")); + + // multi partition columns with single key + ExceptionChecker + .expectThrowsWithMsg(IllegalArgumentException.class, "partition key desc list size[1] is not equal to partition column size[2]", + () -> createTable("create table test.tbl11\n" + + "(k1 int, k2 varchar(128), k3 int, v1 int, v2 int)\n" + +"partition by list(k1, k2)\n" + + "(\n" + + "partition p1 values in ((\"1\", \"beijing\")),\n" + + "partition p2 values in (\"2\", \"beijing\")\n" + + ")\n" + + "distributed by hash(k2) buckets 1\n" + + "properties('replication_num' = '1');")); + + // multi partition columns with multi keys + ExceptionChecker + .expectThrowsWithMsg(IllegalArgumentException.class, "partition key desc list size[3] is not equal to partition column size[2]", + () -> createTable("create table test.tbl12\n" + + "(k1 int, k2 varchar(128), k3 int, v1 int, v2 int)\n" + +"partition by list(k1, k2)\n" + + "(\n" + + "partition p1 values in ((\"1\", \"beijing\"), (\"1\", \"shanghai\")),\n" + + "partition p2 values in ((\"2\", \"beijing\"), (\"2\", \"shanghai\")),\n" + + "partition p3 values in ((\"3\", \"tianjin\", \"3\"))\n" + + ")\n" + + "distributed by hash(k2) buckets 1\n" + + "properties('replication_num' = '1');")); + + // multi partition columns with multi keys + ExceptionChecker + .expectThrowsWithMsg(AnalysisException.class, "Syntax error", + () -> createTable("create table test.tbl13\n" + + "(k1 int, k2 varchar(128), k3 int, v1 int, v2 int)\n" + +"partition by list(k1, k2)\n" + + "(\n" + + "partition p1 values in ((\"1\", \"beijing\"), (\"1\", \"shanghai\")),\n" + + "partition p2 values in ((\"2\", \"beijing\"), (\"2\", \"shanghai\")),\n" + + "partition p3 values in ()\n" + + ")\n" + + "distributed by hash(k2) buckets 1\n" + + "properties('replication_num' = '1');")); + + /** + * create table with both list and range partition + */ + // list contain less than + ExceptionChecker + .expectThrowsWithMsg(AnalysisException.class, "You can only use in values to create list partitions", + () -> createTable("CREATE TABLE test.tbl14 (\n" + + " k1 int, k2 varchar(128), k3 int, v1 int, v2 int\n" + + ")\n" + + "PARTITION BY LIST(k1)\n" + + "(\n" + + " PARTITION p1 VALUES less than (\"1\"),\n" + + " PARTITION p2 VALUES less than (\"2\"),\n" + + " partition p3 values less than (\"5\")\n" + + ")DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES(\"replication_num\" = \"1\");")); + + // range contain in + ExceptionChecker + .expectThrowsWithMsg(AnalysisException.class, "You can only use fixed or less than values to create range partitions", + () -> createTable("CREATE TABLE test.tbl15 (\n" + + " k1 int, k2 varchar(128), k3 int, v1 int, v2 int\n" + + ")\n" + + "PARTITION BY range(k1)\n" + + "(\n" + + " PARTITION p1 VALUES in (\"1\"),\n" + + " PARTITION p2 VALUES in (\"2\"),\n" + + " partition p3 values in (\"5\")\n" + + ")DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES(\"replication_num\" = \"1\");")); + + // list contain both + ExceptionChecker + .expectThrowsWithMsg(AnalysisException.class, "You can only use in values to create list partitions", + () -> createTable("CREATE TABLE test.tbl15 (\n" + + " k1 int, k2 varchar(128), k3 int, v1 int, v2 int\n" + + ")\n" + + "PARTITION BY LIST(k1)\n" + + "(\n" + + " PARTITION p1 VALUES in (\"1\"),\n" + + " PARTITION p2 VALUES in (\"2\"),\n" + + " partition p3 values less than (\"5\")\n" + + ")DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES(\"replication_num\" = \"1\");")); + + // range contain both + ExceptionChecker + .expectThrowsWithMsg(AnalysisException.class, "You can only use fixed or less than values to create range partitions", + () -> createTable("CREATE TABLE test.tbl16 (\n" + + " k1 int, k2 varchar(128), k3 int, v1 int, v2 int\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 VALUES less than (\"1\"),\n" + + " PARTITION p2 VALUES less than (\"2\"),\n" + + " partition p3 values in (\"5\")\n" + + ")DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES(\"replication_num\" = \"1\");")); + + // range: partition content != partition key type + ExceptionChecker + .expectThrowsWithMsg(DdlException.class, "Invalid number format: beijing", + () -> createTable("CREATE TABLE test.tbl17 (\n" + + " k1 int, k2 varchar(128), k3 int, v1 int, v2 int\n" + + ")\n" + + "PARTITION BY range(k1)\n" + + "(\n" + + " PARTITION p1 VALUES less than (\"beijing\"),\n" + + " PARTITION p2 VALUES less than (\"shanghai\"),\n" + + " partition p3 values less than (\"tianjin\")\n" + + ")DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES(\"replication_num\" = \"1\");")); + + // list: partition content != partition key type + ExceptionChecker + .expectThrowsWithMsg(DdlException.class, "Invalid number format: beijing", + () -> createTable("CREATE TABLE test.tbl18 (\n" + + " k1 int, k2 varchar(128), k3 int, v1 int, v2 int\n" + + ")\n" + + "PARTITION BY list(k1)\n" + + "(\n" + + " PARTITION p1 VALUES in (\"beijing\"),\n" + + " PARTITION p2 VALUES in (\"shanghai\"),\n" + + " partition p3 values in (\"tianjin\")\n" + + ")DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES(\"replication_num\" = \"1\");")); + + /** + * dynamic partition table + */ + // list partition with dynamic properties + ExceptionChecker + .expectThrowsWithMsg(DdlException.class, "Only support dynamic partition properties on range partition table", + () -> createTable("CREATE TABLE test.tbl19\n" + + "(\n" + + " k1 DATE\n" + + ")\n" + + "PARTITION BY LIST(k1) ()\n" + + "DISTRIBUTED BY HASH(k1)\n" + + "PROPERTIES\n" + + "(\n" + + " \"dynamic_partition.enable\" = \"true\",\n" + + " \"dynamic_partition.time_unit\" = \"MONTH\",\n" + + " \"dynamic_partition.end\" = \"2\",\n" + + " \"dynamic_partition.prefix\" = \"p\",\n" + + " \"dynamic_partition.buckets\" = \"8\",\n" + + " \"dynamic_partition.start_day_of_month\" = \"3\"\n" + + ");\n")); + + // no partition table with dynamic properties + ExceptionChecker + .expectThrowsWithMsg(DdlException.class, "Only support dynamic partition properties on range partition table", + () -> createTable("CREATE TABLE test.tbl20\n" + + "(\n" + + " k1 DATE\n" + + ")\n" + + "DISTRIBUTED BY HASH(k1)\n" + + "PROPERTIES\n" + + "(\n" + + " \"dynamic_partition.enable\" = \"true\",\n" + + " \"dynamic_partition.time_unit\" = \"MONTH\",\n" + + " \"dynamic_partition.end\" = \"2\",\n" + + " \"dynamic_partition.prefix\" = \"p\",\n" + + " \"dynamic_partition.buckets\" = \"8\",\n" + + " \"dynamic_partition.start_day_of_month\" = \"3\"\n" + + ");")); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ListPartitionInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ListPartitionInfoTest.java new file mode 100644 index 00000000000000..ec481a77a7c169 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ListPartitionInfoTest.java @@ -0,0 +1,227 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.analysis.PartitionKeyDesc; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.analysis.SinglePartitionDesc; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; + +import com.google.common.collect.Lists; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +public class ListPartitionInfoTest { + private List partitionColumns; + private ListPartitionInfo partitionInfo; + + private List singlePartitionDescs; + + @Before + public void setUp() { + partitionColumns = new LinkedList<>(); + singlePartitionDescs = new LinkedList<>(); + } + + @Test + public void testTinyInt() throws AnalysisException, DdlException { + Column k1 = new Column("k1", new ScalarType(PrimitiveType.TINYINT), true, null, "", ""); + partitionColumns.add(k1); + + List> inValues = new ArrayList<>(); + inValues.add(Lists.newArrayList(new PartitionValue("-128"))); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", + PartitionKeyDesc.createIn(inValues), null)); + + partitionInfo = new ListPartitionInfo(partitionColumns); + PartitionItem partitionItem = null; + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(1, null); + partitionItem = partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); + } + Assert.assertEquals("-128", ((ListPartitionItem) partitionItem).getItems().get(0).getKeys().get(0).getStringValue()); + + } + + @Test + public void testSmallInt() throws AnalysisException, DdlException { + Column k1 = new Column("k1", new ScalarType(PrimitiveType.SMALLINT), true, null, "", ""); + partitionColumns.add(k1); + + List> inValues = new ArrayList<>(); + inValues.add(Lists.newArrayList(new PartitionValue("-32768"))); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", + PartitionKeyDesc.createIn(inValues), null)); + + partitionInfo = new ListPartitionInfo(partitionColumns); + PartitionItem partitionItem = null; + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(1, null); + partitionItem = partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); + } + Assert.assertEquals("-32768", ((ListPartitionItem) partitionItem).getItems().get(0).getKeys().get(0).getStringValue()); + } + + @Test + public void testInt() throws DdlException, AnalysisException { + Column k1 = new Column("k1", new ScalarType(PrimitiveType.INT), true, null, "", ""); + partitionColumns.add(k1); + + List> inValues = new ArrayList<>(); + inValues.add(Lists.newArrayList(new PartitionValue("-2147483648"))); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", + PartitionKeyDesc.createIn(inValues), null)); + + partitionInfo = new ListPartitionInfo(partitionColumns); + PartitionItem partitionItem = null; + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(1, null); + partitionItem = partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); + } + Assert.assertEquals("-2147483648", ((ListPartitionItem) partitionItem).getItems().get(0).getKeys().get(0).getStringValue()); + } + + @Test + public void testBigInt() throws AnalysisException, DdlException { + Column k1 = new Column("k1", new ScalarType(PrimitiveType.BIGINT), true, null, "", ""); + partitionColumns.add(k1); + + List> inValues = new ArrayList<>(); + inValues.add(Lists.newArrayList(new PartitionValue("-9223372036854775808"))); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", + PartitionKeyDesc.createIn(inValues), null)); + + partitionInfo = new ListPartitionInfo(partitionColumns); + PartitionItem partitionItem = null; + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(1, null); + partitionItem = partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); + } + Assert.assertEquals("-9223372036854775808", ((ListPartitionItem) partitionItem).getItems().get(0).getKeys().get(0).getStringValue()); + } + + @Test + public void testLargeInt() throws AnalysisException, DdlException { + Column k1 = new Column("k1", new ScalarType(PrimitiveType.LARGEINT), true, null, "", ""); + partitionColumns.add(k1); + + List> inValues = new ArrayList<>(); + inValues.add(Lists.newArrayList(new PartitionValue("-170141183460469231731687303715884105728"))); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", + PartitionKeyDesc.createIn(inValues), null)); + + partitionInfo = new ListPartitionInfo(partitionColumns); + PartitionItem partitionItem = null; + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(1, null); + partitionItem = partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); + } + Assert.assertEquals("-170141183460469231731687303715884105728", ((ListPartitionItem) partitionItem).getItems().get(0).getKeys().get(0).getStringValue()); + } + + @Test + public void testString() throws AnalysisException, DdlException { + Column k1 = new Column("k1", new ScalarType(PrimitiveType.CHAR), true, null, "", ""); + partitionColumns.add(k1); + + List> inValues = new ArrayList<>(); + inValues.add(Lists.newArrayList(new PartitionValue("Beijing"))); + inValues.add(Lists.newArrayList(new PartitionValue("Shanghai"))); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", + PartitionKeyDesc.createIn(inValues), null)); + + partitionInfo = new ListPartitionInfo(partitionColumns); + PartitionItem partitionItem = null; + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(1, null); + partitionItem = partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); + } + Assert.assertEquals("Beijing", ((ListPartitionItem) partitionItem).getItems().get(0).getKeys().get(0).getStringValue()); + Assert.assertEquals("Shanghai", ((ListPartitionItem) partitionItem).getItems().get(1).getKeys().get(0).getStringValue()); + } + + @Test + public void testBoolean() throws AnalysisException, DdlException { + Column k1 = new Column("k1", new ScalarType(PrimitiveType.BOOLEAN), true, null, "", ""); + partitionColumns.add(k1); + + List> inValues = new ArrayList<>(); + inValues.add(Lists.newArrayList(new PartitionValue("true"))); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", + PartitionKeyDesc.createIn(inValues), null)); + + partitionInfo = new ListPartitionInfo(partitionColumns); + PartitionItem partitionItem = null; + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(1, null); + partitionItem = partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); + } + Assert.assertEquals(true, ((ListPartitionItem) partitionItem).getItems().get(0).getKeys().get(0).getRealValue()); + } + + @Test(expected = DdlException.class) + public void testDuplicateKey() throws AnalysisException, DdlException { + Column k1 = new Column("k1", new ScalarType(PrimitiveType.VARCHAR), true, null, "", ""); + partitionColumns.add(k1); + + List> inValues = new ArrayList<>(); + inValues.add(Lists.newArrayList(new PartitionValue("beijing"))); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", + PartitionKeyDesc.createIn(inValues), null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p2", + PartitionKeyDesc.createIn(inValues), null)); + + + partitionInfo = new ListPartitionInfo(partitionColumns); + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(1, null); + partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); + } + } + + @Test + public void testMultiPartitionKeys() throws AnalysisException, DdlException { + Column k1 = new Column("k1", new ScalarType(PrimitiveType.VARCHAR), true, null, "", ""); + Column k2 = new Column("k2", new ScalarType(PrimitiveType.INT), true, null, "", ""); + partitionColumns.add(k1); + partitionColumns.add(k2); + + List> inValues = new ArrayList<>(); + inValues.add(Lists.newArrayList(new PartitionValue("beijing"), new PartitionValue("100"))); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", + PartitionKeyDesc.createIn(inValues), null)); + + + partitionInfo = new ListPartitionInfo(partitionColumns); + PartitionItem partitionItem = null; + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(2, null); + partitionItem = partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); + } + + Assert.assertEquals("beijing", ((ListPartitionItem) partitionItem).getItems().get(0).getKeys().get(0).getRealValue()); + Assert.assertEquals(100, ((ListPartitionItem) partitionItem).getItems().get(0).getKeys().get(1).getLongValue()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java index 5c0142ed2b3825..e6eb4ec0e0e8ea 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java @@ -45,6 +45,9 @@ public class PartitionKeyTest { private static Column largeInt; private static Column date; private static Column datetime; + private static Column charString; + private static Column varchar; + private static Column bool; private Catalog catalog; @@ -60,6 +63,9 @@ public static void setUp() { largeInt = new Column("largeint", PrimitiveType.LARGEINT); date = new Column("date", PrimitiveType.DATE); datetime = new Column("datetime", PrimitiveType.DATETIME); + charString = new Column("char", PrimitiveType.CHAR); + varchar = new Column("varchar", PrimitiveType.VARCHAR); + bool = new Column("bool", PrimitiveType.BOOLEAN); allColumns = Arrays.asList(tinyInt, smallInt, int32, bigInt, largeInt, date, datetime); } @@ -146,6 +152,48 @@ public void compareTest() throws AnalysisException { allColumns); pk2 = PartitionKey.createInfinityPartitionKey(allColumns, false); Assert.assertTrue(!pk1.equals(pk2) && pk1.compareTo(pk2) == 1); + + // case11 + pk1 = PartitionKey.createPartitionKey(Arrays.asList(new PartitionValue("beijing"), new PartitionValue("shanghai")), + Arrays.asList(charString, varchar)); + pk2 = PartitionKey.createPartitionKey(Arrays.asList(new PartitionValue("beijing"), new PartitionValue("shanghai")), + Arrays.asList(charString, varchar)); + Assert.assertTrue(pk1.equals(pk2) && pk1.compareTo(pk2) == 0); + + // case12 + pk1 = PartitionKey.createPartitionKey(Arrays.asList(new PartitionValue("beijing"), new PartitionValue("shanghai")), + Arrays.asList(charString, varchar)); + pk2 = PartitionKey.createPartitionKey(Arrays.asList(new PartitionValue("shijiazhuang"), new PartitionValue("tianjin")), + Arrays.asList(charString, varchar)); + Assert.assertTrue(!pk1.equals(pk2) && pk1.compareTo(pk2) == -1); + + // case13 + pk1 = PartitionKey.createPartitionKey(Arrays.asList(new PartitionValue("beijing"), new PartitionValue("shanghai")), + Arrays.asList(charString, varchar)); + pk2 = PartitionKey.createPartitionKey(Arrays.asList(new PartitionValue("beijing"), new PartitionValue("tianjin")), + Arrays.asList(charString, varchar)); + Assert.assertTrue(!pk1.equals(pk2) && pk1.compareTo(pk2) == -1); + + // case14 + pk1 = PartitionKey.createPartitionKey(Arrays.asList(new PartitionValue("true")), + Arrays.asList(bool)); + pk2 = PartitionKey.createPartitionKey(Arrays.asList(new PartitionValue("false")), + Arrays.asList(bool)); + Assert.assertTrue(!pk1.equals(pk2) && pk1.compareTo(pk2) == 1); + + // case15 + pk1 = PartitionKey.createPartitionKey(Arrays.asList(new PartitionValue("true")), + Arrays.asList(bool)); + pk2 = PartitionKey.createPartitionKey(Arrays.asList(new PartitionValue("true")), + Arrays.asList(bool)); + Assert.assertTrue(pk1.equals(pk2) && pk1.compareTo(pk2) == 0); + + // case16 + pk1 = PartitionKey.createPartitionKey(Arrays.asList(new PartitionValue("false")), + Arrays.asList(bool)); + pk2 = PartitionKey.createPartitionKey(Arrays.asList(new PartitionValue("false")), + Arrays.asList(bool)); + Assert.assertTrue(pk1.equals(pk2) && pk1.compareTo(pk2) == 0); } @Test @@ -175,7 +223,15 @@ public void testSerialization() throws Exception { columns.add(new Column("column10", ScalarType.createType(PrimitiveType.DATE), true, null, "", "")); keys.add(new PartitionValue("2014-12-27 11:12:13")); columns.add(new Column("column11", ScalarType.createType(PrimitiveType.DATETIME), true, null, "", "")); - + keys.add(new PartitionValue("beijing")); + columns.add(new Column("column12", ScalarType.createType(PrimitiveType.VARCHAR), true, null, "", "")); + keys.add(new PartitionValue("shanghai")); + columns.add(new Column("column13", ScalarType.createType(PrimitiveType.CHAR), true, null, "", "")); + keys.add(new PartitionValue("true")); + columns.add(new Column("column14", ScalarType.createType(PrimitiveType.BOOLEAN), true, null, "", "")); + keys.add(new PartitionValue("false")); + columns.add(new Column("column15", ScalarType.createType(PrimitiveType.BOOLEAN), true, null, "", "")); + PartitionKey key = PartitionKey.createPartitionKey(keys, columns); key.write(dos); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RangePartitionInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RangePartitionInfoTest.java index 3a758021d5df37..2f5217849ca284 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RangePartitionInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RangePartitionInfoTest.java @@ -18,9 +18,9 @@ package org.apache.doris.catalog; import org.apache.doris.analysis.PartitionKeyDesc; -import org.apache.doris.analysis.PartitionKeyDesc.PartitionRangeType; +import org.apache.doris.analysis.PartitionKeyDesc.PartitionKeyValueType; import org.apache.doris.analysis.PartitionValue; -import org.apache.doris.analysis.SingleRangePartitionDesc; +import org.apache.doris.analysis.SinglePartitionDesc; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; @@ -38,12 +38,12 @@ public class RangePartitionInfoTest { private List partitionColumns; private RangePartitionInfo partitionInfo; - private List singleRangePartitionDescs; + private List singlePartitionDescs; @Before public void setUp() { partitionColumns = new LinkedList(); - singleRangePartitionDescs = new LinkedList(); + singlePartitionDescs = new LinkedList(); } @Test(expected = DdlException.class) @@ -51,15 +51,15 @@ public void testTinyInt() throws DdlException, AnalysisException { Column k1 = new Column("k1", new ScalarType(PrimitiveType.TINYINT), true, null, "", ""); partitionColumns.add(k1); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", - new PartitionKeyDesc(Lists .newArrayList(new PartitionValue("-128"))), - null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", + PartitionKeyDesc.createLessThan(Lists.newArrayList(new PartitionValue("-128"))), + null)); partitionInfo = new RangePartitionInfo(partitionColumns); - for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { - singleRangePartitionDesc.analyze(1, null); - partitionInfo.handleNewSinglePartitionDesc(singleRangePartitionDesc, 20000L, false); + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(1, null); + partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); } } @@ -68,14 +68,14 @@ public void testSmallInt() throws DdlException, AnalysisException { Column k1 = new Column("k1", new ScalarType(PrimitiveType.SMALLINT), true, null, "", ""); partitionColumns.add(k1); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", - new PartitionKeyDesc(Lists.newArrayList(new PartitionValue("-32768"))), - null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", + PartitionKeyDesc.createLessThan(Lists.newArrayList(new PartitionValue("-32768"))), + null)); partitionInfo = new RangePartitionInfo(partitionColumns); - for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { - singleRangePartitionDesc.analyze(1, null); - partitionInfo.handleNewSinglePartitionDesc(singleRangePartitionDesc, 20000L, false); + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(1, null); + partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); } } @@ -84,14 +84,14 @@ public void testInt() throws DdlException, AnalysisException { Column k1 = new Column("k1", new ScalarType(PrimitiveType.INT), true, null, "", ""); partitionColumns.add(k1); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", - new PartitionKeyDesc(Lists.newArrayList(new PartitionValue("-2147483648"))), - null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", + PartitionKeyDesc.createLessThan(Lists.newArrayList(new PartitionValue("-2147483648"))), + null)); partitionInfo = new RangePartitionInfo(partitionColumns); - for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { - singleRangePartitionDesc.analyze(1, null); - partitionInfo.handleNewSinglePartitionDesc(singleRangePartitionDesc, 20000L, false); + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(1, null); + partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); } } @@ -100,20 +100,20 @@ public void testBigInt() throws DdlException, AnalysisException { Column k1 = new Column("k1", new ScalarType(PrimitiveType.BIGINT), true, null, "", ""); partitionColumns.add(k1); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", new PartitionKeyDesc(Lists + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", PartitionKeyDesc.createLessThan(Lists .newArrayList(new PartitionValue("-9223372036854775808"))), null)); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p2", new PartitionKeyDesc(Lists + singlePartitionDescs.add(new SinglePartitionDesc(false, "p2", PartitionKeyDesc.createLessThan(Lists .newArrayList(new PartitionValue("-9223372036854775806"))), null)); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p3", new PartitionKeyDesc(Lists + singlePartitionDescs.add(new SinglePartitionDesc(false, "p3", PartitionKeyDesc.createLessThan(Lists .newArrayList(new PartitionValue("0"))), null)); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p4", new PartitionKeyDesc(Lists + singlePartitionDescs.add(new SinglePartitionDesc(false, "p4", PartitionKeyDesc.createLessThan(Lists .newArrayList(new PartitionValue("9223372036854775806"))), null)); partitionInfo = new RangePartitionInfo(partitionColumns); - for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { - singleRangePartitionDesc.analyze(1, null); - partitionInfo.handleNewSinglePartitionDesc(singleRangePartitionDesc, 20000L, false); + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(1, null); + partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); } } @@ -122,20 +122,20 @@ public void testBigIntNormal() throws DdlException, AnalysisException { Column k1 = new Column("k1", new ScalarType(PrimitiveType.BIGINT), true, null, "", ""); partitionColumns.add(k1); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", new PartitionKeyDesc(Lists + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", PartitionKeyDesc.createLessThan(Lists .newArrayList(new PartitionValue("-9223372036854775806"))), null)); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p2", new PartitionKeyDesc(Lists + singlePartitionDescs.add(new SinglePartitionDesc(false, "p2", PartitionKeyDesc.createLessThan(Lists .newArrayList(new PartitionValue("-9223372036854775805"))), null)); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p3", new PartitionKeyDesc(Lists + singlePartitionDescs.add(new SinglePartitionDesc(false, "p3", PartitionKeyDesc.createLessThan(Lists .newArrayList(new PartitionValue("0"))), null)); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p4", new PartitionKeyDesc(Lists + singlePartitionDescs.add(new SinglePartitionDesc(false, "p4", PartitionKeyDesc.createLessThan(Lists .newArrayList(new PartitionValue("9223372036854775806"))), null)); partitionInfo = new RangePartitionInfo(partitionColumns); - for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { - singleRangePartitionDesc.analyze(1, null); - partitionInfo.handleNewSinglePartitionDesc(singleRangePartitionDesc, 20000L, false); + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(1, null); + partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); } } @@ -157,29 +157,29 @@ public void testFixedRange() throws DdlException, AnalysisException { partitionColumns.add(k2); //add RangePartitionDescs - PartitionKeyDesc p1 = new PartitionKeyDesc( + PartitionKeyDesc p1 = PartitionKeyDesc.createFixed( Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("100")), Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("200"))); - PartitionKeyDesc p2 = new PartitionKeyDesc( + PartitionKeyDesc p2 = PartitionKeyDesc.createFixed( Lists.newArrayList(new PartitionValue("20190105"), new PartitionValue("10")), Lists.newArrayList(new PartitionValue("20190107"), new PartitionValue("10"))); - PartitionKeyDesc p3 = new PartitionKeyDesc( + PartitionKeyDesc p3 = PartitionKeyDesc.createFixed( Lists.newArrayList(new PartitionValue("20181231"), new PartitionValue("10")), Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("100"))); - PartitionKeyDesc p4 = new PartitionKeyDesc( + PartitionKeyDesc p4 = PartitionKeyDesc.createFixed( Lists.newArrayList(new PartitionValue("20190105"), new PartitionValue("100")), Lists.newArrayList(new PartitionValue("20190120"), new PartitionValue("10000000000"))); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", p1, null)); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p2", p2, null)); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p3", p3, null)); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p4", p4, null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", p1, null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p2", p2, null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p3", p3, null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p4", p4, null)); partitionInfo = new RangePartitionInfo(partitionColumns); - for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { - singleRangePartitionDesc.analyze(columns, null); - partitionInfo.handleNewSinglePartitionDesc(singleRangePartitionDesc, 20000L, false); + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(columns, null); + partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); } } @@ -202,28 +202,28 @@ public void testFixedRange1() throws DdlException, AnalysisException { partitionColumns.add(k3); //add RangePartitionDescs - PartitionKeyDesc p1 = new PartitionKeyDesc( + PartitionKeyDesc p1 = PartitionKeyDesc.createLessThan( Lists.newArrayList(new PartitionValue("2019-02-01"), new PartitionValue("100"), new PartitionValue("200"))); - PartitionKeyDesc p2 = new PartitionKeyDesc( + PartitionKeyDesc p2 = PartitionKeyDesc.createFixed( Lists.newArrayList(new PartitionValue("2020-02-01"), new PartitionValue("100"), new PartitionValue("200")), Lists.newArrayList(new PartitionValue("10000000000"))); - PartitionKeyDesc p3 = new PartitionKeyDesc( + PartitionKeyDesc p3 = PartitionKeyDesc.createLessThan( Lists.newArrayList(new PartitionValue("2021-02-01"))); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", p1, null)); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p2", p2, null)); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p3", p3, null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", p1, null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p2", p2, null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p3", p3, null)); partitionInfo = new RangePartitionInfo(partitionColumns); - PartitionRangeType partitionType = PartitionRangeType.INVALID; - for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { + PartitionKeyValueType partitionKeyValueType = PartitionKeyValueType.INVALID; + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { // check partitionType - if (partitionType == PartitionRangeType.INVALID) { - partitionType = singleRangePartitionDesc.getPartitionKeyDesc().getPartitionType(); - } else if (partitionType != singleRangePartitionDesc.getPartitionKeyDesc().getPartitionType()) { + if (partitionKeyValueType == PartitionKeyValueType.INVALID) { + partitionKeyValueType = singlePartitionDesc.getPartitionKeyDesc().getPartitionType(); + } else if (partitionKeyValueType != singlePartitionDesc.getPartitionKeyDesc().getPartitionType()) { throw new AnalysisException("You can only use one of these methods to create partitions"); } - singleRangePartitionDesc.analyze(partitionColumns.size(), null); - partitionInfo.handleNewSinglePartitionDesc(singleRangePartitionDesc, 20000L, false); + singlePartitionDesc.analyze(partitionColumns.size(), null); + partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); } } @@ -242,16 +242,16 @@ public void testFixedRange2() throws DdlException, AnalysisException { partitionColumns.add(k2); //add RangePartitionDescs - PartitionKeyDesc p1 = new PartitionKeyDesc(new ArrayList<>(), + PartitionKeyDesc p1 = PartitionKeyDesc.createFixed(new ArrayList<>(), Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("200"))); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", p1, null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", p1, null)); partitionInfo = new RangePartitionInfo(partitionColumns); - for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { - singleRangePartitionDesc.analyze(columns, null); - partitionInfo.handleNewSinglePartitionDesc(singleRangePartitionDesc, 20000L, false); + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(columns, null); + partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); } } @@ -271,17 +271,17 @@ public void testFixedRange3() throws DdlException, AnalysisException { partitionColumns.add(k2); //add RangePartitionDescs - PartitionKeyDesc p1 = new PartitionKeyDesc( + PartitionKeyDesc p1 = PartitionKeyDesc.createFixed( Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("200")), new ArrayList<>()); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", p1, null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", p1, null)); partitionInfo = new RangePartitionInfo(partitionColumns); - for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { - singleRangePartitionDesc.analyze(columns, null); - partitionInfo.handleNewSinglePartitionDesc(singleRangePartitionDesc, 20000L, false); + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(columns, null); + partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); } } @@ -300,17 +300,17 @@ public void testFixedRange4() throws DdlException, AnalysisException { partitionColumns.add(k2); //add RangePartitionDescs - PartitionKeyDesc p1 = new PartitionKeyDesc( + PartitionKeyDesc p1 = PartitionKeyDesc.createFixed( Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("100")), Lists.newArrayList(new PartitionValue("20190201"))); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", p1, null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", p1, null)); partitionInfo = new RangePartitionInfo(partitionColumns); - for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { - singleRangePartitionDesc.analyze(columns, null); - partitionInfo.handleNewSinglePartitionDesc(singleRangePartitionDesc, 20000L, false); + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(columns, null); + partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); } } @@ -330,17 +330,17 @@ public void testFixedRange5() throws DdlException, AnalysisException { partitionColumns.add(k2); //add RangePartitionDescs - PartitionKeyDesc p1 = new PartitionKeyDesc( + PartitionKeyDesc p1 = PartitionKeyDesc.createFixed( Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("100")), Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("100"))); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", p1, null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", p1, null)); partitionInfo = new RangePartitionInfo(partitionColumns); - for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { - singleRangePartitionDesc.analyze(columns, null); - partitionInfo.handleNewSinglePartitionDesc(singleRangePartitionDesc, 20000L, false); + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(columns, null); + partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TempPartitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/TempPartitionTest.java index b39b3c0454ffa2..88fce4770a7b72 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TempPartitionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TempPartitionTest.java @@ -68,9 +68,9 @@ public class TempPartitionTest { @BeforeClass public static void setup() throws Exception { + FeConstants.default_scheduler_interval_millisecond = 100; UtFrameUtils.createMinDorisCluster(runningDir); ctx = UtFrameUtils.createDefaultCtx(); - FeConstants.default_scheduler_interval_millisecond = 100; } @AfterClass @@ -558,6 +558,657 @@ public void testForStrictRangeCheck() throws Exception { checkShowPartitionsResultNum("db3.tbl3", false, 2); checkShowPartitionsResultNum("db3.tbl3", true, 0); } + + @Test + public void testForListPartitionTable() throws Exception { + MetaContext metaContext = new MetaContext(); + metaContext.setMetaVersion(FeMetaVersion.VERSION_CURRENT); + metaContext.setThreadLocalInfo(); + + // create database db4 + String createDbStmtStr = "create database db4;"; + CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, ctx); + Catalog.getCurrentCatalog().createDb(createDbStmt); + System.out.println(Catalog.getCurrentCatalog().getDbNames()); + + // create table tbl4 + String createTblStmtStr1 = "create table db4.tbl4 (k1 int, k2 int)\n" + + "partition by list(k1)\n" + + "(\n" + + "partition p1 values in ('1', '2', '3'),\n" + + "partition p2 values in ('4', '5', '6'),\n" + + "partition p3 values in ('7', '8', '9')\n" + + ")\n" + + "distributed by hash(k2) buckets 1\n" + + "properties('replication_num' = '1');"; + CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(createTblStmtStr1, ctx); + Catalog.getCurrentCatalog().createTable(createTableStmt); + + Database db4 = Catalog.getCurrentCatalog().getDb("default_cluster:db4"); + OlapTable tbl4 = (OlapTable) db4.getTable("tbl4"); + + testSerializeOlapTable(tbl4); + + Map originPartitionTabletIds = Maps.newHashMap(); + getPartitionNameToTabletIdMap("db4.tbl4", false, originPartitionTabletIds); + Assert.assertEquals(3, originPartitionTabletIds.keySet().size()); + + // show temp partition + checkShowPartitionsResultNum("db4.tbl4", true, 0); + checkShowPartitionsResultNum("db4.tbl4", false, 3); + + // add temp partition with duplicate name + String stmtStr = "alter table db4.tbl4 add temporary partition p1 values in ('1', '2', '3');"; + alterTable(stmtStr, true); + + // add temp partition + stmtStr = "alter table db4.tbl4 add temporary partition tp1 values in ('1', '2', '3');"; + alterTable(stmtStr, false); + + stmtStr = "alter table db4.tbl4 add temporary partition tp2 values in ('1', '2', '3');"; + alterTable(stmtStr, true); + + stmtStr = "alter table db4.tbl4 add temporary partition tp1 values in ('4', '5', '6');"; + alterTable(stmtStr, true); + + stmtStr = "alter table db4.tbl4 add temporary partition tp2 values in ('4', '5', '6');"; + alterTable(stmtStr, false); + + stmtStr = "alter table db4.tbl4 add temporary partition tp3 values in ('6', '7', '8', '9');"; + alterTable(stmtStr, true); + + stmtStr = "alter table db4.tbl4 add temporary partition tp3 values in ('7', '8', '9');"; + alterTable(stmtStr, false); + + Map tempPartitionTabletIds = Maps.newHashMap(); + getPartitionNameToTabletIdMap("db4.tbl4", true, tempPartitionTabletIds); + Assert.assertEquals(3, tempPartitionTabletIds.keySet().size()); + + System.out.println("partition tablets: " + originPartitionTabletIds); + System.out.println("temp partition tablets: " + tempPartitionTabletIds); + + testSerializeOlapTable(tbl4); + + // drop non exist temp partition + stmtStr = "alter table db4.tbl4 drop temporary partition tp4;"; + alterTable(stmtStr, true); + + stmtStr = "alter table db4.tbl4 drop temporary partition if exists tp4;"; + alterTable(stmtStr, false); + + stmtStr = "alter table db4.tbl4 drop temporary partition tp3;"; + alterTable(stmtStr, false); + + Map originPartitionTabletIds2 = Maps.newHashMap(); + getPartitionNameToTabletIdMap("db4.tbl4", false, originPartitionTabletIds2); + Assert.assertEquals(originPartitionTabletIds2, originPartitionTabletIds); + + Map tempPartitionTabletIds2 = Maps.newHashMap(); + getPartitionNameToTabletIdMap("db4.tbl4", true, tempPartitionTabletIds2); + Assert.assertEquals(2, tempPartitionTabletIds2.keySet().size()); + Assert.assertTrue(!tempPartitionTabletIds2.containsKey("tp3")); + + checkShowPartitionsResultNum("db4.tbl4", true, 2); + checkShowPartitionsResultNum("db4.tbl4", false, 3); + + stmtStr = "alter table db4.tbl4 add temporary partition tp3 values in ('7', '8', '9');"; + alterTable(stmtStr, false); + checkShowPartitionsResultNum("db4.tbl4", true, 3); + + stmtStr = "alter table db4.tbl4 drop partition p1;"; + alterTable(stmtStr, false); + checkShowPartitionsResultNum("db4.tbl4", true, 3); + checkShowPartitionsResultNum("db4.tbl4", false, 2); + + originPartitionTabletIds2 = Maps.newHashMap(); + getPartitionNameToTabletIdMap("db4.tbl4", false, originPartitionTabletIds2); + Assert.assertEquals(2, originPartitionTabletIds2.size()); + Assert.assertTrue(!originPartitionTabletIds2.containsKey("p1")); + + String recoverStr = "recover partition p1 from db4.tbl4;"; + RecoverPartitionStmt recoverStmt = (RecoverPartitionStmt) UtFrameUtils.parseAndAnalyzeStmt(recoverStr, ctx); + Catalog.getCurrentCatalog().recoverPartition(recoverStmt); + checkShowPartitionsResultNum("db4.tbl4", true, 3); + checkShowPartitionsResultNum("db4.tbl4", false, 3); + + originPartitionTabletIds2 = Maps.newHashMap(); + getPartitionNameToTabletIdMap("db4.tbl4", false, originPartitionTabletIds2); + Assert.assertEquals(originPartitionTabletIds2, originPartitionTabletIds); + + tempPartitionTabletIds2 = Maps.newHashMap(); + getPartitionNameToTabletIdMap("db4.tbl4", true, tempPartitionTabletIds2); + Assert.assertEquals(3, tempPartitionTabletIds2.keySet().size()); + + // Here, we should have 3 partitions p1,p2,p3, and 3 temp partitions tp1,tp2,tp3 + System.out.println("we have partition tablets: " + originPartitionTabletIds2); + System.out.println("we have temp partition tablets: " + tempPartitionTabletIds2); + + stmtStr = "alter table db4.tbl4 replace partition(p1, p2) with temporary partition(tp2, tp3);"; + alterTable(stmtStr, true); + + stmtStr = "alter table db4.tbl4 replace partition(p1, p2) with temporary partition(tp1, tp2) properties('invalid' = 'invalid');"; + alterTable(stmtStr, true); + + stmtStr = "alter table db4.tbl4 replace partition(p1, p2) with temporary partition(tp2, tp3);"; + alterTable(stmtStr, true); + + stmtStr = "alter table db4.tbl4 replace partition(p1, p2) with temporary partition(tp1, tp2) properties('use_temp_partition_name' = 'true');"; + alterTable(stmtStr, false); + checkShowPartitionsResultNum("db4.tbl4", true, 1); // tp3 + checkShowPartitionsResultNum("db4.tbl4", false, 3);// tp1, tp2, p3 + + checkTabletExists(tempPartitionTabletIds2.values(), true); + checkTabletExists(Lists.newArrayList(originPartitionTabletIds2.get("p3")), true); + checkTabletExists(Lists.newArrayList(originPartitionTabletIds2.get("p1"), originPartitionTabletIds2.get("p2")), false); + + String truncateStr = "truncate table db4.tbl4 partition (p3);"; + TruncateTableStmt truncateTableStmt = (TruncateTableStmt)UtFrameUtils.parseAndAnalyzeStmt(truncateStr, ctx); + Catalog.getCurrentCatalog().truncateTable(truncateTableStmt); + checkShowPartitionsResultNum("db4.tbl4", true, 1); + checkShowPartitionsResultNum("db4.tbl4", false, 3); + checkPartitionExist(tbl4, "tp1", false, true); + checkPartitionExist(tbl4, "tp2", false, true); + checkPartitionExist(tbl4, "p3", false, true); + checkPartitionExist(tbl4, "tp3", true, true); + + stmtStr = "alter table db4.tbl4 drop partition p3;"; + alterTable(stmtStr, false); + stmtStr = "alter table db4.tbl4 add partition p31 values in ('7', '8');"; + alterTable(stmtStr, false); + stmtStr = "alter table db4.tbl4 add partition p32 values in ('9');"; + alterTable(stmtStr, false); + + // for now, we have 4 partitions: tp1, tp2, p31, p32, 1 temp partition: tp3 + checkShowPartitionsResultNum("db4.tbl4", false, 4); + checkShowPartitionsResultNum("db4.tbl4", true, 1); + + stmtStr = "alter table db4.tbl4 replace partition(p31) with temporary partition(tp3);"; + alterTable(stmtStr, true); + stmtStr = "alter table db4.tbl4 replace partition(p31, p32) with temporary partition(tp3);"; + alterTable(stmtStr, false); + checkShowPartitionsResultNum("db4.tbl4", false, 3); + checkShowPartitionsResultNum("db4.tbl4", true, 0); + checkPartitionExist(tbl4, "tp1", false, true); + checkPartitionExist(tbl4, "tp2", false, true); + checkPartitionExist(tbl4, "tp3", false, true); + + stmtStr = "alter table db4.tbl4 add temporary partition p1 values in ('1', '2', '3');"; + alterTable(stmtStr, false); + stmtStr = "alter table db4.tbl4 add temporary partition p2 values in ('4', '5', '6');"; + alterTable(stmtStr, false); + stmtStr = "alter table db4.tbl4 add temporary partition p3 values in ('7', '8', '9');"; + alterTable(stmtStr, false); + stmtStr = "alter table db4.tbl4 replace partition(tp1, tp2) with temporary partition(p1, p2);"; + alterTable(stmtStr, false); + checkPartitionExist(tbl4, "tp1", false, true); + checkPartitionExist(tbl4, "tp2", false, true); + checkPartitionExist(tbl4, "tp3", false, true); + checkPartitionExist(tbl4, "p1", true, false); + checkPartitionExist(tbl4, "p2", true, false); + checkPartitionExist(tbl4, "p3", true, true); + + stmtStr = "alter table db4.tbl4 replace partition(tp3) with temporary partition(p3) properties('use_temp_partition_name' = 'true');"; + alterTable(stmtStr, false); + checkPartitionExist(tbl4, "tp1", false, true); + checkPartitionExist(tbl4, "tp2", false, true); + checkPartitionExist(tbl4, "p3", false, true); + checkPartitionExist(tbl4, "p1", true, false); + checkPartitionExist(tbl4, "p2", true, false); + checkPartitionExist(tbl4, "p3", true, false); + checkShowPartitionsResultNum("db4.tbl4", false, 3); + checkShowPartitionsResultNum("db4.tbl4", true, 0); + + stmtStr = "alter table db4.tbl4 add temporary partition tp1 values in ('1', '2', '3');"; // name conflict + alterTable(stmtStr, true); + stmtStr = "alter table db4.tbl4 rename partition p3 tp3;"; + alterTable(stmtStr, false); + stmtStr = "alter table db4.tbl4 add temporary partition p1 values in ('1', '2', '3');"; + alterTable(stmtStr, false); + + originPartitionTabletIds2 = Maps.newHashMap(); + getPartitionNameToTabletIdMap("db4.tbl4", false, originPartitionTabletIds2); + Assert.assertEquals(3, originPartitionTabletIds2.size()); + + tempPartitionTabletIds2 = Maps.newHashMap(); + getPartitionNameToTabletIdMap("db4.tbl4", true, tempPartitionTabletIds2); + Assert.assertEquals(1, tempPartitionTabletIds2.keySet().size()); + + // for now , we have 3 partitions: tp1, tp2, tp3, 1 temp partition: p1 + System.out.println("we have partition tablets: " + originPartitionTabletIds2); + System.out.println("we have temp partition tablets: " + tempPartitionTabletIds2); + + stmtStr = "alter table db4.tbl4 add rollup r1(k1);"; + alterTable(stmtStr, true); + + // truncate table will delete temporary partitions + truncateStr = "truncate table db4.tbl4"; + truncateTableStmt = (TruncateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(truncateStr, ctx); + Catalog.getCurrentCatalog().truncateTable(truncateTableStmt); + checkShowPartitionsResultNum("db4.tbl4", false, 3); + checkShowPartitionsResultNum("db4.tbl4", true, 0); + + stmtStr = "alter table db4.tbl4 add rollup r1(k1);"; + alterTable(stmtStr, false); + + stmtStr = "alter table db4.tbl4 add temporary partition p2 values in ('1', '2', '3', '4', '5', '6');"; + alterTable(stmtStr, true); + + // wait rollup finish + Map alterJobs = Catalog.getCurrentCatalog().getRollupHandler().getAlterJobsV2(); + for (AlterJobV2 alterJobV2 : alterJobs.values()) { + while (!alterJobV2.getJobState().isFinalState()) { + System.out.println( + "alter job " + alterJobV2.getDbId() + " is running. state: " + alterJobV2.getJobState()); + Thread.sleep(5000); + } + System.out.println("alter job " + alterJobV2.getDbId() + " is done. state: " + alterJobV2.getJobState()); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, alterJobV2.getJobState()); + } + // waiting table state to normal + Thread.sleep(500); + + stmtStr = "alter table db4.tbl4 add temporary partition p2 values in ('1', '2', '3', '4', '5', '6');"; + alterTable(stmtStr, false); + + TempPartitions tempPartitions = Deencapsulation.getField(tbl4, "tempPartitions"); + testSerializeTempPartitions(tempPartitions); + + stmtStr = "alter table db4.tbl4 replace partition (tp1, tp2) with temporary partition (p2);"; + alterTable(stmtStr, false); + checkShowPartitionsResultNum("db4.tbl4", false, 2); + checkShowPartitionsResultNum("db4.tbl4", true, 0); + checkPartitionExist(tbl4, "p2", false, true); + checkPartitionExist(tbl4, "tp3", false, true); + checkPartitionExist(tbl4, "tp1", false, false); + checkPartitionExist(tbl4, "tp2", false, false); + checkPartitionExist(tbl4, "p2", true, false); + + checkTablet("db4.tbl4", "p2", false, 2); + checkTablet("db4.tbl4", "tp3", false, 2); + + // for now, we have 2 partitions: p2, tp3, ('1', '2', '3', '4', '5', '6'), ('7', '8', '9'). 0 temp partition. + stmtStr = "alter table db4.tbl4 add temporary partition tp4 values in ('1', '2', '3', '4', '5', '6') ('in_memory' = 'true') distributed by hash(k1) buckets 3"; + alterTable(stmtStr, true); + stmtStr = "alter table db4.tbl4 add temporary partition tp4 values in ('1', '2', '3', '4', '5', '6') ('in_memory' = 'true', 'replication_num' = '2') distributed by hash(k2) buckets 3"; + alterTable(stmtStr, true); + stmtStr = "alter table db4.tbl4 add temporary partition tp4 values in ('1', '2', '3', '4', '5', '6') ('in_memory' = 'true', 'replication_num' = '1') distributed by hash(k2) buckets 3"; + alterTable(stmtStr, false); + + Partition p2 = tbl4.getPartition("p2"); + Assert.assertNotNull(p2); + Assert.assertFalse(tbl4.getPartitionInfo().getIsInMemory(p2.getId())); + Assert.assertEquals(1, p2.getDistributionInfo().getBucketNum()); + + stmtStr = "alter table db4.tbl4 replace partition (p2) with temporary partition (tp4)"; + alterTable(stmtStr, false); + + // for now, we have 2 partitions: p2, tp3, ('1', '2', '3', '4', '5', '6'), ('7', '8', '9'). 0 temp partition. and p2 bucket is 3, 'in_memory' is true. + p2 = tbl4.getPartition("p2"); + Assert.assertNotNull(p2); + Assert.assertTrue(tbl4.getPartitionInfo().getIsInMemory(p2.getId())); + Assert.assertEquals(3, p2.getDistributionInfo().getBucketNum()); + + stmtStr = "alter table db4.tbl4 add temporary partition tp1 values in ('1', '2', '3');"; + alterTable(stmtStr, false); + stmtStr = "alter table db4.tbl4 add temporary partition tp2 values in ('4', '5', '6');"; + alterTable(stmtStr, false); + + checkShowPartitionsResultNum("db4.tbl4", false, 2); + checkShowPartitionsResultNum("db4.tbl4", true, 2); + checkPartitionExist(tbl4, "p2", false, true); + checkPartitionExist(tbl4, "tp3", false, true); + checkPartitionExist(tbl4, "tp1", true, true); + checkPartitionExist(tbl4, "tp2", true, true); + + stmtStr = "alter table db4.tbl4 replace partition (p2) with temporary partition (tp1, tp2)"; + alterTable(stmtStr, false); + + checkShowPartitionsResultNum("db4.tbl4", false, 3); + checkShowPartitionsResultNum("db4.tbl4", true, 0); + checkPartitionExist(tbl4, "tp1", false, true); + checkPartitionExist(tbl4, "tp2", false, true); + checkPartitionExist(tbl4, "tp3", false, true); + checkPartitionExist(tbl4, "tp1", true, false); + checkPartitionExist(tbl4, "tp2", true, false); + + // for now, tbl4 has 3 formal partition: + // tp1 (1, 2, 3) + // tp2 (4, 5, 6) + // tp3 (7, 8, 9) + // Test strict range + stmtStr = "alter table db4.tbl4 add temporary partition p2 values in ('4', '5')"; + alterTable(stmtStr, false); + stmtStr = "alter table db4.tbl4 add temporary partition p31 values in ('7', '8')"; + alterTable(stmtStr, false); + stmtStr = "alter table db4.tbl4 add temporary partition p32 values in ('9')"; + alterTable(stmtStr, false); + + stmtStr = "alter table db4.tbl4 replace partition (tp2) with temporary partition (p2) properties('strict_range' = 'true');"; + alterTable(stmtStr, true); + stmtStr = "alter table db4.tbl4 replace partition (tp2) with temporary partition (p2) properties('strict_range' = 'false', 'use_temp_partition_name' = 'true');"; + alterTable(stmtStr, false); + stmtStr = "alter table db4.tbl4 replace partition (tp3) with temporary partition (p31, p32) properties('strict_range' = 'true', 'use_temp_partition_name' = 'true');"; + alterTable(stmtStr, false); + + stmtStr = "alter table db4.tbl4 add temporary partition p4 values in ('1', '2', '3', '4')"; + alterTable(stmtStr, false); + + stmtStr = "alter table db4.tbl4 replace partition (tp1) with temporary partition (p4) properties('strict_range' = 'false');"; + alterTable(stmtStr, true); + } + @Test + public void testForMultiListPartitionTable() throws Exception { + MetaContext metaContext = new MetaContext(); + metaContext.setMetaVersion(FeMetaVersion.VERSION_CURRENT); + metaContext.setThreadLocalInfo(); + + // create database db5 + String createDbStmtStr = "create database db5;"; + CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, ctx); + Catalog.getCurrentCatalog().createDb(createDbStmt); + System.out.println(Catalog.getCurrentCatalog().getDbNames()); + + // create table tbl5 + String createTblStmtStr1 = "create table db5.tbl5 (k1 int, k2 varchar)\n" + + "partition by list(k1, k2)\n" + + "(\n" + + "partition p1 values in ((\"1\",\"beijing\"), (\"1\", \"shanghai\")),\n" + + "partition p2 values in ((\"2\",\"beijing\"), (\"2\", \"shanghai\")),\n" + + "partition p3 values in ((\"3\",\"beijing\"), (\"3\", \"shanghai\"))\n" + + ")\n" + + "distributed by hash(k2) buckets 1\n" + + "properties('replication_num' = '1');"; + CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(createTblStmtStr1, ctx); + Catalog.getCurrentCatalog().createTable(createTableStmt); + + Database db5 = Catalog.getCurrentCatalog().getDb("default_cluster:db5"); + OlapTable tbl5 = (OlapTable) db5.getTable("tbl5"); + + testSerializeOlapTable(tbl5); + + Map originPartitionTabletIds = Maps.newHashMap(); + getPartitionNameToTabletIdMap("db5.tbl5", false, originPartitionTabletIds); + Assert.assertEquals(3, originPartitionTabletIds.keySet().size()); + + // show temp partition + checkShowPartitionsResultNum("db5.tbl5", true, 0); + checkShowPartitionsResultNum("db5.tbl5", false, 3); + + // add temp partition with duplicate name + String stmtStr = "alter table db5.tbl5 add temporary partition p1 values in ((\"1\",\"beijing\"), (\"1\", \"shanghai\"));"; + alterTable(stmtStr, true); + + // add temp partition + stmtStr = "alter table db5.tbl5 add temporary partition tp1 values in ((\"1\",\"beijing\"), (\"1\", \"shanghai\"));"; + alterTable(stmtStr, false); + + stmtStr = "alter table db5.tbl5 add temporary partition tp2 values in ((\"1\",\"beijing\"), (\"1\", \"shanghai\");"; + alterTable(stmtStr, true); + + stmtStr = "alter table db5.tbl5 add temporary partition tp1 values in ((\"2\",\"beijing\"), (\"2\", \"shanghai\"));"; + alterTable(stmtStr, true); + + stmtStr = "alter table db5.tbl5 add temporary partition tp2 values in ((\"2\",\"beijing\"), (\"2\", \"shanghai\"));"; + alterTable(stmtStr, false); + + stmtStr = "alter table db5.tbl5 add temporary partition tp3 values in ((\"2\",\"beijing\"), (\"3\",\"beijing\"), (\"3\", \"shanghai\"));"; + alterTable(stmtStr, true); + + stmtStr = "alter table db5.tbl5 add temporary partition tp3 values in ((\"3\",\"beijing\"), (\"3\", \"shanghai\"));"; + alterTable(stmtStr, false); + + Map tempPartitionTabletIds = Maps.newHashMap(); + getPartitionNameToTabletIdMap("db5.tbl5", true, tempPartitionTabletIds); + Assert.assertEquals(3, tempPartitionTabletIds.keySet().size()); + + System.out.println("partition tablets: " + originPartitionTabletIds); + System.out.println("temp partition tablets: " + tempPartitionTabletIds); + + testSerializeOlapTable(tbl5); + + // drop non exist temp partition + stmtStr = "alter table db5.tbl5 drop temporary partition tp4;"; + alterTable(stmtStr, true); + + stmtStr = "alter table db5.tbl5 drop temporary partition if exists tp4;"; + alterTable(stmtStr, false); + + stmtStr = "alter table db5.tbl5 drop temporary partition tp3;"; + alterTable(stmtStr, false); + + Map originPartitionTabletIds2 = Maps.newHashMap(); + getPartitionNameToTabletIdMap("db5.tbl5", false, originPartitionTabletIds2); + Assert.assertEquals(originPartitionTabletIds2, originPartitionTabletIds); + + Map tempPartitionTabletIds2 = Maps.newHashMap(); + getPartitionNameToTabletIdMap("db5.tbl5", true, tempPartitionTabletIds2); + Assert.assertEquals(2, tempPartitionTabletIds2.keySet().size()); + Assert.assertTrue(!tempPartitionTabletIds2.containsKey("tp3")); + + checkShowPartitionsResultNum("db5.tbl5", true, 2); + checkShowPartitionsResultNum("db5.tbl5", false, 3); + + stmtStr = "alter table db5.tbl5 add temporary partition tp3 values in ((\"3\",\"beijing\"), (\"3\", \"shanghai\"));"; + alterTable(stmtStr, false); + checkShowPartitionsResultNum("db5.tbl5", true, 3); + + stmtStr = "alter table db5.tbl5 drop partition p1;"; + alterTable(stmtStr, false); + checkShowPartitionsResultNum("db5.tbl5", true, 3); + checkShowPartitionsResultNum("db5.tbl5", false, 2); + + originPartitionTabletIds2 = Maps.newHashMap(); + getPartitionNameToTabletIdMap("db5.tbl5", false, originPartitionTabletIds2); + Assert.assertEquals(2, originPartitionTabletIds2.size()); + Assert.assertTrue(!originPartitionTabletIds2.containsKey("p1")); + + String recoverStr = "recover partition p1 from db5.tbl5;"; + RecoverPartitionStmt recoverStmt = (RecoverPartitionStmt) UtFrameUtils.parseAndAnalyzeStmt(recoverStr, ctx); + Catalog.getCurrentCatalog().recoverPartition(recoverStmt); + checkShowPartitionsResultNum("db5.tbl5", true, 3); + checkShowPartitionsResultNum("db5.tbl5", false, 3); + + originPartitionTabletIds2 = Maps.newHashMap(); + getPartitionNameToTabletIdMap("db5.tbl5", false, originPartitionTabletIds2); + Assert.assertEquals(originPartitionTabletIds2, originPartitionTabletIds); + + tempPartitionTabletIds2 = Maps.newHashMap(); + getPartitionNameToTabletIdMap("db5.tbl5", true, tempPartitionTabletIds2); + Assert.assertEquals(3, tempPartitionTabletIds2.keySet().size()); + + // Here, we should have 3 partitions p1,p2,p3, and 3 temp partitions tp1,tp2,tp3 + System.out.println("we have partition tablets: " + originPartitionTabletIds2); + System.out.println("we have temp partition tablets: " + tempPartitionTabletIds2); + + stmtStr = "alter table db5.tbl5 replace partition(p1, p2) with temporary partition(tp2, tp3);"; + alterTable(stmtStr, true); + + stmtStr = "alter table db5.tbl5 replace partition(p1, p2) with temporary partition(tp1, tp2) properties('invalid' = 'invalid');"; + alterTable(stmtStr, true); + + stmtStr = "alter table db5.tbl5 replace partition(p1, p2) with temporary partition(tp2, tp3);"; + alterTable(stmtStr, true); + + stmtStr = "alter table db5.tbl5 replace partition(p1, p2) with temporary partition(tp1, tp2) properties('use_temp_partition_name' = 'true');"; + alterTable(stmtStr, false); + checkShowPartitionsResultNum("db5.tbl5", true, 1); // tp3 + checkShowPartitionsResultNum("db5.tbl5", false, 3);// tp1, tp2, p3 + + checkTabletExists(tempPartitionTabletIds2.values(), true); + checkTabletExists(Lists.newArrayList(originPartitionTabletIds2.get("p3")), true); + checkTabletExists(Lists.newArrayList(originPartitionTabletIds2.get("p1"), originPartitionTabletIds2.get("p2")), false); + + String truncateStr = "truncate table db5.tbl5 partition (p3);"; + TruncateTableStmt truncateTableStmt = (TruncateTableStmt)UtFrameUtils.parseAndAnalyzeStmt(truncateStr, ctx); + Catalog.getCurrentCatalog().truncateTable(truncateTableStmt); + checkShowPartitionsResultNum("db5.tbl5", true, 1); + checkShowPartitionsResultNum("db5.tbl5", false, 3); + checkPartitionExist(tbl5, "tp1", false, true); + checkPartitionExist(tbl5, "tp2", false, true); + checkPartitionExist(tbl5, "p3", false, true); + checkPartitionExist(tbl5, "tp3", true, true); + + stmtStr = "alter table db5.tbl5 drop partition p3;"; + alterTable(stmtStr, false); + stmtStr = "alter table db5.tbl5 add partition p31 values in ((\"3\",\"beijing\"));"; + alterTable(stmtStr, false); + stmtStr = "alter table db5.tbl5 add partition p32 values in ((\"3\", \"shanghai\"));"; + alterTable(stmtStr, false); + + // for now, we have 4 partitions: tp1, tp2, p31, p32, 1 temp partition: tp3 + checkShowPartitionsResultNum("db5.tbl5", false, 4); + checkShowPartitionsResultNum("db5.tbl5", true, 1); + + stmtStr = "alter table db5.tbl5 replace partition(p31) with temporary partition(tp3);"; + alterTable(stmtStr, true); + stmtStr = "alter table db5.tbl5 replace partition(p31, p32) with temporary partition(tp3);"; + alterTable(stmtStr, false); + checkShowPartitionsResultNum("db5.tbl5", false, 3); + checkShowPartitionsResultNum("db5.tbl5", true, 0); + checkPartitionExist(tbl5, "tp1", false, true); + checkPartitionExist(tbl5, "tp2", false, true); + checkPartitionExist(tbl5, "tp3", false, true); + + stmtStr = "alter table db5.tbl5 add temporary partition p1 values in ((\"1\",\"beijing\"), (\"1\", \"shanghai\"));"; + alterTable(stmtStr, false); + stmtStr = "alter table db5.tbl5 add temporary partition p2 values in ((\"2\",\"beijing\"), (\"2\", \"shanghai\"));"; + alterTable(stmtStr, false); + stmtStr = "alter table db5.tbl5 add temporary partition p3 values in ((\"3\",\"beijing\"), (\"3\", \"shanghai\"));"; + alterTable(stmtStr, false); + stmtStr = "alter table db5.tbl5 replace partition(tp1, tp2) with temporary partition(p1, p2);"; + alterTable(stmtStr, false); + checkPartitionExist(tbl5, "tp1", false, true); + checkPartitionExist(tbl5, "tp2", false, true); + checkPartitionExist(tbl5, "tp3", false, true); + checkPartitionExist(tbl5, "p1", true, false); + checkPartitionExist(tbl5, "p2", true, false); + checkPartitionExist(tbl5, "p3", true, true); + + stmtStr = "alter table db5.tbl5 replace partition(tp3) with temporary partition(p3) properties('use_temp_partition_name' = 'true');"; + alterTable(stmtStr, false); + checkPartitionExist(tbl5, "tp1", false, true); + checkPartitionExist(tbl5, "tp2", false, true); + checkPartitionExist(tbl5, "p3", false, true); + checkPartitionExist(tbl5, "p1", true, false); + checkPartitionExist(tbl5, "p2", true, false); + checkPartitionExist(tbl5, "p3", true, false); + checkShowPartitionsResultNum("db5.tbl5", false, 3); + checkShowPartitionsResultNum("db5.tbl5", true, 0); + + stmtStr = "alter table db5.tbl5 add temporary partition tp1 values in ((\"1\",\"beijing\"), (\"1\", \"shanghai\"));"; // name conflict + alterTable(stmtStr, true); + stmtStr = "alter table db5.tbl5 rename partition p3 tp3;"; + alterTable(stmtStr, false); + stmtStr = "alter table db5.tbl5 add temporary partition p1 values in ((\"1\",\"beijing\"), (\"1\", \"shanghai\"));"; + alterTable(stmtStr, false); + + originPartitionTabletIds2 = Maps.newHashMap(); + getPartitionNameToTabletIdMap("db5.tbl5", false, originPartitionTabletIds2); + Assert.assertEquals(3, originPartitionTabletIds2.size()); + + tempPartitionTabletIds2 = Maps.newHashMap(); + getPartitionNameToTabletIdMap("db5.tbl5", true, tempPartitionTabletIds2); + Assert.assertEquals(1, tempPartitionTabletIds2.keySet().size()); + + // for now , we have 3 partitions: tp1, tp2, tp3, 1 temp partition: p1 + System.out.println("we have partition tablets: " + originPartitionTabletIds2); + System.out.println("we have temp partition tablets: " + tempPartitionTabletIds2); + + stmtStr = "alter table db5.tbl5 add rollup r1(k1);"; + alterTable(stmtStr, true); + + // truncate table will delete temporary partitions + truncateStr = "truncate table db5.tbl5"; + truncateTableStmt = (TruncateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(truncateStr, ctx); + Catalog.getCurrentCatalog().truncateTable(truncateTableStmt); + checkShowPartitionsResultNum("db5.tbl5", false, 3); + checkShowPartitionsResultNum("db5.tbl5", true, 0); + + stmtStr = "alter table db5.tbl5 add rollup r1(k1);"; + alterTable(stmtStr, false); + + stmtStr = "alter table db5.tbl5 add temporary partition p2 values in ((\"1\",\"beijing\"), (\"1\", \"shanghai\"), (\"2\",\"beijing\"), (\"2\", \"shanghai\"));"; + alterTable(stmtStr, true); + + // wait rollup finish + Map alterJobs = Catalog.getCurrentCatalog().getRollupHandler().getAlterJobsV2(); + for (AlterJobV2 alterJobV2 : alterJobs.values()) { + while (!alterJobV2.getJobState().isFinalState()) { + System.out.println( + "alter job " + alterJobV2.getDbId() + " is running. state: " + alterJobV2.getJobState()); + Thread.sleep(5000); + } + System.out.println("alter job " + alterJobV2.getDbId() + " is done. state: " + alterJobV2.getJobState()); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, alterJobV2.getJobState()); + } + // waiting table state to normal + Thread.sleep(500); + + stmtStr = "alter table db5.tbl5 add temporary partition p2 values in ((\"1\",\"beijing\"), (\"1\", \"shanghai\"), (\"2\",\"beijing\"), (\"2\", \"shanghai\"));"; + alterTable(stmtStr, false); + + TempPartitions tempPartitions = Deencapsulation.getField(tbl5, "tempPartitions"); + testSerializeTempPartitions(tempPartitions); + + stmtStr = "alter table db5.tbl5 replace partition (tp1, tp2) with temporary partition (p2);"; + alterTable(stmtStr, false); + checkShowPartitionsResultNum("db5.tbl5", false, 2); + checkShowPartitionsResultNum("db5.tbl5", true, 0); + checkPartitionExist(tbl5, "p2", false, true); + checkPartitionExist(tbl5, "tp3", false, true); + checkPartitionExist(tbl5, "tp1", false, false); + checkPartitionExist(tbl5, "tp2", false, false); + checkPartitionExist(tbl5, "p2", true, false); + + checkTablet("db5.tbl5", "p2", false, 2); + checkTablet("db5.tbl5", "tp3", false, 2); + + // for now, we have 2 partitions: p2, tp3, (("1","beijing"), ("1", "shanghai"), ("2","beijing"), ("2", "shanghai")), ('7', '8', '9'). 0 temp partition. + stmtStr = "alter table db5.tbl5 add temporary partition tp4 values in ((\"1\",\"beijing\"), (\"1\", \"shanghai\"), (\"2\",\"beijing\"), (\"2\", \"shanghai\")) ('in_memory' = 'true') distributed by hash(k1) buckets 3"; + alterTable(stmtStr, true); + stmtStr = "alter table db5.tbl5 add temporary partition tp4 values in ((\"1\",\"beijing\"), (\"1\", \"shanghai\"), (\"2\",\"beijing\"), (\"2\", \"shanghai\")) ('in_memory' = 'true', 'replication_num' = '2') distributed by hash(k2) buckets 3"; + alterTable(stmtStr, true); + stmtStr = "alter table db5.tbl5 add temporary partition tp4 values in ((\"1\",\"beijing\"), (\"1\", \"shanghai\"), (\"2\",\"beijing\"), (\"2\", \"shanghai\")) ('in_memory' = 'true', 'replication_num' = '1') distributed by hash(k2) buckets 3"; + alterTable(stmtStr, false); + + Partition p2 = tbl5.getPartition("p2"); + Assert.assertNotNull(p2); + Assert.assertFalse(tbl5.getPartitionInfo().getIsInMemory(p2.getId())); + Assert.assertEquals(1, p2.getDistributionInfo().getBucketNum()); + + stmtStr = "alter table db5.tbl5 replace partition (p2) with temporary partition (tp4)"; + alterTable(stmtStr, false); + + // for now, we have 2 partitions: p2, tp3, (("1","beijing"), ("1", "shanghai"), ("2","beijing"), ("2", "shanghai")), ('7', '8', '9'). 0 temp partition. and p2 bucket is 3, 'in_memory' is true. + p2 = tbl5.getPartition("p2"); + Assert.assertNotNull(p2); + Assert.assertTrue(tbl5.getPartitionInfo().getIsInMemory(p2.getId())); + Assert.assertEquals(3, p2.getDistributionInfo().getBucketNum()); + + stmtStr = "alter table db5.tbl5 add temporary partition tp1 values in ((\"1\",\"beijing\"), (\"1\", \"shanghai\"));"; + alterTable(stmtStr, false); + stmtStr = "alter table db5.tbl5 add temporary partition tp2 values in ((\"2\",\"beijing\"), (\"2\", \"shanghai\"));"; + alterTable(stmtStr, false); + + checkShowPartitionsResultNum("db5.tbl5", false, 2); + checkShowPartitionsResultNum("db5.tbl5", true, 2); + checkPartitionExist(tbl5, "p2", false, true); + checkPartitionExist(tbl5, "tp3", false, true); + checkPartitionExist(tbl5, "tp1", true, true); + checkPartitionExist(tbl5, "tp2", true, true); + + stmtStr = "alter table db5.tbl5 replace partition (p2) with temporary partition (tp1, tp2)"; + alterTable(stmtStr, false); + + checkShowPartitionsResultNum("db5.tbl5", false, 3); + checkShowPartitionsResultNum("db5.tbl5", true, 0); + checkPartitionExist(tbl5, "tp1", false, true); + checkPartitionExist(tbl5, "tp2", false, true); + checkPartitionExist(tbl5, "tp3", false, true); + checkPartitionExist(tbl5, "tp1", true, false); + checkPartitionExist(tbl5, "tp2", true, false); + + } private void testSerializeOlapTable(OlapTable tbl) throws IOException, AnalysisException { // 1. Write objects to file diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/ListUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/ListUtilTest.java index 75858189efb722..342fc78d9fbf84 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/ListUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/ListUtilTest.java @@ -18,15 +18,52 @@ package org.apache.doris.common.util; import com.google.common.collect.Lists; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; public class ListUtilTest { + private static PartitionKey pk1; + private static PartitionKey pk2; + private static PartitionKey pk3; + private static List listA = new ArrayList<>(); + private static List listB = new ArrayList<>(); + private static List listC = new ArrayList<>(); + + @BeforeClass + public static void setUp() throws AnalysisException { + Column charString = new Column("char", PrimitiveType.CHAR); + Column varchar = new Column("varchar", PrimitiveType.VARCHAR); + + pk1 = PartitionKey.createPartitionKey(Arrays.asList(new PartitionValue("beijing")), Arrays.asList(charString)); + pk2 = PartitionKey.createPartitionKey(Arrays.asList(new PartitionValue("shanghai")), Arrays.asList(varchar)); + pk3 = PartitionKey.createPartitionKey(Arrays.asList(new PartitionValue("tianjin")), Arrays.asList(varchar)); + + listA.add(pk1); + + listB.add(pk1); + listB.add(pk2); + + listC.add(pk1); + listC.add(pk2); + listC.add(pk3); + } + @Rule public ExpectedException expectedEx = ExpectedException.none(); @@ -98,4 +135,31 @@ public void testSplitBySizeWithNegativeSize() { ListUtil.splitBySize(lists, expectSize); } + + @Test + public void testListsMatchNormal() throws DdlException { + + List list1 = Arrays.asList(new ListPartitionItem(listA), new ListPartitionItem(listB)); + List list2 = Arrays.asList(new ListPartitionItem(listA), new ListPartitionItem(listB)); + + ListUtil.checkPartitionKeyListsMatch(list1, list2); + + } + + @Test(expected = DdlException.class) + public void testListsMatchSameSize() throws DdlException { + List list1 = Arrays.asList(new ListPartitionItem(listA), new ListPartitionItem(listB)); + List list2 = Arrays.asList(new ListPartitionItem(listA), new ListPartitionItem(listC)); + + ListUtil.checkPartitionKeyListsMatch(list1, list2); + } + + @Test(expected = DdlException.class) + public void testListMatchDiffSize() throws DdlException { + List list1 = Arrays.asList(new ListPartitionItem(listA), new ListPartitionItem(listB)); + List list2 = Arrays.asList(new ListPartitionItem(listA), new ListPartitionItem(listB), + new ListPartitionItem(listC)); + + ListUtil.checkPartitionKeyListsMatch(list1, list2); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/SparkLoadPendingTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/SparkLoadPendingTaskTest.java index a487c181ce036d..491773b7764d8a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/SparkLoadPendingTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/SparkLoadPendingTaskTest.java @@ -17,17 +17,11 @@ package org.apache.doris.load.loadv2; -import mockit.Expectations; -import mockit.Injectable; -import mockit.Mock; -import mockit.MockUp; -import mockit.Mocked; - import org.apache.doris.analysis.BrokerDesc; import org.apache.doris.analysis.DataDescription; import org.apache.doris.analysis.PartitionKeyDesc; import org.apache.doris.analysis.PartitionValue; -import org.apache.doris.analysis.SingleRangePartitionDesc; +import org.apache.doris.analysis.SinglePartitionDesc; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; @@ -55,6 +49,7 @@ import org.apache.doris.load.loadv2.etl.EtlJobConfig.EtlPartition; import org.apache.doris.load.loadv2.etl.EtlJobConfig.EtlPartitionInfo; import org.apache.doris.load.loadv2.etl.EtlJobConfig.EtlTable; + import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -64,6 +59,12 @@ import java.util.List; import java.util.Map; +import mockit.Expectations; +import mockit.Injectable; +import mockit.Mock; +import mockit.MockUp; +import mockit.Mocked; + public class SparkLoadPendingTaskTest { @Test @@ -231,12 +232,12 @@ public void testRangePartitionHashDistribution(@Injectable SparkLoadJob sparkLoa int partitionColumnIndex = 0; List partitions = Lists.newArrayList(partition1, partition2); RangePartitionInfo partitionInfo = new RangePartitionInfo(Lists.newArrayList(columns.get(partitionColumnIndex))); - PartitionKeyDesc partitionKeyDesc1 = new PartitionKeyDesc(Lists.newArrayList(new PartitionValue("10"))); - SingleRangePartitionDesc partitionDesc1 = new SingleRangePartitionDesc(false, "p1", partitionKeyDesc1, null); + PartitionKeyDesc partitionKeyDesc1 = PartitionKeyDesc.createLessThan(Lists.newArrayList(new PartitionValue("10"))); + SinglePartitionDesc partitionDesc1 = new SinglePartitionDesc(false, "p1", partitionKeyDesc1, null); partitionDesc1.analyze(1, null); partitionInfo.handleNewSinglePartitionDesc(partitionDesc1, partition1Id, false); - PartitionKeyDesc partitionKeyDesc2 = new PartitionKeyDesc(Lists.newArrayList(new PartitionValue("20"))); - SingleRangePartitionDesc partitionDesc2 = new SingleRangePartitionDesc(false, "p2", partitionKeyDesc2, null); + PartitionKeyDesc partitionKeyDesc2 = PartitionKeyDesc.createLessThan(Lists.newArrayList(new PartitionValue("20"))); + SinglePartitionDesc partitionDesc2 = new SinglePartitionDesc(false, "p2", partitionKeyDesc2, null); partitionDesc2.analyze(1, null); partitionInfo.handleNewSinglePartitionDesc(partitionDesc2, partition2Id, false); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapTableSinkTest.java index 57964ac814ee75..0e4e764c8ba0a8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapTableSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapTableSinkTest.java @@ -23,6 +23,7 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.ListPartitionInfo; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; @@ -157,4 +158,38 @@ public void testRangeUnknownPartition( LOG.info("sink is {}", sink.toThrift()); LOG.info("{}", sink.getExplainString("", TExplainLevel.NORMAL)); } + + @Test + public void testListPartition( + @Injectable ListPartitionInfo partInfo, + @Injectable MaterializedIndex index) throws UserException { + TupleDescriptor tuple = getTuple(); + + HashDistributionInfo distInfo = new HashDistributionInfo( + 2, Lists.newArrayList(new Column("k1", PrimitiveType.BIGINT))); + + Column partKey = new Column("k2", PrimitiveType.VARCHAR); + PartitionKey key = PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("123")), Lists.newArrayList(partKey)); + Partition p1 = new Partition(1, "p1", index, distInfo); + Partition p2 = new Partition(2, "p2", index, distInfo); + + new Expectations() {{ + dstTable.getId(); result = 1; + dstTable.getPartitionInfo(); result = partInfo; + partInfo.getType(); result = PartitionType.LIST; + partInfo.getPartitionColumns(); result = Lists.newArrayList(partKey); + dstTable.getPartitions(); result = Lists.newArrayList(p1, p2); + dstTable.getPartition(p1.getId()); result = p1; + }}; + + OlapTableSink sink = new OlapTableSink(dstTable, tuple, Lists.newArrayList(p1.getId())); + sink.init(new TUniqueId(1, 2), 3, 4, 1000); + try { + sink.complete(); + } catch (UserException e) { + + } + LOG.info("sink is {}", sink.toThrift()); + LOG.info("{}", sink.getExplainString("", TExplainLevel.NORMAL)); + } } diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index ba5de789d90b71..4224e86a4f628a 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -133,6 +133,7 @@ struct TOlapTablePartition { 6: optional list start_keys 7: optional list end_keys + 8: optional list> in_keys } struct TOlapTablePartitionParam { diff --git a/gensrc/thrift/Partitions.thrift b/gensrc/thrift/Partitions.thrift index 9a3305b73b578b..8eecbb417b1184 100644 --- a/gensrc/thrift/Partitions.thrift +++ b/gensrc/thrift/Partitions.thrift @@ -34,6 +34,9 @@ enum TPartitionType { // ordered partition on a list of exprs // (partition bounds don't overlap) RANGE_PARTITIONED, + + // partition on a list of exprs + LIST_PARTITIONED, // unordered partition on a set of exprs // (only use in bucket shuffle join) From 6e8f99eadaa580fd59587e96eb84f72100eb17db Mon Sep 17 00:00:00 2001 From: qijianliang01 Date: Fri, 26 Mar 2021 18:06:39 +0800 Subject: [PATCH 2/6] add duplicate item check in one partition key and optimize partition conflict exception Change-Id: I42dc64541454b72389d1bc2d1f2fe455f918652e --- .../org/apache/doris/catalog/ListPartitionInfo.java | 11 +++++++++-- .../org/apache/doris/catalog/ListPartitionItem.java | 5 +++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java index 87141450c0c5fe..76ef0b271ba578 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java @@ -67,6 +67,10 @@ public PartitionItem createAndCheckPartitionItem(SinglePartitionDesc desc, boole for (List values : partitionKeyDesc.getInValues()) { PartitionKey partitionKey = PartitionKey.createListPartitionKey(values, partitionColumns); checkNewPartitionKey(partitionKey, partitionKeyDesc, isTemp); + if (partitionKeys.contains(partitionKey)) { + throw new AnalysisException("The partition key[" + partitionKeyDesc.toSql() + "] has duplicate item [" + + partitionKey.toSql() + "]."); + } partitionKeys.add(partitionKey); } } catch (AnalysisException e) { @@ -83,8 +87,11 @@ private void checkNewPartitionKey(PartitionKey newKey, PartitionKeyDesc keyDesc, // check new partition key not exists. for (Map.Entry entry : id2Item.entrySet()) { if (((ListPartitionItem)entry.getValue()).getItems().contains(newKey)) { - throw new AnalysisException("The partition key[" + keyDesc.toSql() + "] is overlap with current " + - entry.getValue().toString()); + StringBuilder sb = new StringBuilder(); + sb.append("The partition key[").append(newKey.toSql()).append("] in partition item[") + .append(keyDesc.toSql()).append("] is conflict with current partitionKeys[") + .append(((ListPartitionItem) entry.getValue()).toSql()).append("]"); + throw new AnalysisException(sb.toString()); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java index d74d76306bd9f3..5395d83fdad142 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java @@ -135,8 +135,13 @@ public String toSql() { sb.append("("); } + int i = 0; for (PartitionKey partitionKey : partitionKeys) { sb.append(partitionKey.toSql()); + if (i < partitionKeys.size() - 1) { + sb.append(","); + } + i++; } if (size > 1) { From 4bddbe10453b366ece7f3d4a282e359c033958de Mon Sep 17 00:00:00 2001 From: qijianliang01 Date: Tue, 30 Mar 2021 19:33:32 +0800 Subject: [PATCH 3/6] fix loading failed, when as partition column Change-Id: Id2ef6d3aa895b4bb34e710f2e6ac99a3d74389a6 --- be/src/exec/tablet_info.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp index 7c2d07463aaf91..e0ec90a51fc8ab 100644 --- a/be/src/exec/tablet_info.cpp +++ b/be/src/exec/tablet_info.cpp @@ -344,8 +344,21 @@ Status OlapTablePartitionParam::_create_partition_key(const TExprNode& t_expr, T break; } case TExprNodeType::STRING_LITERAL: { - *reinterpret_cast(slot) = StringValue(const_cast(t_expr.string_literal.value.c_str()), - t_expr.string_literal.value.size()); + int len = t_expr.string_literal.value.size(); + const char* str_val = t_expr.string_literal.value.c_str(); + + // CHAR is a fixed-length string and needs to use the length in the slot definition, + // VARVHAR is a variable-length string and needs to use the length of the string itself + // padding 0 to CHAR field + if (TYPE_CHAR == slot_desc->type().type && len < slot_desc->type().len) { + auto new_ptr = (char*)_mem_pool->allocate(slot_desc->type().len); + memset(new_ptr, 0, slot_desc->type().len); + memcpy(new_ptr, str_val, len); + + str_val = new_ptr; + len = slot_desc->type().len; + } + *reinterpret_cast(slot) = StringValue(const_cast(str_val), len); break; } case TExprNodeType::BOOL_LITERAL: { From d00a7797d6d27aa733cb2fc8ff547ae284f531f9 Mon Sep 17 00:00:00 2001 From: qijianliang01 Date: Wed, 31 Mar 2021 11:16:43 +0800 Subject: [PATCH 4/6] support delete operation with delete_without_partition variable in list partition table. Change-Id: I9ff5fa4bc1bde35f4e10f3baecc14f1489eaa2f2 --- .../src/main/java/org/apache/doris/load/DeleteHandler.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java index 4542ec2be81d96..b583c7b517906d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java @@ -149,9 +149,10 @@ public void process(DeleteStmt stmt) throws DdlException, QueryStateException { } if (noPartitionSpecified) { - if (olapTable.getPartitionInfo().getType() == PartitionType.RANGE) { + if (olapTable.getPartitionInfo().getType() == PartitionType.RANGE || + olapTable.getPartitionInfo().getType() == PartitionType.LIST) { if (!ConnectContext.get().getSessionVariable().isDeleteWithoutPartition()) { - throw new DdlException("This is a range partitioned table." + throw new DdlException("This is a range or list partitioned table." + " You should specify partition in delete stmt, or set delete_without_partition to true"); } else { partitionNames.addAll(olapTable.getPartitionNames()); From eded97bbf3ee93a95bb043f4ab53cde328496dca Mon Sep 17 00:00:00 2001 From: qijianliang Date: Wed, 14 Apr 2021 15:59:43 +0800 Subject: [PATCH 5/6] fix iterator and docs --- be/src/exec/tablet_info.cpp | 2 +- docs/en/getting-started/data-partition.md | 26 +++++++++---------- .../load-data/delete-manual.md | 8 +++--- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp index e0ec90a51fc8ab..f73e308d52001f 100644 --- a/be/src/exec/tablet_info.cpp +++ b/be/src/exec/tablet_info.cpp @@ -269,7 +269,7 @@ Status OlapTablePartitionParam::init() { bool OlapTablePartitionParam::find_tablet(Tuple* tuple, const OlapTablePartition** partition, uint32_t* dist_hashes) const { const TOlapTablePartition& t_part = _t_param.partitions[0]; - decltype(_partitions_map->find(tuple)) it; + std::map::iterator it; if (t_part.__isset.in_keys) { it = _partitions_map->find(tuple); } else { diff --git a/docs/en/getting-started/data-partition.md b/docs/en/getting-started/data-partition.md index 0d41d777b9ffc5..4ab0241d951f9b 100644 --- a/docs/en/getting-started/data-partition.md +++ b/docs/en/getting-started/data-partition.md @@ -61,12 +61,12 @@ This section introduces Doris's approach to building tables with an example. CREATE TABLE IF NOT EXISTS example_db.expamle_range_tbl ( - `user_id` LARGEINT NOT NULL COMMENT "user id", + `user_id` LARGEINT NOT NULL COMMENT "User id", `date` DATE NOT NULL COMMENT "Data fill in date time", `timestamp` DATETIME NOT NULL COMMENT "Timestamp of data being poured", `city` VARCHAR(20) COMMENT "The city where the user is located", - `age` SMALLINT COMMENT "user age", - `sex` TINYINT COMMENT "User Gender", + `age` SMALLINT COMMENT "User age", + `sex` TINYINT COMMENT "User gender", `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "User last visit time", `cost` BIGINT SUM DEFAULT "0" COMMENT "Total user consumption", `max_dwell_time` INT MAX DEFAULT "0" COMMENT "User maximum dwell time", @@ -93,16 +93,16 @@ PROPERTIES CREATE TABLE IF NOT EXISTS example_db.expamle_list_tbl ( - `user_id` LARGEINT NOT NULL COMMENT "用户id", - `date` DATE NOT NULL COMMENT "数据灌入日期时间", - `timestamp` DATETIME NOT NULL COMMENT "数据灌入的时间戳", - `city` VARCHAR(20) COMMENT "用户所在城市", - `age` SMALLINT COMMENT "用户年龄", - `sex` TINYINT COMMENT "用户性别", - `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", - `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", - `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", - `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间" + `user_id` LARGEINT NOT NULL COMMENT "User id", + `date` DATE NOT NULL COMMENT "Data fill in date time", + `timestamp` DATETIME NOT NULL COMMENT "Timestamp of data being poured", + `city` VARCHAR(20) COMMENT "The city where the user is located", + `age` SMALLINT COMMENT "User Age", + `sex` TINYINT COMMENT "User gender", + `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "User last visit time", + `cost` BIGINT SUM DEFAULT "0" COMMENT "Total user consumption", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "User maximum dwell time", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "User minimum dwell time" ) ENGINE=olap AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) diff --git a/docs/zh-CN/administrator-guide/load-data/delete-manual.md b/docs/zh-CN/administrator-guide/load-data/delete-manual.md index 0f460b9998b343..6bfdeab57eea7a 100644 --- a/docs/zh-CN/administrator-guide/load-data/delete-manual.md +++ b/docs/zh-CN/administrator-guide/load-data/delete-manual.md @@ -64,10 +64,10 @@ DELETE FROM my_table PARTITION p1 WHERE k1 < 3 AND k2 = "abc"; 1. `Where`语句中的op的类型可包括`=, >, <, >=, <=, !=, in, not in`。 2. `Where`语句中的列只能是`key`列 -3. 当选定的`key`列不存在某个rollup表内时,无法进行delete -4. 条件语句中各个条件只能是`and`关系,如希望达成`or`可将条件分别写入两个delete语句中 -5. 如果指定表为RANGE或者LIST分区表,则必须指定 `PARTITION`。如果是单分区表,可以不指定。 -6. 不同于Insert into命令,delete不能手动指定`label`,有关label的概念可以查看[Insert Into文档](./insert-into-manual.md) +3. 当选定的`key`列不存在某个 rollup 表内时,无法进行 delete +4. 条件语句中各个条件只能是`and`关系,如希望达成`or`可将条件分别写入两个 delete 语句中 +5. 如果指定表为 RANGE 或者 LIST 分区表,则必须指定 `PARTITION`。如果是单分区表,可以不指定。 +6. 不同于 Insert into 命令,delete 不能手动指定`label`,有关 label 的概念可以查看[Insert Into文档](./insert-into-manual.md) ## 返回结果 From 032b83852883c8c8e525be4fb559024d26bcb8be Mon Sep 17 00:00:00 2001 From: qijianliang01 Date: Fri, 23 Apr 2021 11:07:29 +0800 Subject: [PATCH 6/6] support filter partitions with list partition in stream load plan Change-Id: I35900b2e3ff4d7a78b5785751996e6563c919f9d --- .../doris/planner/StreamLoadPlanner.java | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java index 32a1b344cbcc4f..39c0f5d35c0494 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java @@ -30,9 +30,9 @@ import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionType; -import org.apache.doris.catalog.RangePartitionInfo; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; @@ -54,7 +54,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Range; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -219,11 +218,11 @@ private List getAllPartitionIds() throws DdlException, AnalysisException { } } else { List conjuncts = scanNode.getConjuncts(); - if (destTable.getPartitionInfo().getType() == PartitionType.RANGE && !conjuncts.isEmpty()) { - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo)(destTable.getPartitionInfo()); - Map> keyRangeById = rangePartitionInfo.getIdToRange(false); + if (destTable.getPartitionInfo().getType() != PartitionType.UNPARTITIONED && !conjuncts.isEmpty()) { + PartitionInfo partitionInfo = destTable.getPartitionInfo(); + Map itemById = partitionInfo.getIdToItem(false); Map columnFilters = Maps.newHashMap(); - for (Column column : rangePartitionInfo.getPartitionColumns()) { + for (Column column : partitionInfo.getPartitionColumns()) { SlotDescriptor slotDesc = tupleDesc.getColumnSlot(column.getName()); if (null == slotDesc) { continue; @@ -234,10 +233,16 @@ private List getAllPartitionIds() throws DdlException, AnalysisException { } } if (columnFilters.isEmpty()) { - partitionIds.addAll(keyRangeById.keySet()); + partitionIds.addAll(itemById.keySet()); } else { - PartitionPruner partitionPruner = new RangePartitionPruner(keyRangeById, - rangePartitionInfo.getPartitionColumns(), columnFilters); + PartitionPruner partitionPruner = null; + if (destTable.getPartitionInfo().getType() == PartitionType.RANGE) { + partitionPruner = new RangePartitionPruner(itemById, + partitionInfo.getPartitionColumns(), columnFilters); + } else if (destTable.getPartitionInfo().getType() == PartitionType.LIST) { + partitionPruner = new ListPartitionPruner(itemById, + partitionInfo.getPartitionColumns(), columnFilters); + } partitionIds.addAll(partitionPruner.prune()); } } else {