Skip to content

[fix](statistics)Fix select mv with specified partitions bug. #36817

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Jibing-Li merged 1 commit into from
Jun 26, 2024
Merged

[fix](statistics)Fix select mv with specified partitions bug. #36817

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,12 +187,13 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation,
LogicalOlapScan scan;
List<Long> partIds = getPartitionIds(table, unboundRelation, qualifiedTableName);
List<Long> tabletIds = unboundRelation.getTabletIds();
if (!CollectionUtils.isEmpty(partIds)) {
if (!CollectionUtils.isEmpty(partIds) && !unboundRelation.getIndexName().isPresent()) {
scan = new LogicalOlapScan(unboundRelation.getRelationId(),
(OlapTable) table, qualifiedTableName, partIds,
tabletIds, unboundRelation.getHints(), unboundRelation.getTableSample());
} else {
Optional<String> indexName = unboundRelation.getIndexName();
// For direct mv scan.
if (indexName.isPresent()) {
OlapTable olapTable = (OlapTable) table;
Long indexId = olapTable.getIndexIdByName(indexName.get());
Expand All @@ -204,8 +205,10 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation,
: PreAggStatus.off("For direct index scan on mor/agg.");

scan = new LogicalOlapScan(unboundRelation.getRelationId(),
(OlapTable) table, qualifiedTableName, tabletIds, indexId,
preAggStatus, unboundRelation.getHints(), unboundRelation.getTableSample());
(OlapTable) table, qualifiedTableName, tabletIds,
CollectionUtils.isEmpty(partIds) ? ((OlapTable) table).getPartitionIds() : partIds, indexId,
preAggStatus, CollectionUtils.isEmpty(partIds) ? ImmutableList.of() : partIds,
unboundRelation.getHints(), unboundRelation.getTableSample());
} else {
scan = new LogicalOlapScan(unboundRelation.getRelationId(),
(OlapTable) table, qualifiedTableName, tabletIds, unboundRelation.getHints(),
Expand Down
2 changes: 2 additions & 0 deletions ...re/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,8 +212,10 @@ public static Plan generateMvScanPlan(MTMV materializedView, CascadesContext cas
materializedView,
materializedView.getFullQualifiers(),
ImmutableList.of(),
materializedView.getPartitionIds(),
materializedView.getBaseIndexId(),
PreAggStatus.on(),
ImmutableList.of(),
// this must be empty, or it will be used to sample
ImmutableList.of(),
Optional.empty());
Expand Down
8 changes: 4 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,12 +148,12 @@ public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, L
}

public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> tabletIds,
long selectedIndexId, PreAggStatus preAggStatus, List<String> hints,
Optional<TableSample> tableSample) {
List<Long> selectedPartitionIds, long selectedIndexId, PreAggStatus preAggStatus,
List<Long> specifiedPartitions, List<String> hints, Optional<TableSample> tableSample) {
this(id, table, qualifier, Optional.empty(), Optional.empty(),
table.getPartitionIds(), false, tabletIds,
selectedPartitionIds, false, tabletIds,
selectedIndexId, true, preAggStatus,
ImmutableList.of(), hints, Maps.newHashMap(), tableSample, true, ImmutableMap.of());
specifiedPartitions, hints, Maps.newHashMap(), tableSample, true, ImmutableMap.of());
}

/**
Expand Down
113 changes: 113 additions & 0 deletions regression-test/suites/statistics/test_partition_stats.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,24 @@ suite("test_partition_stats") {

}

def wait_mv_finish = { db, table ->
for (int loop = 0; loop < 300; loop++) {
Thread.sleep(1000)
boolean finished = true;
def result = sql """SHOW ALTER TABLE MATERIALIZED VIEW FROM ${db} WHERE tableName="${table}";"""
for (int i = 0; i < result.size(); i++) {
if (result[i][8] != 'FINISHED') {
finished = false;
break;
}
}
if (finished) {
return;
}
}
throw new Exception("Wait mv finish timeout.")
}

def enable = sql """show variables like "%enable_partition_analyze%" """
if (enable[0][1].equalsIgnoreCase("false")) {
logger.info("partition analyze disabled. " + enable)
Expand Down Expand Up @@ -815,6 +833,101 @@ suite("test_partition_stats") {
assertEquals("0.0", result[7][2])
assertEquals("0.0", result[8][2])

// Test mv and rollup
sql """CREATE TABLE `part8` (
`id` INT NULL,
`colint` INT NULL,
`coltinyint` tinyint NULL,
`colsmallint` smallINT NULL,
`colbigint` bigINT NULL,
`collargeint` largeINT NULL,
`colfloat` float NULL,
`coldouble` double NULL,
`coldecimal` decimal(27, 9) NULL
) ENGINE=OLAP
DUPLICATE KEY(`id`)
COMMENT 'OLAP'
PARTITION BY RANGE(`id`)
(
PARTITION p1 VALUES [("-2147483648"), ("10000")),
PARTITION p2 VALUES [("10000"), ("20000")),
PARTITION p3 VALUES [("20000"), ("30000"))
)
DISTRIBUTED BY HASH(`id`) BUCKETS 3
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
)"""
sql """create materialized view mv1 as select id, colint from part8;"""
wait_mv_finish("test_partition_stats", "part8")
sql """create materialized view mv2 as select colsmallint, sum(colbigint) from part8 group by colsmallint;"""
wait_mv_finish("test_partition_stats", "part8")
sql """alter table part8 ADD ROLLUP rollup1(coltinyint, collargeint)"""
wait_mv_finish("test_partition_stats", "part8")

sql """Insert into part8 values (1, 1, 1, 1, 1, 1, 1.1, 1.1, 1.1), (2, 2, 2, 2, 2, 2, 2.2, 2.2, 2.2), (3, 3, 3, 3, 3, 3, 3.3, 3.3, 3.3),(4, 4, 4, 4, 4, 4, 4.4, 4.4, 4.4),(5, 5, 5, 5, 5, 5, 5.5, 5.5, 5.5),(6, 6, 6, 6, 6, 6, 6.6, 6.6, 6.6),(1, 1, 1, 1, 1, 1, 1.1, 1.1, 1.1), (2, 2, 2, 2, 2, 2, 2.2, 2.2, 2.2), (3, 3, 3, 3, 3, 3, 3.3, 3.3, 3.3),(4, 4, 4, 4, 4, 4, 4.4, 4.4, 4.4),(5, 5, 5, 5, 5, 5, 5.5, 5.5, 5.5),(6, 6, 6, 6, 6, 6, 6.6, 6.6, 6.6),(10001, 10001, 10001, 10001, 10001, 10001, 10001.10001, 10001.10001, 10001.10001),(10002, 10002, 10002, 10002, 10002, 10002, 10002.10002, 10002.10002, 10002.10002),(10003, 10003, 10003, 10003, 10003, 10003, 10003.10003, 10003.10003, 10003.10003),(10004, 10004, 10004, 10004, 10004, 10004, 10004.10004, 10004.10004, 10004.10004),(10005, 10005, 10005, 10005, 10005, 10005, 10005.10005, 10005.10005, 10005.10005),(20001, 20001, 20001, 20001, 20001, 20001, 20001.20001, 20001.20001, 20001.20001),(20002, 20002, 20002, 20002, 20002, 20002, 20002.20002, 20002.20002, 20002.20002),(20003, 20003, 20003, 20003, 20003, 20003, 20003.20003, 20003.20003, 20003.20003),(20004, 20004, 20004, 20004, 20004, 20004, 20004.20004, 20004.20004, 20004.20004)"""
sql """analyze table part8 with sync"""
result = sql """show column stats part8"""
assertEquals(15, result.size())
result = sql """show column stats part8 (mv_id)"""
assertEquals(1, result.size())
assertEquals("mv_id", result[0][0])
assertEquals("mv1", result[0][1])
assertEquals("21.0", result[0][2])
assertEquals("15.0", result[0][3])
assertEquals("0.0", result[0][4])
assertEquals("1", result[0][7])
assertEquals("20004", result[0][8])

result = sql """show column stats part8 (mv_colint)"""
assertEquals(1, result.size())
assertEquals("mv_colint", result[0][0])
assertEquals("mv1", result[0][1])
assertEquals("21.0", result[0][2])
assertEquals("15.0", result[0][3])
assertEquals("0.0", result[0][4])
assertEquals("1", result[0][7])
assertEquals("20004", result[0][8])

result = sql """show column stats part8 (mv_colsmallint)"""
assertEquals(1, result.size())
assertEquals("mv_colsmallint", result[0][0])
assertEquals("mv2", result[0][1])
assertEquals("15.0", result[0][2])
assertEquals("15.0", result[0][3])
assertEquals("0.0", result[0][4])
assertEquals("1", result[0][7])
assertEquals("20004", result[0][8])

result = sql """show column stats part8 (`mva_SUM__``colbigint```)"""
assertEquals(1, result.size())
assertEquals("mva_SUM__`colbigint`", result[0][0])
assertEquals("mv2", result[0][1])
assertEquals("15.0", result[0][2])
assertEquals("15.0", result[0][3])
assertEquals("0.0", result[0][4])
assertEquals("2", result[0][7])
assertEquals("20004", result[0][8])

result = sql """show column stats part8 (coltinyint)"""
assertEquals(2, result.size())
assertTrue(result[0][1] == "part8" && result[1][1] == "rollup1" || result[0][1] == "rollup1" && result[1][1] == "part8")
assertEquals("coltinyint", result[0][0])
assertEquals("21.0", result[0][2])
assertEquals("15.0", result[0][3])
assertEquals("0.0", result[0][4])
assertEquals("1", result[0][7])
assertEquals("36", result[0][8])

result = sql """show column stats part8 (collargeint)"""
assertEquals(2, result.size())
assertTrue(result[0][1] == "part8" && result[1][1] == "rollup1" || result[0][1] == "rollup1" && result[1][1] == "part8")
assertEquals("collargeint", result[0][0])
assertEquals("21.0", result[0][2])
assertEquals("15.0", result[0][3])
assertEquals("0.0", result[0][4])
assertEquals("1", result[0][7])
assertEquals("20004", result[0][8])

sql """drop database test_partition_stats"""
}