Skip to content

[fix](statistics)Refresh follower FE cache after alter column stats. Support alter index column stats (#31108) #31139

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Jibing-Li merged 1 commit into from
Feb 20, 2024
Merged

[fix](statistics)Refresh follower FE cache after alter column stats. Support alter index column stats (#31108) #31139

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion fe/fe-core/src/main/cup/sql_parser.cup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1356,7 +1356,12 @@ alter_stmt ::=
| KW_ALTER KW_TABLE table_name:tbl KW_MODIFY KW_COLUMN ident:columnName
KW_SET KW_STATS LPAREN key_value_map:map RPAREN opt_partition_names:partitionNames
{:
RESULT = new AlterColumnStatsStmt(tbl, columnName, map, partitionNames);
RESULT = new AlterColumnStatsStmt(tbl, null, columnName, map, partitionNames);
:}
| KW_ALTER KW_TABLE table_name:tbl KW_INDEX ident:idx KW_MODIFY KW_COLUMN ident:columnName
KW_SET KW_STATS LPAREN key_value_map:map RPAREN opt_partition_names:partitionNames
{:
RESULT = new AlterColumnStatsStmt(tbl, idx, columnName, map, partitionNames);
:}
| KW_ALTER KW_TABLE table_name:tbl KW_SET LPAREN key_value_map:properties RPAREN
{:
Expand Down
27 changes: 26 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/analysis/AlterColumnStatsStmt.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,16 +73,20 @@ public class AlterColumnStatsStmt extends DdlStmt {
.build();

private final TableName tableName;
private final String indexName;
private final String columnName;
private final Map<String, String> properties;
private final PartitionNames optPartitionNames;

private final List<Long> partitionIds = Lists.newArrayList();
private final Map<StatsType, String> statsTypeToValue = Maps.newHashMap();

public AlterColumnStatsStmt(TableName tableName, String columnName,
private long indexId = -1;

public AlterColumnStatsStmt(TableName tableName, String indexName, String columnName,
Map<String, String> properties, PartitionNames optPartitionNames) {
this.tableName = tableName;
this.indexName = indexName;
this.columnName = columnName;
this.properties = properties == null ? Collections.emptyMap() : properties;
this.optPartitionNames = optPartitionNames;
Expand All @@ -96,6 +100,10 @@ public String getColumnName() {
return columnName;
}

public long getIndexId() {
return indexId;
}

public List<Long> getPartitionIds() {
return partitionIds;
}
Expand Down Expand Up @@ -148,6 +156,19 @@ private void checkPartitionAndColumn() throws AnalysisException {
DatabaseIf db = catalog.getDbOrAnalysisException(tableName.getDb());
TableIf table = db.getTableOrAnalysisException(tableName.getTbl());

if (indexName != null) {
if (!(table instanceof OlapTable)) {
throw new AnalysisException("Only OlapTable support alter index stats. "
+ "Table " + table.getName() + " is not OlapTable.");
}
OlapTable olapTable = (OlapTable) table;
Long idxId = olapTable.getIndexIdByName(indexName);
if (idxId == null) {
throw new AnalysisException("Index " + indexName + " not exist in table " + table.getName());
}
indexId = idxId;
}

if (table.getColumn(columnName) == null) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
columnName, FeNameFormat.getColumnNameRegex());
Expand Down Expand Up @@ -176,6 +197,10 @@ public String toSql() {
StringBuilder sb = new StringBuilder();
sb.append("ALTER TABLE ");
sb.append(tableName.toSql());
if (indexName != null) {
sb.append(" INDEX ");
sb.append(indexName);
}
sb.append(" MODIFY COLUMN ");
sb.append(columnName);
sb.append(" SET STATS ");
Expand Down
12 changes: 12 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,18 @@ public ColStatsData(ResultRow row) {
this.updateTime = row.get(13);
}

public ColStatsData(String id, long catalogId, long dbId, long tblId, long idxId, String colId, String partId,
ColumnStatistic columnStatistic) {
this.statsId = new StatsId(id, catalogId, dbId, tblId, idxId, colId, partId);
this.count = Math.round(columnStatistic.count);
this.ndv = Math.round(columnStatistic.ndv);
this.nullCount = Math.round(columnStatistic.numNulls);
this.minLit = columnStatistic.minExpr == null ? null : columnStatistic.minExpr.getStringValue();
this.maxLit = columnStatistic.maxExpr == null ? null : columnStatistic.maxExpr.getStringValue();
this.dataSizeInBytes = Math.round(columnStatistic.dataSize);
this.updateTime = columnStatistic.updatedTime;
}

public String toSQL(boolean roundByParentheses) {
StringJoiner sj = null;
if (roundByParentheses) {
Expand Down
11 changes: 7 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,7 @@ public static void alterColumnStatistics(AlterColumnStatsStmt alterColumnStatsSt
String min = alterColumnStatsStmt.getValue(StatsType.MIN_VALUE);
String max = alterColumnStatsStmt.getValue(StatsType.MAX_VALUE);
String dataSize = alterColumnStatsStmt.getValue(StatsType.DATA_SIZE);
long indexId = alterColumnStatsStmt.getIndexId();
ColumnStatisticBuilder builder = new ColumnStatisticBuilder();
String colName = alterColumnStatsStmt.getColumnName();
Column column = objects.table.getColumn(colName);
Expand Down Expand Up @@ -284,10 +285,10 @@ public static void alterColumnStatistics(AlterColumnStatsStmt alterColumnStatsSt

ColumnStatistic columnStatistic = builder.build();
Map<String, String> params = new HashMap<>();
params.put("id", constructId(objects.table.getId(), -1, colName));
params.put("id", constructId(objects.table.getId(), indexId, colName));
params.put("catalogId", String.valueOf(objects.catalog.getId()));
params.put("dbId", String.valueOf(objects.db.getId()));
params.put("idxId", "-1");
params.put("idxId", String.valueOf(indexId));
params.put("tblId", String.valueOf(objects.table.getId()));
params.put("colId", String.valueOf(colName));
params.put("count", String.valueOf(columnStatistic.count));
Expand All @@ -301,8 +302,10 @@ public static void alterColumnStatistics(AlterColumnStatsStmt alterColumnStatsSt
// update table granularity statistics
params.put("partId", "NULL");
StatisticsUtil.execUpdate(INSERT_INTO_COLUMN_STATISTICS, params);
Env.getCurrentEnv().getStatisticsCache()
.updateColStatsCache(objects.table.getId(), -1, colName, columnStatistic);
ColStatsData data = new ColStatsData(constructId(objects.table.getId(), indexId, colName),
objects.catalog.getId(), objects.db.getId(), objects.table.getId(), indexId, colName,
null, columnStatistic);
Env.getCurrentEnv().getStatisticsCache().syncColStats(data);
AnalysisInfo mockedJobInfo = new AnalysisInfoBuilder()
.setTblUpdateTime(System.currentTimeMillis())
.setColName("")
Expand Down
42 changes: 42 additions & 0 deletions regression-test/suites/statistics/test_analyze_mv.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,48 @@ suite("test_analyze_mv") {
verifyTaskStatus(result_sample, "mva_MIN__`value3`", "mv3")
verifyTaskStatus(result_sample, "mva_SUM__CAST(`value1` AS BIGINT)", "mv3")

// Test alter column stats
sql """drop stats mvTestDup"""
sql """alter table mvTestDup modify column key1 set stats ('ndv'='1', 'num_nulls'='1', 'min_value'='10', 'max_value'='40', 'row_count'='50');"""
sql """alter table mvTestDup index mv3 modify column mv_key1 set stats ('ndv'='5', 'num_nulls'='0', 'min_value'='0', 'max_value'='4', 'row_count'='5');"""
sql """alter table mvTestDup index mv3 modify column `mva_SUM__CAST(``value1`` AS BIGINT)` set stats ('ndv'='10', 'num_nulls'='2', 'min_value'='1', 'max_value'='5', 'row_count'='11');"""

def result = sql """show column cached stats mvTestDup(key1)"""
assertEquals(1, result.size())
assertEquals("key1", result[0][0])
assertEquals("N/A", result[0][1])
assertEquals("50.0", result[0][2])
assertEquals("1.0", result[0][3])
assertEquals("1.0", result[0][4])
assertEquals("0.0", result[0][5])
assertEquals("0.0", result[0][6])
assertEquals("10", result[0][7])
assertEquals("40", result[0][8])

result = sql """show column cached stats mvTestDup(mv_key1)"""
assertEquals(1, result.size())
assertEquals("mv_key1", result[0][0])
assertEquals("mv3", result[0][1])
assertEquals("5.0", result[0][2])
assertEquals("5.0", result[0][3])
assertEquals("0.0", result[0][4])
assertEquals("0.0", result[0][5])
assertEquals("0.0", result[0][6])
assertEquals("0", result[0][7])
assertEquals("4", result[0][8])

result = sql """show column cached stats mvTestDup(`mva_SUM__CAST(``value1`` AS BIGINT)`)"""
assertEquals(1, result.size())
assertEquals("mva_SUM__CAST(`value1` AS BIGINT)", result[0][0])
assertEquals("mv3", result[0][1])
assertEquals("11.0", result[0][2])
assertEquals("10.0", result[0][3])
assertEquals("2.0", result[0][4])
assertEquals("0.0", result[0][5])
assertEquals("0.0", result[0][6])
assertEquals("1", result[0][7])
assertEquals("5", result[0][8])

sql """drop database if exists test_analyze_mv"""
}