Skip to content

branch-2.1: [improvement](statistics)Add session variable for partition sample count. #48218 #49092

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 1 commit into from
Mar 14, 2025
Merged

branch-2.1: [improvement](statistics)Add session variable for partition sample count. #48218 #49092

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -531,6 +531,8 @@ public class SessionVariable implements Serializable, Writable {

public static final String HUGE_TABLE_DEFAULT_SAMPLE_ROWS = "huge_table_default_sample_rows";
public static final String HUGE_TABLE_LOWER_BOUND_SIZE_IN_BYTES = "huge_table_lower_bound_size_in_bytes";
public static final String PARTITION_SAMPLE_COUNT = "partition_sample_count";
public static final String PARTITION_SAMPLE_ROW_COUNT = "partition_sample_row_count";

// for spill to disk
public static final String EXTERNAL_SORT_BYTES_THRESHOLD = "external_sort_bytes_threshold";
Expand Down Expand Up @@ -1916,6 +1918,18 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
+ "considered outdated."})
public int tableStatsHealthThreshold = 90;

@VariableMgr.VarAttr(name = PARTITION_SAMPLE_COUNT, flag = VariableMgr.GLOBAL,
description = {
"大分区表采样的分区数上限",
"The upper limit of the number of partitions for sampling large partitioned tables.\n"})
public int partitionSampleCount = 30;

@VariableMgr.VarAttr(name = PARTITION_SAMPLE_ROW_COUNT, flag = VariableMgr.GLOBAL,
description = {
"大分区表采样的行数上限",
"The upper limit of the number of rows for sampling large partitioned tables.\n"})
public long partitionSampleRowCount = 3_000_000_000L;

@VariableMgr.VarAttr(name = ENABLE_MATERIALIZED_VIEW_REWRITE, needForward = true,
description = {"是否开启基于结构信息的物化视图透明改写",
"Whether to enable materialized view rewriting based on struct info"})
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
private boolean partitionColumnSampleTooManyRows = false;
private boolean scanFullTable = false;
private static final long MAXIMUM_SAMPLE_ROWS = 1_000_000_000;
private static final int PARTITION_COUNT_TO_SAMPLE = 5;

@VisibleForTesting
public OlapAnalysisTask() {
Expand Down Expand Up @@ -336,7 +335,8 @@ protected long pickSamplePartition(List<Partition> partitions, List<Long> picked
MaterializedIndex materializedIndex = p.getIndex(indexId);
pickedTabletIds.addAll(materializedIndex.getTabletIdsInOrder());
}
if (pickedRows >= MAXIMUM_SAMPLE_ROWS || pickedPartitionCount > PARTITION_COUNT_TO_SAMPLE) {
if (pickedRows >= StatisticsUtil.getPartitionSampleRowCount()
|| pickedPartitionCount >= StatisticsUtil.getPartitionSampleCount()) {
break;
}
}
Expand Down
4 changes: 4 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,10 @@ public class StatisticConstants {

public static final int MSG_LEN_UPPER_BOUND = 1024;

public static final int PARTITION_SAMPLE_COUNT = 30;

public static final long PARTITION_SAMPLE_ROW_COUNT = 3_000_000_000L;

static {
SYSTEM_DBS.add(FeConstants.INTERNAL_DB_NAME);
SYSTEM_DBS.add(InfoSchemaDb.DATABASE_NAME);
Expand Down
18 changes: 18 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -858,6 +858,24 @@ public static int getAutoAnalyzeTableWidthThreshold() {
return StatisticConstants.AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD;
}

public static int getPartitionSampleCount() {
try {
return findConfigFromGlobalSessionVar(SessionVariable.PARTITION_SAMPLE_COUNT).partitionSampleCount;
} catch (Exception e) {
LOG.warn("Fail to get value of partition_sample_count, return default", e);
}
return StatisticConstants.PARTITION_SAMPLE_COUNT;
}

public static long getPartitionSampleRowCount() {
try {
return findConfigFromGlobalSessionVar(SessionVariable.PARTITION_SAMPLE_ROW_COUNT).partitionSampleRowCount;
} catch (Exception e) {
LOG.warn("Fail to get value of partition_sample_row_count, return default", e);
}
return StatisticConstants.PARTITION_SAMPLE_ROW_COUNT;
}

public static String encodeValue(ResultRow row, int index) {
if (row == null || row.getValues().size() <= index) {
return "NULL";
Expand Down