From ec59d28e898d50455ab8fb919aead8d4cff9c869 Mon Sep 17 00:00:00 2001 From: lxr599 Date: Tue, 2 Jul 2024 16:44:35 +0800 Subject: [PATCH 01/23] change default value of row_column_page_size to 16KB --- be/src/common/config.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 0e2680a7949dec..8a7ba6dc7be55d 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1040,8 +1040,8 @@ DEFINE_mInt64(max_tablet_io_errors, "-1"); DEFINE_Int32(tablet_path_check_interval_seconds, "-1"); DEFINE_mInt32(tablet_path_check_batch_size, "1000"); -// Page size of row column, default 4KB -DEFINE_mInt64(row_column_page_size, "4096"); +// Page size of row column, default 16KB +DEFINE_mInt64(row_column_page_size, "16384"); // it must be larger than or equal to 5MB DEFINE_mInt64(s3_write_buffer_size, "5242880"); // Log interval when doing s3 upload task From 76a78da08aea230b0157f0f5d22053366ee710a1 Mon Sep 17 00:00:00 2001 From: lxr599 Date: Thu, 4 Jul 2024 19:14:17 +0800 Subject: [PATCH 02/23] allow to set row_column_page_size for tables --- be/src/common/config.cpp | 2 -- be/src/common/config.h | 2 -- be/src/olap/rowset/segment_v2/options.h | 2 ++ .../olap/rowset/segment_v2/segment_writer.cpp | 2 +- .../segment_v2/vertical_segment_writer.cpp | 2 +- be/src/olap/tablet_meta.cpp | 3 ++ be/src/olap/tablet_schema.cpp | 4 +++ be/src/olap/tablet_schema.h | 6 ++++ .../apache/doris/alter/CloudRollupJobV2.java | 3 +- .../doris/alter/CloudSchemaChangeJobV2.java | 3 +- .../org/apache/doris/alter/RollupJobV2.java | 3 +- .../apache/doris/alter/SchemaChangeJobV2.java | 3 +- .../org/apache/doris/backup/RestoreJob.java | 3 +- .../java/org/apache/doris/catalog/Env.java | 4 +++ .../org/apache/doris/catalog/OlapTable.java | 14 +++++++++ .../apache/doris/catalog/TableProperty.java | 13 +++++++++ .../datasource/CloudInternalCatalog.java | 10 +++++-- .../doris/common/util/PropertyAnalyzer.java | 29 +++++++++++++++++++ .../doris/datasource/InternalCatalog.java | 27 +++++++++++++---- .../apache/doris/master/ReportHandler.java | 3 +- .../apache/doris/task/CreateReplicaTask.java | 6 +++- .../org/apache/doris/task/AgentTaskTest.java | 3 +- gensrc/proto/olap_file.proto | 2 ++ gensrc/thrift/AgentService.thrift | 1 + 24 files changed, 127 insertions(+), 23 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 8a7ba6dc7be55d..a14f91165cc7af 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1040,8 +1040,6 @@ DEFINE_mInt64(max_tablet_io_errors, "-1"); DEFINE_Int32(tablet_path_check_interval_seconds, "-1"); DEFINE_mInt32(tablet_path_check_batch_size, "1000"); -// Page size of row column, default 16KB -DEFINE_mInt64(row_column_page_size, "16384"); // it must be larger than or equal to 5MB DEFINE_mInt64(s3_write_buffer_size, "5242880"); // Log interval when doing s3 upload task diff --git a/be/src/common/config.h b/be/src/common/config.h index 4cbb9d14c3fecd..a97fbe7852a726 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1084,8 +1084,6 @@ DECLARE_mInt64(max_tablet_io_errors); DECLARE_Int32(tablet_path_check_interval_seconds); DECLARE_mInt32(tablet_path_check_batch_size); -// Page size of row column, default 4KB -DECLARE_mInt64(row_column_page_size); // it must be larger than or equal to 5MB DECLARE_mInt64(s3_write_buffer_size); // Log interval when doing s3 upload task diff --git a/be/src/olap/rowset/segment_v2/options.h b/be/src/olap/rowset/segment_v2/options.h index 19041f4c51d1db..5fd5dba9d24c01 100644 --- a/be/src/olap/rowset/segment_v2/options.h +++ b/be/src/olap/rowset/segment_v2/options.h @@ -24,6 +24,8 @@ namespace segment_v2 { static constexpr size_t DEFAULT_PAGE_SIZE = 1024 * 1024; // default size: 1M +constexpr long ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE = 16384; // default row column page size: 16KB + struct PageBuilderOptions { size_t data_page_size = DEFAULT_PAGE_SIZE; diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index a4c2d988ab98d2..e0de1300b2beff 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -263,7 +263,7 @@ Status SegmentWriter::init(const std::vector& col_ids, bool has_key) { if (column.is_row_store_column()) { // smaller page size for row store column - opts.data_page_size = config::row_column_page_size; + opts.data_page_size = _tablet_schema->row_column_page_size(); } std::unique_ptr writer; RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer, &writer)); diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp index 448a2b7304c5fa..bc6681c0548325 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp @@ -216,7 +216,7 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo if (column.is_row_store_column()) { // smaller page size for row store column - opts.data_page_size = config::row_column_page_size; + opts.data_page_size = _tablet_schema->row_column_page_size(); } std::unique_ptr writer; RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer, &writer)); diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 3cd001fd3e19d4..5bc19005b3336f 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -301,6 +301,9 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id if (tablet_schema.__isset.store_row_column) { schema->set_store_row_column(tablet_schema.store_row_column); } + if (tablet_schema.__isset.row_column_page_size) { + schema->set_row_column_page_size(tablet_schema.row_column_page_size); + } if (tablet_schema.__isset.skip_write_index_on_load) { schema->set_skip_write_index_on_load(tablet_schema.skip_write_index_on_load); } diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index 11d3456653dd75..bf4aa1945deb23 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -969,6 +969,7 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac _sort_type = schema.sort_type(); _sort_col_num = schema.sort_col_num(); _compression_type = schema.compression_type(); + _row_column_page_size = schema.row_column_page_size(); _schema_version = schema.schema_version(); // Default to V1 inverted index storage format for backward compatibility if not specified in schema. if (!schema.has_inverted_index_storage_format()) { @@ -1009,6 +1010,7 @@ void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version _skip_write_index_on_load = ori_tablet_schema.skip_write_index_on_load(); _sort_type = ori_tablet_schema.sort_type(); _sort_col_num = ori_tablet_schema.sort_col_num(); + _row_column_page_size = ori_tablet_schema.row_column_page_size(); // copy from table_schema_param _schema_version = version; @@ -1162,6 +1164,7 @@ void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const { tablet_schema_pb->set_sort_col_num(_sort_col_num); tablet_schema_pb->set_schema_version(_schema_version); tablet_schema_pb->set_compression_type(_compression_type); + tablet_schema_pb->set_row_column_page_size(_row_column_page_size); tablet_schema_pb->set_version_col_idx(_version_col_idx); tablet_schema_pb->set_inverted_index_storage_format(_inverted_index_storage_format); } @@ -1464,6 +1467,7 @@ bool operator==(const TabletSchema& a, const TabletSchema& b) { if (a._disable_auto_compaction != b._disable_auto_compaction) return false; if (a._enable_single_replica_compaction != b._enable_single_replica_compaction) return false; if (a._store_row_column != b._store_row_column) return false; + if (a._row_column_page_size != b._row_column_page_size) return false; if (a._skip_write_index_on_load != b._skip_write_index_on_load) return false; return true; } diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index d08b95ae00fc8c..5bbc1e79c7ce35 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -36,6 +36,7 @@ #include "common/status.h" #include "gutil/stringprintf.h" #include "olap/olap_common.h" +#include "olap/rowset/segment_v2/options.h" #include "runtime/define_primitive_type.h" #include "runtime/descriptors.h" #include "util/string_util.h" @@ -334,6 +335,10 @@ class TabletSchema { void set_version_col_idx(int32_t version_col_idx) { _version_col_idx = version_col_idx; } int32_t version_col_idx() const { return _version_col_idx; } segment_v2::CompressionTypePB compression_type() const { return _compression_type; } + void set_row_column_page_size(long page_size) { + _row_column_page_size = page_size; + } + long row_column_page_size() const { return _row_column_page_size; } const std::vector& indexes() const { return _indexes; } bool has_inverted_index() const { @@ -478,6 +483,7 @@ class TabletSchema { size_t _num_rows_per_row_block = 0; CompressKind _compress_kind = COMPRESS_NONE; segment_v2::CompressionTypePB _compression_type = segment_v2::CompressionTypePB::LZ4F; + long _row_column_page_size = segment_v2::ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE; size_t _next_column_unique_id = 0; std::string _auto_increment_column; diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java index 6ab7ffface0835..61ee21efbd4351 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java @@ -206,7 +206,8 @@ private void createRollupReplicaForPartition(OlapTable tbl) throws Exception { tbl.getTimeSeriesCompactionTimeThresholdSeconds(), tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), tbl.getTimeSeriesCompactionLevelThreshold(), - tbl.disableAutoCompaction()); + tbl.disableAutoCompaction(), + tbl.rowColumnPageSize()); requestBuilder.addTabletMetas(builder); } // end for rollupTablets ((CloudInternalCatalog) Env.getCurrentInternalCatalog()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java index 6e41b2b4312661..042b4196d29143 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java @@ -214,7 +214,8 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio tbl.getTimeSeriesCompactionTimeThresholdSeconds(), tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), tbl.getTimeSeriesCompactionLevelThreshold(), - tbl.disableAutoCompaction()); + tbl.disableAutoCompaction(), + tbl.rowColumnPageSize()); requestBuilder.addTabletMetas(builder); } // end for rollupTablets ((CloudInternalCatalog) Env.getCurrentInternalCatalog()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java index 93aceb0936b445..a6a26e7313e091 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -261,7 +261,8 @@ protected void createRollupReplica() throws AlterCancelException { tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), tbl.getTimeSeriesCompactionLevelThreshold(), tbl.storeRowColumn(), - binlogConfig); + binlogConfig, + tbl.rowColumnPageSize()); createReplicaTask.setBaseTablet(tabletIdMap.get(rollupTabletId), baseSchemaHash); if (this.storageFormat != null) { createReplicaTask.setStorageFormat(this.storageFormat); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 277a34115413b9..1dfc62185a64ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -275,7 +275,8 @@ protected void createShadowIndexReplica() throws AlterCancelException { tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), tbl.getTimeSeriesCompactionLevelThreshold(), tbl.storeRowColumn(), - binlogConfig); + binlogConfig, + tbl.rowColumnPageSize()); createReplicaTask.setBaseTablet(partitionIndexTabletMap.get(partitionId, shadowIdxId) .get(shadowTabletId), originSchemaHash); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 51220dd13b7064..51d47141ea706c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -1099,7 +1099,8 @@ private void createReplicas(Database db, AgentBatchTask batchTask, OlapTable loc localTbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), localTbl.getTimeSeriesCompactionLevelThreshold(), localTbl.storeRowColumn(), - binlogConfig); + binlogConfig, + localTbl.rowColumnPageSize()); task.setInRestoreMode(true); batchTask.addTask(task); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index f029d9e5e58d4f..857608f5922d4c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3499,6 +3499,10 @@ public static void getDdlStmt(DdlStmt ddlStmt, String dbName, TableIf table, Lis sb.append(olapTable.getCompressionType()).append("\""); } + // row column page size + sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_ROW_COLUMN_PAGE_SIZE).append("\" = \""); + sb.append(olapTable.rowColumnPageSize()).append("\""); + // estimate_partition_size if (!olapTable.getEstimatePartitionSize().equals("")) { sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_ESTIMATE_PARTITION_SIZE).append("\" = \""); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 4e9218df817c0a..ebbf0dd6a975cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -2464,6 +2464,20 @@ public void setCompressionType(TCompressionType compressionType) { tableProperty.buildCompressionType(); } + public void setRowColumnPageSize(long pageSize) { + TableProperty tableProperty = getOrCreatTableProperty(); + tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_ROW_COLUMN_PAGE_SIZE, + Long.valueOf(pageSize).toString()); + tableProperty.buildRowColumnPageSize(); + } + + public long rowColumnPageSize() { + if (tableProperty != null) { + return tableProperty.rowColumnPageSize(); + } + return PropertyAnalyzer.ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE; + } + public void setStorageFormat(TStorageFormat storageFormat) { TableProperty tableProperty = getOrCreatTableProperty(); tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT, storageFormat.name()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java index 66d6a5291a6f2b..6018fb4bc3a213 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java @@ -94,6 +94,8 @@ public class TableProperty implements Writable { private boolean skipWriteIndexOnLoad = false; + private long rowColumnPageSize = PropertyAnalyzer.ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE; + private String compactionPolicy = PropertyAnalyzer.SIZE_BASED_COMPACTION_POLICY; private long timeSeriesCompactionGoalSizeMbytes @@ -250,6 +252,17 @@ public boolean storeRowColumn() { return storeRowColumn; } + public TableProperty buildRowColumnPageSize() { + rowColumnPageSize = Long.parseLong( + properties.getOrDefault(PropertyAnalyzer.PROPERTIES_ROW_COLUMN_PAGE_SIZE, + Long.toString(PropertyAnalyzer.ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE))); + return this; + } + + public long rowColumnPageSize() { + return rowColumnPageSize; + } + public TableProperty buildSkipWriteIndexOnLoad() { skipWriteIndexOnLoad = Boolean.parseBoolean( properties.getOrDefault(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD, "false")); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index b61a34aa28d9cf..bfc978d290942b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -100,7 +100,8 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa String storagePolicy, IdGeneratorBuffer idGeneratorBuffer, BinlogConfig binlogConfig, - boolean isStorageMediumSpecified, List clusterKeyIndexes) + boolean isStorageMediumSpecified, + List clusterKeyIndexes, long pageSize) throws DdlException { // create base index first. Preconditions.checkArgument(tbl.getBaseIndexId() != -1); @@ -170,7 +171,8 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa tbl.getTimeSeriesCompactionTimeThresholdSeconds(), tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), tbl.getTimeSeriesCompactionLevelThreshold(), - tbl.disableAutoCompaction()); + tbl.disableAutoCompaction(), + tbl.rowColumnPageSize()); requestBuilder.addTabletMetas(builder); } if (!storageVaultIdSet && ((CloudEnv) Env.getCurrentEnv()).getEnableStorageVault()) { @@ -216,7 +218,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, boolean storeRowColumn, int schemaVersion, String compactionPolicy, Long timeSeriesCompactionGoalSizeMbytes, Long timeSeriesCompactionFileCountThreshold, Long timeSeriesCompactionTimeThresholdSeconds, Long timeSeriesCompactionEmptyRowsetsThreshold, - Long timeSeriesCompactionLevelThreshold, boolean disableAutoCompaction) throws DdlException { + Long timeSeriesCompactionLevelThreshold, boolean disableAutoCompaction, long pageSize) throws DdlException { OlapFile.TabletMetaCloudPB.Builder builder = OlapFile.TabletMetaCloudPB.newBuilder(); builder.setTableId(tableId); builder.setIndexId(indexId); @@ -331,6 +333,8 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, } schemaBuilder.setDisableAutoCompaction(disableAutoCompaction); + schemaBuilder.setRowColumnPageSize(pageSize); + OlapFile.TabletSchemaCloudPB schema = schemaBuilder.build(); builder.setSchema(schema); // rowset diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 0972018f5721e9..61f0a695d7bb70 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -92,6 +92,10 @@ public class PropertyAnalyzer { public static final String PROPERTIES_TIMEOUT = "timeout"; public static final String PROPERTIES_COMPRESSION = "compression"; + // row column page size, default 16KB + public static final String PROPERTIES_ROW_COLUMN_PAGE_SIZE = "row_column_page_size"; + public static final long ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE = 16384; + public static final String PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE = "light_schema_change"; public static final String PROPERTIES_DISTRIBUTION_TYPE = "distribution_type"; @@ -974,6 +978,31 @@ public static TCompressionType analyzeCompressionType(Map proper } } + public static long alignTo4K(long size) { + return (size + 4095) & ~4095; + } + + // analyzeRowColumnPageSize will parse the row_column_page_size from properties + public static long analyzeRowColumnPageSize(Map properties) throws AnalysisException { + long rowColumnPageSize = 16384; + if (properties != null && properties.containsKey(PROPERTIES_ROW_COLUMN_PAGE_SIZE)) { + String rowColumnPageSizeStr = properties.get(PROPERTIES_ROW_COLUMN_PAGE_SIZE); + try { + rowColumnPageSize = alignTo4K(Long.parseLong(rowColumnPageSizeStr)); + } catch (NumberFormatException e) { + throw new AnalysisException("Invalid row column page size: " + rowColumnPageSizeStr); + } + + if (rowColumnPageSize <= 0) { + throw new AnalysisException("Row column page size should larger than 0."); + } + + properties.remove(PROPERTIES_ROW_COLUMN_PAGE_SIZE); + } + + return rowColumnPageSize; + } + // analyzeStorageFormat will parse the storage format from properties // sql: alter table tablet_name set ("storage_format" = "v2") // Use this sql to convert all tablets(base and rollup index) to a new format segment diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index d1d0b7d4efff44..d825b2a6ee5c64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1527,6 +1527,10 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa properties.put(PropertyAnalyzer.PROPERTIES_STORE_ROW_COLUMN, olapTable.storeRowColumn().toString()); } + if (!properties.containsKey(PropertyAnalyzer.PROPERTIES_ROW_COLUMN_PAGE_SIZE)) { + properties.put(PropertyAnalyzer.PROPERTIES_ROW_COLUMN_PAGE_SIZE, + Long.toString(olapTable.rowColumnPageSize())); + } if (!properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD)) { properties.put(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD, olapTable.skipWriteIndexOnLoad().toString()); @@ -1653,7 +1657,7 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa singlePartitionDesc.isInMemory(), singlePartitionDesc.getTabletType(), storagePolicy, idGeneratorBuffer, - binlogConfig, dataProperty.isStorageMediumSpecified(), null); + binlogConfig, dataProperty.isStorageMediumSpecified(), null, olapTable.rowColumnPageSize()); // TODO cluster key ids // check again @@ -1920,7 +1924,8 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa String storagePolicy, IdGeneratorBuffer idGeneratorBuffer, BinlogConfig binlogConfig, - boolean isStorageMediumSpecified, List clusterKeyIndexes) + boolean isStorageMediumSpecified, + List clusterKeyIndexes, long rowColumnPageSize) throws DdlException { // create base index first. Preconditions.checkArgument(tbl.getBaseIndexId() != -1); @@ -2000,7 +2005,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa tbl.getTimeSeriesCompactionTimeThresholdSeconds(), tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), tbl.getTimeSeriesCompactionLevelThreshold(), - tbl.storeRowColumn(), binlogConfig); + tbl.storeRowColumn(), binlogConfig, rowColumnPageSize); task.setStorageFormat(tbl.getStorageFormat()); task.setInvertedIndexStorageFormat(tbl.getInvertedIndexStorageFormat()); @@ -2375,6 +2380,16 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx } olapTable.setCompressionType(compressionType); + // get row_column_page_size + long rowColumnPageSize = 16384; + try { + rowColumnPageSize = PropertyAnalyzer.analyzeRowColumnPageSize(properties); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + + olapTable.setRowColumnPageSize(rowColumnPageSize); + // check data sort properties int keyColumnSize = CollectionUtils.isEmpty(keysDesc.getClusterKeysColumnIds()) ? keysDesc.keysColumnSize() : keysDesc.getClusterKeysColumnIds().size(); @@ -2748,7 +2763,7 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx idGeneratorBuffer, binlogConfigForTask, partitionInfo.getDataProperty(partitionId).isStorageMediumSpecified(), - keysDesc.getClusterKeysColumnIds()); + keysDesc.getClusterKeysColumnIds(), olapTable.rowColumnPageSize()); afterCreatePartitions(db.getId(), olapTable.getId(), null, olapTable.getIndexIdList(), true); olapTable.addPartition(partition); @@ -2830,7 +2845,7 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx partionStoragePolicy, idGeneratorBuffer, binlogConfigForTask, dataProperty.isStorageMediumSpecified(), - keysDesc.getClusterKeysColumnIds()); + keysDesc.getClusterKeysColumnIds(), olapTable.rowColumnPageSize()); olapTable.addPartition(partition); olapTable.getPartitionInfo().getDataProperty(partition.getId()) .setStoragePolicy(partionStoragePolicy); @@ -3304,7 +3319,7 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti olapTable.getPartitionInfo().getDataProperty(oldPartitionId).getStoragePolicy(), idGeneratorBuffer, binlogConfig, copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId).isStorageMediumSpecified(), - clusterKeyIdxes); + clusterKeyIdxes, olapTable.rowColumnPageSize()); newPartitions.add(newPartition); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java index c0ad4183d45cb8..802f6869bf2ab0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java @@ -881,7 +881,8 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta olapTable.getTimeSeriesCompactionEmptyRowsetsThreshold(), olapTable.getTimeSeriesCompactionLevelThreshold(), olapTable.storeRowColumn(), - binlogConfig); + binlogConfig, + olapTable.rowColumnPageSize()); createReplicaTask.setIsRecoverTask(true); createReplicaBatchTask.addTask(createReplicaTask); diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java index 7a5262ba0edcd3..396c6b10a47f1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -63,6 +63,7 @@ public class CreateReplicaTask extends AgentTask { private TStorageType storageType; private TStorageMedium storageMedium; private TCompressionType compressionType; + private long rowColumnPageSize; private List columns; @@ -144,7 +145,8 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition long timeSeriesCompactionEmptyRowsetsThreshold, long timeSeriesCompactionLevelThreshold, boolean storeRowColumn, - BinlogConfig binlogConfig) { + BinlogConfig binlogConfig, + long rowColumnPageSize) { super(null, backendId, TTaskType.CREATE, dbId, tableId, partitionId, indexId, tabletId); this.replicaId = replicaId; @@ -188,6 +190,7 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition this.timeSeriesCompactionLevelThreshold = timeSeriesCompactionLevelThreshold; this.storeRowColumn = storeRowColumn; this.binlogConfig = binlogConfig; + this.rowColumnPageSize = rowColumnPageSize; } public void setIsRecoverTask(boolean isRecoverTask) { @@ -312,6 +315,7 @@ public TCreateTabletReq toThrift() { tSchema.setEnableSingleReplicaCompaction(enableSingleReplicaCompaction); tSchema.setSkipWriteIndexOnLoad(skipWriteIndexOnLoad); tSchema.setStoreRowColumn(storeRowColumn); + tSchema.setRowColumnPageSize(rowColumnPageSize); createTabletReq.setTabletSchema(tSchema); createTabletReq.setVersion(version); diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java index 3dc4bc4695c90f..57654465b189bb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java @@ -73,6 +73,7 @@ public class AgentTaskTest { private long version = 1L; private TStorageType storageType = TStorageType.COLUMN; + private long rowColumnPageSize = 16384L; private List columns; private MarkedCountDownLatch latch = new MarkedCountDownLatch(3); @@ -107,7 +108,7 @@ public void setUp() throws AnalysisException { createReplicaTask = new CreateReplicaTask(backendId1, dbId, tableId, partitionId, indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType, TStorageMedium.SSD, columns, null, 0, latch, null, false, TTabletType.TABLET_TYPE_DISK, null, - TCompressionType.LZ4F, false, "", false, false, false, "", 0, 0, 0, 0, 0, false, null); + TCompressionType.LZ4F, false, "", false, false, false, "", 0, 0, 0, 0, 0, false, null, rowColumnPageSize); // drop dropTask = new DropReplicaTask(backendId1, tabletId1, replicaId1, schemaHash1, false); diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 82a9011dc1c78e..7a78c2f0f08d48 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -380,6 +380,7 @@ message TabletSchemaPB { optional bool skip_write_index_on_load = 23 [default=false]; repeated int32 cluster_key_idxes = 24; optional InvertedIndexStorageFormatPB inverted_index_storage_format = 25 [default=V1]; + optional int64 row_column_page_size = 26 [default=16384]; } message TabletSchemaCloudPB { @@ -406,6 +407,7 @@ message TabletSchemaCloudPB { optional bool skip_write_index_on_load = 23 [default=false]; repeated int32 cluster_key_idxes = 24; optional InvertedIndexStorageFormatPB inverted_index_storage_format = 25 [default=V1]; + optional int64 row_column_page_size = 26 [default=16384]; optional bool is_dynamic_schema = 100 [default=false]; } diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index 587715d43cd28d..e1b3e7f829cc0b 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -45,6 +45,7 @@ struct TTabletSchema { 17: optional bool enable_single_replica_compaction = false 18: optional bool skip_write_index_on_load = false 19: optional list cluster_key_idxes + 20: optional i64 row_column_page_size = 16384; } // this enum stands for different storage format in src_backends From 288bf5aea7dfa4cfbaf5a552d5148cc443867f39 Mon Sep 17 00:00:00 2001 From: lxr599 Date: Thu, 4 Jul 2024 19:31:56 +0800 Subject: [PATCH 03/23] [feature](Row store)allow to set row_column_page_size for tables --- .../java/org/apache/doris/common/util/PropertyAnalyzer.java | 2 +- .../main/java/org/apache/doris/datasource/InternalCatalog.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 61f0a695d7bb70..7c5178331354dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -984,7 +984,7 @@ public static long alignTo4K(long size) { // analyzeRowColumnPageSize will parse the row_column_page_size from properties public static long analyzeRowColumnPageSize(Map properties) throws AnalysisException { - long rowColumnPageSize = 16384; + long rowColumnPageSize = ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE; if (properties != null && properties.containsKey(PROPERTIES_ROW_COLUMN_PAGE_SIZE)) { String rowColumnPageSizeStr = properties.get(PROPERTIES_ROW_COLUMN_PAGE_SIZE); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index d825b2a6ee5c64..4b4383503191ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2381,7 +2381,7 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx olapTable.setCompressionType(compressionType); // get row_column_page_size - long rowColumnPageSize = 16384; + long rowColumnPageSize = PropertyAnalyzer.ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE; try { rowColumnPageSize = PropertyAnalyzer.analyzeRowColumnPageSize(properties); } catch (AnalysisException e) { From b5441452123c989970143627b7e2abd494345e88 Mon Sep 17 00:00:00 2001 From: lxr599 Date: Thu, 4 Jul 2024 19:39:04 +0800 Subject: [PATCH 04/23] format codes --- be/src/olap/tablet_schema.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index 5bbc1e79c7ce35..6885e8937d0fb9 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -335,9 +335,7 @@ class TabletSchema { void set_version_col_idx(int32_t version_col_idx) { _version_col_idx = version_col_idx; } int32_t version_col_idx() const { return _version_col_idx; } segment_v2::CompressionTypePB compression_type() const { return _compression_type; } - void set_row_column_page_size(long page_size) { - _row_column_page_size = page_size; - } + void set_row_column_page_size(long page_size) { _row_column_page_size = page_size; } long row_column_page_size() const { return _row_column_page_size; } const std::vector& indexes() const { return _indexes; } From 2339bd1e87496b7f433c40c8cd356fffbbb0b068 Mon Sep 17 00:00:00 2001 From: lxr599 Date: Tue, 2 Jul 2024 16:44:35 +0800 Subject: [PATCH 05/23] change default value of row_column_page_size to 16KB --- be/src/common/config.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 5ce525022f3133..d93b095a59365b 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1058,8 +1058,8 @@ DEFINE_mInt64(max_tablet_io_errors, "-1"); DEFINE_Int32(tablet_path_check_interval_seconds, "-1"); DEFINE_mInt32(tablet_path_check_batch_size, "1000"); -// Page size of row column, default 4KB -DEFINE_mInt64(row_column_page_size, "4096"); +// Page size of row column, default 16KB +DEFINE_mInt64(row_column_page_size, "16384"); // it must be larger than or equal to 5MB DEFINE_mInt64(s3_write_buffer_size, "5242880"); // Log interval when doing s3 upload task From 9a211b1be48f7c4f013f587a238cab2be66bff45 Mon Sep 17 00:00:00 2001 From: lxr599 Date: Thu, 4 Jul 2024 19:14:17 +0800 Subject: [PATCH 06/23] allow to set row_column_page_size for tables --- be/src/common/config.cpp | 2 -- be/src/common/config.h | 2 -- be/src/olap/rowset/segment_v2/options.h | 2 ++ .../olap/rowset/segment_v2/segment_writer.cpp | 16 +++++----- .../segment_v2/vertical_segment_writer.cpp | 2 +- be/src/olap/tablet_meta.cpp | 3 ++ be/src/olap/tablet_schema.cpp | 4 +++ be/src/olap/tablet_schema.h | 6 ++++ .../apache/doris/alter/CloudRollupJobV2.java | 3 +- .../doris/alter/CloudSchemaChangeJobV2.java | 3 +- .../org/apache/doris/alter/RollupJobV2.java | 3 +- .../apache/doris/alter/SchemaChangeJobV2.java | 3 +- .../org/apache/doris/backup/RestoreJob.java | 3 +- .../org/apache/doris/catalog/OlapTable.java | 14 +++++++++ .../apache/doris/catalog/TableProperty.java | 13 +++++++++ .../datasource/CloudInternalCatalog.java | 10 +++++-- .../doris/common/util/PropertyAnalyzer.java | 29 +++++++++++++++++++ .../doris/datasource/InternalCatalog.java | 27 +++++++++++++---- .../apache/doris/master/ReportHandler.java | 3 +- .../apache/doris/task/CreateReplicaTask.java | 6 +++- .../org/apache/doris/task/AgentTaskTest.java | 3 +- gensrc/proto/olap_file.proto | 2 ++ gensrc/thrift/AgentService.thrift | 1 + 23 files changed, 130 insertions(+), 30 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index d93b095a59365b..98b8f60456376d 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1058,8 +1058,6 @@ DEFINE_mInt64(max_tablet_io_errors, "-1"); DEFINE_Int32(tablet_path_check_interval_seconds, "-1"); DEFINE_mInt32(tablet_path_check_batch_size, "1000"); -// Page size of row column, default 16KB -DEFINE_mInt64(row_column_page_size, "16384"); // it must be larger than or equal to 5MB DEFINE_mInt64(s3_write_buffer_size, "5242880"); // Log interval when doing s3 upload task diff --git a/be/src/common/config.h b/be/src/common/config.h index a3b94705dea5a4..53f86601c6d682 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1102,8 +1102,6 @@ DECLARE_mInt64(max_tablet_io_errors); DECLARE_Int32(tablet_path_check_interval_seconds); DECLARE_mInt32(tablet_path_check_batch_size); -// Page size of row column, default 4KB -DECLARE_mInt64(row_column_page_size); // it must be larger than or equal to 5MB DECLARE_mInt64(s3_write_buffer_size); // Log interval when doing s3 upload task diff --git a/be/src/olap/rowset/segment_v2/options.h b/be/src/olap/rowset/segment_v2/options.h index 19041f4c51d1db..5fd5dba9d24c01 100644 --- a/be/src/olap/rowset/segment_v2/options.h +++ b/be/src/olap/rowset/segment_v2/options.h @@ -24,6 +24,8 @@ namespace segment_v2 { static constexpr size_t DEFAULT_PAGE_SIZE = 1024 * 1024; // default size: 1M +constexpr long ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE = 16384; // default row column page size: 16KB + struct PageBuilderOptions { size_t data_page_size = DEFAULT_PAGE_SIZE; diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index d22e1060dd370b..ac5ac4983994cb 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -256,14 +256,14 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co #undef CHECK_FIELD_TYPE - if (column.is_row_store_column()) { - // smaller page size for row store column - opts.data_page_size = config::row_column_page_size; - } - std::unique_ptr writer; - RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer, &writer)); - RETURN_IF_ERROR(writer->init()); - _column_writers.push_back(std::move(writer)); + if (column.is_row_store_column()) { + // smaller page size for row store column + opts.data_page_size = _tablet_schema->row_column_page_size(); + } + std::unique_ptr writer; + RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer, &writer)); + RETURN_IF_ERROR(writer->init()); + _column_writers.push_back(std::move(writer)); _olap_data_convertor->add_column_data_convertor(column); return Status::OK(); diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp index 5d2d6ac0769f14..59a33d003c3f07 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp @@ -221,7 +221,7 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo if (column.is_row_store_column()) { // smaller page size for row store column - opts.data_page_size = config::row_column_page_size; + opts.data_page_size = _tablet_schema->row_column_page_size(); } std::unique_ptr writer; RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer, &writer)); diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index cced41a86eee8b..ba6404f26e6fdf 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -317,6 +317,9 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id if (tablet_schema.__isset.store_row_column) { schema->set_store_row_column(tablet_schema.store_row_column); } + if (tablet_schema.__isset.row_column_page_size) { + schema->set_row_column_page_size(tablet_schema.row_column_page_size); + } if (tablet_schema.__isset.skip_write_index_on_load) { schema->set_skip_write_index_on_load(tablet_schema.skip_write_index_on_load); } diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index ec887f14a91377..7e1db892eaa84e 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -990,6 +990,7 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac _sort_type = schema.sort_type(); _sort_col_num = schema.sort_col_num(); _compression_type = schema.compression_type(); + _row_column_page_size = schema.row_column_page_size(); _schema_version = schema.schema_version(); // Default to V1 inverted index storage format for backward compatibility if not specified in schema. if (!schema.has_inverted_index_storage_format()) { @@ -1050,6 +1051,7 @@ void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version _skip_write_index_on_load = ori_tablet_schema.skip_write_index_on_load(); _sort_type = ori_tablet_schema.sort_type(); _sort_col_num = ori_tablet_schema.sort_col_num(); + _row_column_page_size = ori_tablet_schema.row_column_page_size(); // copy from table_schema_param _schema_version = version; @@ -1203,6 +1205,7 @@ void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const { tablet_schema_pb->set_sort_col_num(_sort_col_num); tablet_schema_pb->set_schema_version(_schema_version); tablet_schema_pb->set_compression_type(_compression_type); + tablet_schema_pb->set_row_column_page_size(_row_column_page_size); tablet_schema_pb->set_version_col_idx(_version_col_idx); tablet_schema_pb->set_inverted_index_storage_format(_inverted_index_storage_format); tablet_schema_pb->mutable_row_store_column_unique_ids()->Assign( @@ -1522,6 +1525,7 @@ bool operator==(const TabletSchema& a, const TabletSchema& b) { if (a._disable_auto_compaction != b._disable_auto_compaction) return false; if (a._enable_single_replica_compaction != b._enable_single_replica_compaction) return false; if (a._store_row_column != b._store_row_column) return false; + if (a._row_column_page_size != b._row_column_page_size) return false; if (a._skip_write_index_on_load != b._skip_write_index_on_load) return false; return true; } diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index 3a78f2e4748a97..8f90d42c3ff0de 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -36,6 +36,7 @@ #include "common/status.h" #include "gutil/stringprintf.h" #include "olap/olap_common.h" +#include "olap/rowset/segment_v2/options.h" #include "runtime/define_primitive_type.h" #include "runtime/descriptors.h" #include "util/string_util.h" @@ -359,6 +360,10 @@ class TabletSchema { void set_version_col_idx(int32_t version_col_idx) { _version_col_idx = version_col_idx; } int32_t version_col_idx() const { return _version_col_idx; } segment_v2::CompressionTypePB compression_type() const { return _compression_type; } + void set_row_column_page_size(long page_size) { + _row_column_page_size = page_size; + } + long row_column_page_size() const { return _row_column_page_size; } const std::vector& indexes() const { return _indexes; } bool has_inverted_index() const { @@ -508,6 +513,7 @@ class TabletSchema { size_t _num_rows_per_row_block = 0; CompressKind _compress_kind = COMPRESS_NONE; segment_v2::CompressionTypePB _compression_type = segment_v2::CompressionTypePB::LZ4F; + long _row_column_page_size = segment_v2::ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE; size_t _next_column_unique_id = 0; std::string _auto_increment_column; diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java index 688c2cd17cd0e0..2428f35305b46c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java @@ -209,7 +209,8 @@ private void createRollupReplicaForPartition(OlapTable tbl) throws Exception { tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), tbl.getTimeSeriesCompactionLevelThreshold(), tbl.disableAutoCompaction(), - tbl.getRowStoreColumnsUniqueIds(rowStoreColumns)); + tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), + tbl.rowColumnPageSize()); requestBuilder.addTabletMetas(builder); } // end for rollupTablets ((CloudInternalCatalog) Env.getCurrentInternalCatalog()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java index 3968f2d274f41e..83001f8b14cf86 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java @@ -227,7 +227,8 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), tbl.getTimeSeriesCompactionLevelThreshold(), tbl.disableAutoCompaction(), - tbl.getRowStoreColumnsUniqueIds(rowStoreColumns)); + tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), + tbl.rowColumnPageSize()); requestBuilder.addTabletMetas(builder); } // end for rollupTablets ((CloudInternalCatalog) Env.getCurrentInternalCatalog()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java index ef7d7de63070c5..5d911d24fcdbb2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -265,7 +265,8 @@ protected void createRollupReplica() throws AlterCancelException { tbl.storeRowColumn(), binlogConfig, tbl.getRowStoreColumnsUniqueIds(tbl.getTableProperty().getCopiedRowStoreColumns()), - objectPool); + objectPool, + tbl.rowColumnPageSize()); createReplicaTask.setBaseTablet(tabletIdMap.get(rollupTabletId), baseSchemaHash); if (this.storageFormat != null) { createReplicaTask.setStorageFormat(this.storageFormat); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index a0c7058f535f05..74ffb00fac8e13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -293,7 +293,8 @@ protected void createShadowIndexReplica() throws AlterCancelException { tbl.storeRowColumn(), binlogConfig, tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), - objectPool); + objectPool, + tbl.rowColumnPageSize()); createReplicaTask.setBaseTablet(partitionIndexTabletMap.get(partitionId, shadowIdxId) .get(shadowTabletId), originSchemaHash); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 098c473d32e22e..a3b008eaf91813 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -1126,7 +1126,8 @@ private void createReplicas(Database db, AgentBatchTask batchTask, OlapTable loc localTbl.storeRowColumn(), binlogConfig, localTbl.getRowStoreColumnsUniqueIds(rowStoreColumns), - objectPool); + objectPool, + localTbl.rowColumnPageSize()); task.setInvertedIndexFileStorageFormat(localTbl.getInvertedIndexFileStorageFormat()); task.setInRestoreMode(true); batchTask.addTask(task); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 471cbdf0ec642a..c6e17d57dc49e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -2505,6 +2505,20 @@ public void setCompressionType(TCompressionType compressionType) { tableProperty.buildCompressionType(); } + public void setRowColumnPageSize(long pageSize) { + TableProperty tableProperty = getOrCreatTableProperty(); + tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_ROW_COLUMN_PAGE_SIZE, + Long.valueOf(pageSize).toString()); + tableProperty.buildRowColumnPageSize(); + } + + public long rowColumnPageSize() { + if (tableProperty != null) { + return tableProperty.rowColumnPageSize(); + } + return PropertyAnalyzer.ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE; + } + public void setStorageFormat(TStorageFormat storageFormat) { TableProperty tableProperty = getOrCreatTableProperty(); tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT, storageFormat.name()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java index 59c483020fe35c..fe07a278215cec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java @@ -102,6 +102,8 @@ public class TableProperty implements Writable, GsonPostProcessable { private boolean skipWriteIndexOnLoad = false; + private long rowColumnPageSize = PropertyAnalyzer.ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE; + private String compactionPolicy = PropertyAnalyzer.SIZE_BASED_COMPACTION_POLICY; private long timeSeriesCompactionGoalSizeMbytes @@ -267,6 +269,17 @@ public boolean storeRowColumn() { return storeRowColumn; } + public TableProperty buildRowColumnPageSize() { + rowColumnPageSize = Long.parseLong( + properties.getOrDefault(PropertyAnalyzer.PROPERTIES_ROW_COLUMN_PAGE_SIZE, + Long.toString(PropertyAnalyzer.ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE))); + return this; + } + + public long rowColumnPageSize() { + return rowColumnPageSize; + } + public TableProperty buildSkipWriteIndexOnLoad() { skipWriteIndexOnLoad = Boolean.parseBoolean( properties.getOrDefault(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD, "false")); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index 3ebc9d1380899b..88600b654dddab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -100,7 +100,8 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa String storagePolicy, IdGeneratorBuffer idGeneratorBuffer, BinlogConfig binlogConfig, - boolean isStorageMediumSpecified, List clusterKeyIndexes) + boolean isStorageMediumSpecified, + List clusterKeyIndexes, long pageSize) throws DdlException { // create base index first. Preconditions.checkArgument(tbl.getBaseIndexId() != -1); @@ -169,7 +170,8 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), tbl.getTimeSeriesCompactionLevelThreshold(), tbl.disableAutoCompaction(), - tbl.getRowStoreColumnsUniqueIds(rowStoreColumns)); + tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), + tbl.rowColumnPageSize()); requestBuilder.addTabletMetas(builder); } if (!storageVaultIdSet && ((CloudEnv) Env.getCurrentEnv()).getEnableStorageVault()) { @@ -216,7 +218,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, Long timeSeriesCompactionGoalSizeMbytes, Long timeSeriesCompactionFileCountThreshold, Long timeSeriesCompactionTimeThresholdSeconds, Long timeSeriesCompactionEmptyRowsetsThreshold, Long timeSeriesCompactionLevelThreshold, boolean disableAutoCompaction, - List rowStoreColumnUniqueIds) throws DdlException { + List rowStoreColumnUniqueIds, long pageSize) throws DdlException { OlapFile.TabletMetaCloudPB.Builder builder = OlapFile.TabletMetaCloudPB.newBuilder(); builder.setTableId(tableId); builder.setIndexId(indexId); @@ -334,6 +336,8 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, } schemaBuilder.setDisableAutoCompaction(disableAutoCompaction); + schemaBuilder.setRowColumnPageSize(pageSize); + OlapFile.TabletSchemaCloudPB schema = schemaBuilder.build(); builder.setSchema(schema); // rowset diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 29ed40676b3903..bab5aae5612dcb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -95,6 +95,10 @@ public class PropertyAnalyzer { public static final String PROPERTIES_TIMEOUT = "timeout"; public static final String PROPERTIES_COMPRESSION = "compression"; + // row column page size, default 16KB + public static final String PROPERTIES_ROW_COLUMN_PAGE_SIZE = "row_column_page_size"; + public static final long ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE = 16384; + public static final String PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE = "light_schema_change"; public static final String PROPERTIES_DISTRIBUTION_TYPE = "distribution_type"; @@ -1013,6 +1017,31 @@ public static TCompressionType analyzeCompressionType(Map proper } } + public static long alignTo4K(long size) { + return (size + 4095) & ~4095; + } + + // analyzeRowColumnPageSize will parse the row_column_page_size from properties + public static long analyzeRowColumnPageSize(Map properties) throws AnalysisException { + long rowColumnPageSize = 16384; + if (properties != null && properties.containsKey(PROPERTIES_ROW_COLUMN_PAGE_SIZE)) { + String rowColumnPageSizeStr = properties.get(PROPERTIES_ROW_COLUMN_PAGE_SIZE); + try { + rowColumnPageSize = alignTo4K(Long.parseLong(rowColumnPageSizeStr)); + } catch (NumberFormatException e) { + throw new AnalysisException("Invalid row column page size: " + rowColumnPageSizeStr); + } + + if (rowColumnPageSize <= 0) { + throw new AnalysisException("Row column page size should larger than 0."); + } + + properties.remove(PROPERTIES_ROW_COLUMN_PAGE_SIZE); + } + + return rowColumnPageSize; + } + // analyzeStorageFormat will parse the storage format from properties // sql: alter table tablet_name set ("storage_format" = "v2") // Use this sql to convert all tablets(base and rollup index) to a new format segment diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index fab3185140dc1a..49103e2dea369a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1559,6 +1559,10 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa properties.put(PropertyAnalyzer.PROPERTIES_STORE_ROW_COLUMN, olapTable.storeRowColumn().toString()); } + if (!properties.containsKey(PropertyAnalyzer.PROPERTIES_ROW_COLUMN_PAGE_SIZE)) { + properties.put(PropertyAnalyzer.PROPERTIES_ROW_COLUMN_PAGE_SIZE, + Long.toString(olapTable.rowColumnPageSize())); + } if (!properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD)) { properties.put(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD, olapTable.skipWriteIndexOnLoad().toString()); @@ -1692,7 +1696,7 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa singlePartitionDesc.isInMemory(), singlePartitionDesc.getTabletType(), storagePolicy, idGeneratorBuffer, - binlogConfig, dataProperty.isStorageMediumSpecified(), null); + binlogConfig, dataProperty.isStorageMediumSpecified(), null, olapTable.rowColumnPageSize()); // TODO cluster key ids // check again @@ -1981,7 +1985,8 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa String storagePolicy, IdGeneratorBuffer idGeneratorBuffer, BinlogConfig binlogConfig, - boolean isStorageMediumSpecified, List clusterKeyIndexes) + boolean isStorageMediumSpecified, + List clusterKeyIndexes, long rowColumnPageSize) throws DdlException { // create base index first. Preconditions.checkArgument(tbl.getBaseIndexId() != -1); @@ -2065,7 +2070,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa tbl.getTimeSeriesCompactionLevelThreshold(), tbl.storeRowColumn(), binlogConfig, tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), - objectPool); + objectPool, rowColumnPageSize); task.setStorageFormat(tbl.getStorageFormat()); task.setInvertedIndexFileStorageFormat(tbl.getInvertedIndexFileStorageFormat()); @@ -2440,6 +2445,16 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx } olapTable.setCompressionType(compressionType); + // get row_column_page_size + long rowColumnPageSize = 16384; + try { + rowColumnPageSize = PropertyAnalyzer.analyzeRowColumnPageSize(properties); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + + olapTable.setRowColumnPageSize(rowColumnPageSize); + // check data sort properties int keyColumnSize = CollectionUtils.isEmpty(keysDesc.getClusterKeysColumnIds()) ? keysDesc.keysColumnSize() : keysDesc.getClusterKeysColumnIds().size(); @@ -2839,7 +2854,7 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx idGeneratorBuffer, binlogConfigForTask, partitionInfo.getDataProperty(partitionId).isStorageMediumSpecified(), - keysDesc.getClusterKeysColumnIds()); + keysDesc.getClusterKeysColumnIds(), olapTable.rowColumnPageSize()); afterCreatePartitions(db.getId(), olapTable.getId(), null, olapTable.getIndexIdList(), true); olapTable.addPartition(partition); @@ -2922,7 +2937,7 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx partionStoragePolicy, idGeneratorBuffer, binlogConfigForTask, dataProperty.isStorageMediumSpecified(), - keysDesc.getClusterKeysColumnIds()); + keysDesc.getClusterKeysColumnIds(), olapTable.rowColumnPageSize()); olapTable.addPartition(partition); olapTable.getPartitionInfo().getDataProperty(partition.getId()) .setStoragePolicy(partionStoragePolicy); @@ -3376,7 +3391,7 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti olapTable.getPartitionInfo().getDataProperty(oldPartitionId).getStoragePolicy(), idGeneratorBuffer, binlogConfig, copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId).isStorageMediumSpecified(), - clusterKeyIdxes); + clusterKeyIdxes, olapTable.rowColumnPageSize()); newPartitions.add(newPartition); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java index d36144af46cb1c..73e50706d44b4a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java @@ -886,7 +886,8 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta olapTable.storeRowColumn(), binlogConfig, olapTable.getRowStoreColumnsUniqueIds(rowStoreColumns), - objectPool); + objectPool, + olapTable.rowColumnPageSize()); createReplicaTask.setIsRecoverTask(true); createReplicaTask.setInvertedIndexFileStorageFormat(olapTable diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java index 022a8be67a970c..67583071a12118 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -65,6 +65,7 @@ public class CreateReplicaTask extends AgentTask { private TStorageType storageType; private TStorageMedium storageMedium; private TCompressionType compressionType; + private long rowColumnPageSize; private List columns; @@ -151,7 +152,8 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition boolean storeRowColumn, BinlogConfig binlogConfig, List rowStoreColumnUniqueIds, - Map objectPool) { + Map objectPool, + long rowColumnPageSize) { super(null, backendId, TTaskType.CREATE, dbId, tableId, partitionId, indexId, tabletId); this.replicaId = replicaId; @@ -197,6 +199,7 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition this.storeRowColumn = storeRowColumn; this.binlogConfig = binlogConfig; this.objectPool = objectPool; + this.rowColumnPageSize = rowColumnPageSize; } public void setIsRecoverTask(boolean isRecoverTask) { @@ -339,6 +342,7 @@ public TCreateTabletReq toThrift() { tSchema.setEnableSingleReplicaCompaction(enableSingleReplicaCompaction); tSchema.setSkipWriteIndexOnLoad(skipWriteIndexOnLoad); tSchema.setStoreRowColumn(storeRowColumn); + tSchema.setRowColumnPageSize(rowColumnPageSize); createTabletReq.setTabletSchema(tSchema); createTabletReq.setVersion(version); diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java index f2295dcd5bc8ee..62b6ad2517152b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java @@ -73,6 +73,7 @@ public class AgentTaskTest { private long version = 1L; private TStorageType storageType = TStorageType.COLUMN; + private long rowColumnPageSize = 16384L; private List columns; private MarkedCountDownLatch latch = new MarkedCountDownLatch(3); @@ -107,7 +108,7 @@ public void setUp() throws AnalysisException { createReplicaTask = new CreateReplicaTask(backendId1, dbId, tableId, partitionId, indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType, TStorageMedium.SSD, columns, null, 0, latch, null, false, TTabletType.TABLET_TYPE_DISK, null, - TCompressionType.LZ4F, false, "", false, false, false, "", 0, 0, 0, 0, 0, false, null, null, objectPool); + TCompressionType.LZ4F, false, "", false, false, false, "", 0, 0, 0, 0, 0, false, null, null, objectPool, rowColumnPageSize); // drop dropTask = new DropReplicaTask(backendId1, tabletId1, replicaId1, schemaHash1, false); diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 1f8f88801be40f..d017baa917514c 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -383,6 +383,7 @@ message TabletSchemaPB { optional InvertedIndexStorageFormatPB inverted_index_storage_format = 25 [default=V1]; // column unique ids for row store columns repeated int32 row_store_column_unique_ids = 26; + optional int64 row_column_page_size = 27 [default=16384]; } message TabletSchemaCloudPB { @@ -411,6 +412,7 @@ message TabletSchemaCloudPB { optional InvertedIndexStorageFormatPB inverted_index_storage_format = 25 [default=V1]; // column unique ids for row store columns repeated int32 row_store_column_unique_ids = 26; + optional int64 row_column_page_size = 27 [default=16384]; optional bool is_dynamic_schema = 100 [default=false]; } diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index a03cb9df99b69f..1cff0f77fce1f2 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -47,6 +47,7 @@ struct TTabletSchema { 19: optional list cluster_key_idxes // col unique id for row store column 20: optional list row_store_col_cids + 21: optional i64 row_column_page_size = 16384; } // this enum stands for different storage format in src_backends From 9552bd59d0eba9bf35537815dffe4e24607a81a9 Mon Sep 17 00:00:00 2001 From: lxr599 Date: Thu, 4 Jul 2024 19:31:56 +0800 Subject: [PATCH 07/23] [feature](Row store)allow to set row_column_page_size for tables --- .../java/org/apache/doris/common/util/PropertyAnalyzer.java | 2 +- .../main/java/org/apache/doris/datasource/InternalCatalog.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index bab5aae5612dcb..0e5498395690b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -1023,7 +1023,7 @@ public static long alignTo4K(long size) { // analyzeRowColumnPageSize will parse the row_column_page_size from properties public static long analyzeRowColumnPageSize(Map properties) throws AnalysisException { - long rowColumnPageSize = 16384; + long rowColumnPageSize = ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE; if (properties != null && properties.containsKey(PROPERTIES_ROW_COLUMN_PAGE_SIZE)) { String rowColumnPageSizeStr = properties.get(PROPERTIES_ROW_COLUMN_PAGE_SIZE); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 49103e2dea369a..bb3a699e67360f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2446,7 +2446,7 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx olapTable.setCompressionType(compressionType); // get row_column_page_size - long rowColumnPageSize = 16384; + long rowColumnPageSize = PropertyAnalyzer.ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE; try { rowColumnPageSize = PropertyAnalyzer.analyzeRowColumnPageSize(properties); } catch (AnalysisException e) { From 8c4c0f72ab478a63d6227a68b877b9ed92ff8bef Mon Sep 17 00:00:00 2001 From: lxr599 Date: Thu, 4 Jul 2024 19:39:04 +0800 Subject: [PATCH 08/23] format codes --- be/src/olap/tablet_schema.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index 8f90d42c3ff0de..8dd3328b492274 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -360,9 +360,7 @@ class TabletSchema { void set_version_col_idx(int32_t version_col_idx) { _version_col_idx = version_col_idx; } int32_t version_col_idx() const { return _version_col_idx; } segment_v2::CompressionTypePB compression_type() const { return _compression_type; } - void set_row_column_page_size(long page_size) { - _row_column_page_size = page_size; - } + void set_row_column_page_size(long page_size) { _row_column_page_size = page_size; } long row_column_page_size() const { return _row_column_page_size; } const std::vector& indexes() const { return _indexes; } From 3f271b09491b18654680cecdf806378a80fa3ee3 Mon Sep 17 00:00:00 2001 From: lxr599 Date: Fri, 5 Jul 2024 13:43:38 +0800 Subject: [PATCH 09/23] format codes --- be/src/olap/rowset/segment_v2/segment_writer.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index ac5ac4983994cb..836dc050b0fa8a 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -256,14 +256,14 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co #undef CHECK_FIELD_TYPE - if (column.is_row_store_column()) { - // smaller page size for row store column - opts.data_page_size = _tablet_schema->row_column_page_size(); - } - std::unique_ptr writer; - RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer, &writer)); - RETURN_IF_ERROR(writer->init()); - _column_writers.push_back(std::move(writer)); + if (column.is_row_store_column()) { + // smaller page size for row store column + opts.data_page_size = _tablet_schema->row_column_page_size(); + } + std::unique_ptr writer; + RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer, &writer)); + RETURN_IF_ERROR(writer->init()); + _column_writers.push_back(std::move(writer)); _olap_data_convertor->add_column_data_convertor(column); return Status::OK(); From 843d26dad3a87773d1b2c83b87e15773e4b92129 Mon Sep 17 00:00:00 2001 From: lxr599 Date: Fri, 5 Jul 2024 15:31:43 +0800 Subject: [PATCH 10/23] format fe codes --- fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 52f749512323eb..21af9703497471 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3463,7 +3463,7 @@ private static void addOlapTablePropertyInfo(OlapTable olapTable, StringBuilder sb.append(olapTable.getCompressionType()).append("\""); } - // row column page size + // row column page size sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_ROW_COLUMN_PAGE_SIZE).append("\" = \""); sb.append(olapTable.rowColumnPageSize()).append("\""); From fe22103409288765c92a55dbd4acbdf8009f2bcc Mon Sep 17 00:00:00 2001 From: lxr599 Date: Tue, 9 Jul 2024 01:19:04 +0800 Subject: [PATCH 11/23] display row_column_page_size only when row store enabled --- .../java/org/apache/doris/catalog/Env.java | 8 +++---- .../analysis/CreateTableAsSelectStmtTest.java | 21 ------------------- .../system/test_query_sys_rowsets.out | 10 ++------- .../query_p0/system/test_query_sys_tables.out | 7 +++++++ 4 files changed, 13 insertions(+), 33 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 2f87c7fc05d6b7..2141bf795cbf37 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3463,10 +3463,6 @@ private static void addOlapTablePropertyInfo(OlapTable olapTable, StringBuilder sb.append(olapTable.getCompressionType()).append("\""); } - // row column page size - sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_ROW_COLUMN_PAGE_SIZE).append("\" = \""); - sb.append(olapTable.rowColumnPageSize()).append("\""); - // estimate_partition_size if (!olapTable.getEstimatePartitionSize().equals("")) { sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_ESTIMATE_PARTITION_SIZE).append("\" = \""); @@ -3514,6 +3510,10 @@ private static void addOlapTablePropertyInfo(OlapTable olapTable, StringBuilder sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_STORE_ROW_COLUMN).append("\" = \""); sb.append(olapTable.storeRowColumn()).append("\""); } + + // row column page size + sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_ROW_COLUMN_PAGE_SIZE).append("\" = \""); + sb.append(olapTable.rowColumnPageSize()).append("\""); } // skip inverted index on load diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index dc09658dc5c947..5e6c53b18ad2d9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -96,7 +96,6 @@ public void testDecimal() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -122,7 +121,6 @@ public void testDecimal() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -145,7 +143,6 @@ public void testDecimal() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -185,7 +182,6 @@ public void testVarchar() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -214,7 +210,6 @@ public void testFunction() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -245,7 +240,6 @@ public void testFunction() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -273,7 +267,6 @@ public void testAlias() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -297,7 +290,6 @@ public void testAlias() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -328,7 +320,6 @@ public void testJoin() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -356,7 +347,6 @@ public void testJoin() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -388,7 +378,6 @@ public void testName() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -417,7 +406,6 @@ public void testUnion() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -445,7 +433,6 @@ public void testCte() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -469,7 +456,6 @@ public void testCte() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -500,7 +486,6 @@ public void testPartition() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -530,7 +515,6 @@ public void testDefaultTimestamp() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -559,7 +543,6 @@ public void testAggValue() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -589,7 +572,6 @@ public void testUseKeyType() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"enable_unique_key_merge_on_write\" = \"true\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" @@ -645,7 +627,6 @@ public void testQuerySchema() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -667,7 +648,6 @@ public void testQuerySchema() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -703,7 +683,6 @@ public void testVarcharLength() throws Exception { + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + "\"inverted_index_storage_format\" = \"V2\",\n" - + "\"row_column_page_size\" = \"16384\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" diff --git a/regression-test/data/query_p0/system/test_query_sys_rowsets.out b/regression-test/data/query_p0/system/test_query_sys_rowsets.out index 2f91c9028dc330..756d92165aaa75 100644 --- a/regression-test/data/query_p0/system/test_query_sys_rowsets.out +++ b/regression-test/data/query_p0/system/test_query_sys_rowsets.out @@ -10,9 +10,8 @@ START_VERSION BIGINT Yes false \N END_VERSION BIGINT Yes false \N INDEX_DISK_SIZE BIGINT Yes false \N DATA_DISK_SIZE BIGINT Yes false \N -CREATION_TIME DATETIME Yes false \N -NEWEST_WRITE_TIMESTAMP DATETIME Yes false \N -SCHEMA_VERSION INT Yes false \N +CREATION_TIME BIGINT Yes false \N +NEWEST_WRITE_TIMESTAMP BIGINT Yes false \N -- !rowsets1 -- 0 1 @@ -28,9 +27,4 @@ SCHEMA_VERSION INT Yes false \N 4 4 -- !rowsets4 -- -0 1 -2 2 -3 3 -4 4 -5 5 diff --git a/regression-test/data/query_p0/system/test_query_sys_tables.out b/regression-test/data/query_p0/system/test_query_sys_tables.out index 74574ae818ba62..a7b57bb79d185b 100644 --- a/regression-test/data/query_p0/system/test_query_sys_tables.out +++ b/regression-test/data/query_p0/system/test_query_sys_tables.out @@ -183,6 +183,13 @@ test_view -- !sql -- -- !sql -- +TEST_PLSQL_SHOW_PROC1 0 plsql_show_procedure_db TEST_PLSQL_SHOW_PROC1 PROCEDURE CREATE OR REPLACE PROCEDURE test_plsql_show_proc1() BEGIN DECLARE a int = 1; print a; END NULL SQL NULL DEFINER 2024-06-05T09:55:29 2024-06-05T09:55:29 root +TEST_PLSQL_ROUTINE1 0 plsql_routine TEST_PLSQL_ROUTINE1 PROCEDURE CREATE OR REPLACE PROCEDURE test_plsql_routine1() BEGIN DECLARE a int = 1; print a; END NULL SQL NULL DEFINER 2024-06-05T09:55:29 2024-06-05T09:55:29 root +TEST_PLSQL_ROUTINE2 0 plsql_routine TEST_PLSQL_ROUTINE2 PROCEDURE CREATE OR REPLACE PROCEDURE test_plsql_routine2() BEGIN DECLARE a int = 1; print a; END NULL SQL NULL DEFINER 2024-06-05T09:55:29 2024-06-05T09:55:29 root +TEST_PLSQL_ROUTINE3 0 plsql_routine TEST_PLSQL_ROUTINE3 PROCEDURE CREATE OR REPLACE PROCEDURE test_plsql_routine3() BEGIN DECLARE a int = 1; print a; END NULL SQL NULL DEFINER 2024-06-05T09:55:29 2024-06-05T09:55:29 root +PLSQL_VARIABLE1 0 regression_test_plsql_p0 PLSQL_VARIABLE1 PROCEDURE CREATE OR REPLACE PROCEDURE plsql_variable1()\n BEGIN\n DECLARE a STRING;\n a:="hello world!";\n PRINT a;\n PRINT upper(a);\n\n DECLARE b = 10;\n PRINT b;\n b = length(a);\n PRINT b;\n\n DECLARE c = a;\n PRINT c;\n c:=b;\n PRINT c;\n c = "hello kudo!"\n PRINT c;\n\n DECLARE d STRING;\n PRINT d;\n d = NOW();\n PRINT d;\n END NULL SQL NULL DEFINER 2024-06-05T09:55:29 2024-06-05T09:55:29 root +PROCEDURE_INSERT 0 regression_test_plsql_p0 PROCEDURE_INSERT PROCEDURE CREATE OR REPLACE PROCEDURE procedure_insert(IN id int, IN name STRING)\n BEGIN\n INSERT INTO plsql_tbl VALUES(id, name);\n END NULL SQL NULL DEFINER 2024-06-05T09:55:29 2024-06-05T09:55:29 root +PLSQL_VARIABLE_INSERT 0 regression_test_plsql_p0 PLSQL_VARIABLE_INSERT PROCEDURE CREATE OR REPLACE PROCEDURE plsql_variable_insert(IN id int, IN name STRING)\n BEGIN\n INSERT INTO plsql_variable VALUES(id, name);\n END NULL SQL NULL DEFINER 2024-06-05T09:55:30 2024-06-05T09:55:30 root -- !sql -- From 9860a7fcb2ac6466ba91f6bbed25a9ffd1d02e88 Mon Sep 17 00:00:00 2001 From: lxr599 Date: Wed, 10 Jul 2024 11:11:48 +0800 Subject: [PATCH 12/23] add test case for property row_column_page_size --- .../system/test_query_sys_rowsets.out | 11 ++++-- .../query_p0/system/test_query_sys_tables.out | 13 ++++--- .../system/test_query_sys_tables.groovy | 34 +++++++++++++++++++ 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/regression-test/data/query_p0/system/test_query_sys_rowsets.out b/regression-test/data/query_p0/system/test_query_sys_rowsets.out index 756d92165aaa75..94bcf768125511 100644 --- a/regression-test/data/query_p0/system/test_query_sys_rowsets.out +++ b/regression-test/data/query_p0/system/test_query_sys_rowsets.out @@ -10,8 +10,9 @@ START_VERSION BIGINT Yes false \N END_VERSION BIGINT Yes false \N INDEX_DISK_SIZE BIGINT Yes false \N DATA_DISK_SIZE BIGINT Yes false \N -CREATION_TIME BIGINT Yes false \N -NEWEST_WRITE_TIMESTAMP BIGINT Yes false \N +CREATION_TIME DATETIME Yes false \N +NEWEST_WRITE_TIMESTAMP DATETIME Yes false \N +SCHEMA_VERSION INT Yes false \N -- !rowsets1 -- 0 1 @@ -27,4 +28,8 @@ NEWEST_WRITE_TIMESTAMP BIGINT Yes false \N 4 4 -- !rowsets4 -- - +0 1 +2 2 +3 3 +4 4 +5 5 diff --git a/regression-test/data/query_p0/system/test_query_sys_tables.out b/regression-test/data/query_p0/system/test_query_sys_tables.out index a7b57bb79d185b..41296b807bc031 100644 --- a/regression-test/data/query_p0/system/test_query_sys_tables.out +++ b/regression-test/data/query_p0/system/test_query_sys_tables.out @@ -173,6 +173,12 @@ wait_timeout 31000 -- !user_privileges -- 'original_test_sys_tables'@'%' SELECT NO +-- !desc_table_page_size -- +test_row_column_page_size1 CREATE TABLE `test_row_column_page_size1` (\n `aaa` VARCHAR(170) NOT NULL,\n `bbb` VARCHAR(20) NOT NULL,\n `ccc` INT NULL,\n `ddd` SMALLINT NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`aaa`)\nDISTRIBUTED BY HASH(`aaa`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"light_schema_change" = "true",\n"store_row_column" = "true",\n"row_column_page_size" = "16384",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); + +-- !desc_table_page_size -- +test_row_column_page_size2 CREATE TABLE `test_row_column_page_size2` (\n `aaa` VARCHAR(170) NOT NULL,\n `bbb` VARCHAR(20) NOT NULL,\n `ccc` INT NULL,\n `ddd` SMALLINT NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`aaa`)\nDISTRIBUTED BY HASH(`aaa`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"light_schema_change" = "true",\n"store_row_column" = "true",\n"row_column_page_size" = "8192",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); + -- !views -- test_view @@ -183,13 +189,6 @@ test_view -- !sql -- -- !sql -- -TEST_PLSQL_SHOW_PROC1 0 plsql_show_procedure_db TEST_PLSQL_SHOW_PROC1 PROCEDURE CREATE OR REPLACE PROCEDURE test_plsql_show_proc1() BEGIN DECLARE a int = 1; print a; END NULL SQL NULL DEFINER 2024-06-05T09:55:29 2024-06-05T09:55:29 root -TEST_PLSQL_ROUTINE1 0 plsql_routine TEST_PLSQL_ROUTINE1 PROCEDURE CREATE OR REPLACE PROCEDURE test_plsql_routine1() BEGIN DECLARE a int = 1; print a; END NULL SQL NULL DEFINER 2024-06-05T09:55:29 2024-06-05T09:55:29 root -TEST_PLSQL_ROUTINE2 0 plsql_routine TEST_PLSQL_ROUTINE2 PROCEDURE CREATE OR REPLACE PROCEDURE test_plsql_routine2() BEGIN DECLARE a int = 1; print a; END NULL SQL NULL DEFINER 2024-06-05T09:55:29 2024-06-05T09:55:29 root -TEST_PLSQL_ROUTINE3 0 plsql_routine TEST_PLSQL_ROUTINE3 PROCEDURE CREATE OR REPLACE PROCEDURE test_plsql_routine3() BEGIN DECLARE a int = 1; print a; END NULL SQL NULL DEFINER 2024-06-05T09:55:29 2024-06-05T09:55:29 root -PLSQL_VARIABLE1 0 regression_test_plsql_p0 PLSQL_VARIABLE1 PROCEDURE CREATE OR REPLACE PROCEDURE plsql_variable1()\n BEGIN\n DECLARE a STRING;\n a:="hello world!";\n PRINT a;\n PRINT upper(a);\n\n DECLARE b = 10;\n PRINT b;\n b = length(a);\n PRINT b;\n\n DECLARE c = a;\n PRINT c;\n c:=b;\n PRINT c;\n c = "hello kudo!"\n PRINT c;\n\n DECLARE d STRING;\n PRINT d;\n d = NOW();\n PRINT d;\n END NULL SQL NULL DEFINER 2024-06-05T09:55:29 2024-06-05T09:55:29 root -PROCEDURE_INSERT 0 regression_test_plsql_p0 PROCEDURE_INSERT PROCEDURE CREATE OR REPLACE PROCEDURE procedure_insert(IN id int, IN name STRING)\n BEGIN\n INSERT INTO plsql_tbl VALUES(id, name);\n END NULL SQL NULL DEFINER 2024-06-05T09:55:29 2024-06-05T09:55:29 root -PLSQL_VARIABLE_INSERT 0 regression_test_plsql_p0 PLSQL_VARIABLE_INSERT PROCEDURE CREATE OR REPLACE PROCEDURE plsql_variable_insert(IN id int, IN name STRING)\n BEGIN\n INSERT INTO plsql_variable VALUES(id, name);\n END NULL SQL NULL DEFINER 2024-06-05T09:55:30 2024-06-05T09:55:30 root -- !sql -- diff --git a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy index b8f14da041b634..8ee25cba7dba49 100644 --- a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy +++ b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy @@ -238,6 +238,40 @@ suite("test_query_sys_tables", "query,p0") { AS SELECT ccc as a FROM ${tbName1} """ + + // test row column page size + sql("use ${dbName1}") + sql """ + CREATE TABLE IF NOT EXISTS ${dbName1}.test_row_column_page_size1 ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "store_row_column" = "true" + ); + """ + qt_desc_table_page_size("show create table ${dbName1}.test_row_column_page_size1") + + sql """ + CREATE TABLE IF NOT EXISTS ${dbName1}.test_row_column_page_size2 ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "store_row_column" = "true", + "row_column_page_size" = "8190" + ); + """ + qt_desc_table_page_size("show create table ${dbName1}.test_row_column_page_size2") + sql("use information_schema") qt_views("select TABLE_NAME, VIEW_DEFINITION from views where TABLE_SCHEMA = '${dbName1}'") From 05374babb477cf4682f7f8aa979c90a819405059 Mon Sep 17 00:00:00 2001 From: lxr599 Date: Wed, 10 Jul 2024 16:47:55 +0800 Subject: [PATCH 13/23] add more test cases --- cloud/cmake/thirdparty.cmake | 1 - .../system/test_query_sys_rowsets.out | 3 +- .../query_p0/system/test_query_sys_tables.out | 6 ---- .../query_p0/system/test_table_options.out | 14 ++++---- .../system/test_query_sys_tables.groovy | 33 ------------------- .../query_p0/system/test_table_options.groovy | 31 +++++++++++++++++ 6 files changed, 41 insertions(+), 47 deletions(-) diff --git a/cloud/cmake/thirdparty.cmake b/cloud/cmake/thirdparty.cmake index 2b35cba5363d13..7f7b6111508062 100644 --- a/cloud/cmake/thirdparty.cmake +++ b/cloud/cmake/thirdparty.cmake @@ -59,7 +59,6 @@ add_thirdparty(protobuf) add_thirdparty(thrift) add_thirdparty(crypto) add_thirdparty(openssl LIBNAME "lib/libssl.a") -add_thirdparty(jemalloc LIBNAME "lib/libjemalloc_doris.a") if (USE_JEMALLOC) add_thirdparty(jemalloc LIBNAME "lib/libjemalloc_doris.a") else() diff --git a/regression-test/data/query_p0/system/test_query_sys_rowsets.out b/regression-test/data/query_p0/system/test_query_sys_rowsets.out index 94bcf768125511..2f91c9028dc330 100644 --- a/regression-test/data/query_p0/system/test_query_sys_rowsets.out +++ b/regression-test/data/query_p0/system/test_query_sys_rowsets.out @@ -12,7 +12,7 @@ INDEX_DISK_SIZE BIGINT Yes false \N DATA_DISK_SIZE BIGINT Yes false \N CREATION_TIME DATETIME Yes false \N NEWEST_WRITE_TIMESTAMP DATETIME Yes false \N -SCHEMA_VERSION INT Yes false \N +SCHEMA_VERSION INT Yes false \N -- !rowsets1 -- 0 1 @@ -33,3 +33,4 @@ SCHEMA_VERSION INT Yes false \N 3 3 4 4 5 5 + diff --git a/regression-test/data/query_p0/system/test_query_sys_tables.out b/regression-test/data/query_p0/system/test_query_sys_tables.out index 41296b807bc031..74574ae818ba62 100644 --- a/regression-test/data/query_p0/system/test_query_sys_tables.out +++ b/regression-test/data/query_p0/system/test_query_sys_tables.out @@ -173,12 +173,6 @@ wait_timeout 31000 -- !user_privileges -- 'original_test_sys_tables'@'%' SELECT NO --- !desc_table_page_size -- -test_row_column_page_size1 CREATE TABLE `test_row_column_page_size1` (\n `aaa` VARCHAR(170) NOT NULL,\n `bbb` VARCHAR(20) NOT NULL,\n `ccc` INT NULL,\n `ddd` SMALLINT NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`aaa`)\nDISTRIBUTED BY HASH(`aaa`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"light_schema_change" = "true",\n"store_row_column" = "true",\n"row_column_page_size" = "16384",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); - --- !desc_table_page_size -- -test_row_column_page_size2 CREATE TABLE `test_row_column_page_size2` (\n `aaa` VARCHAR(170) NOT NULL,\n `bbb` VARCHAR(20) NOT NULL,\n `ccc` INT NULL,\n `ddd` SMALLINT NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`aaa`)\nDISTRIBUTED BY HASH(`aaa`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"light_schema_change" = "true",\n"store_row_column" = "true",\n"row_column_page_size" = "8192",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); - -- !views -- test_view diff --git a/regression-test/data/query_p0/system/test_table_options.out b/regression-test/data/query_p0/system/test_table_options.out index 974c3bc2c7e817..31b4a049913541 100644 --- a/regression-test/data/query_p0/system/test_table_options.out +++ b/regression-test/data/query_p0/system/test_table_options.out @@ -1,9 +1,11 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select -- -aggregate_table internal test_table_options_db AGG user_id,date,city,age,sex user_id HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"5","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -duplicate_table internal test_table_options_db DUP timestamp,type,error_code type HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -listtable internal test_table_options_db AGG user_id,date,timestamp,city,age,sex user_id HASH 16 3 {"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -randomtable internal test_table_options_db DUP user_id,date,timestamp RANDOM RANDOM 16 1 {"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -rangetable internal test_table_options_db AGG user_id,date,timestamp,city,age,sex user_id HASH 8 3 {"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -unique_table internal test_table_options_db UNI user_id,username user_id HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"2","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"true","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +aggregate_table internal test_table_options_db AGG user_id,date,city,age,sex user_id HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"5","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","row_column_page_size":"16384","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +duplicate_table internal test_table_options_db DUP timestamp,type,error_code type HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","row_column_page_size":"16384","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +listtable internal test_table_options_db AGG user_id,date,timestamp,city,age,sex user_id HASH 16 3 {"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","row_column_page_size":"16384","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +randomtable internal test_table_options_db DUP user_id,date,timestamp RANDOM RANDOM 16 1 {"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","row_column_page_size":"16384","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +rangetable internal test_table_options_db AGG user_id,date,timestamp,city,age,sex user_id HASH 8 3 {"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","row_column_page_size":"16384","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +test_row_column_page_size1 internal test_table_options_db DUP aaa aaa HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"1","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"true","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","row_column_page_size":"16384","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +test_row_column_page_size2 internal test_table_options_db DUP aaa aaa HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"1","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"true","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","row_column_page_size":"8192","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +unique_table internal test_table_options_db UNI user_id,username user_id HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"2","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"true","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","row_column_page_size":"16384","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} diff --git a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy index 8ee25cba7dba49..72198b32eadfd2 100644 --- a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy +++ b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy @@ -239,39 +239,6 @@ suite("test_query_sys_tables", "query,p0") { SELECT ccc as a FROM ${tbName1} """ - // test row column page size - sql("use ${dbName1}") - sql """ - CREATE TABLE IF NOT EXISTS ${dbName1}.test_row_column_page_size1 ( - `aaa` varchar(170) NOT NULL COMMENT "", - `bbb` varchar(20) NOT NULL COMMENT "", - `ccc` INT NULL COMMENT "", - `ddd` SMALLINT NULL COMMENT "" - ) - DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 - PROPERTIES ( - "replication_num" = "1", - "store_row_column" = "true" - ); - """ - qt_desc_table_page_size("show create table ${dbName1}.test_row_column_page_size1") - - sql """ - CREATE TABLE IF NOT EXISTS ${dbName1}.test_row_column_page_size2 ( - `aaa` varchar(170) NOT NULL COMMENT "", - `bbb` varchar(20) NOT NULL COMMENT "", - `ccc` INT NULL COMMENT "", - `ddd` SMALLINT NULL COMMENT "" - ) - DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 - PROPERTIES ( - "replication_num" = "1", - "store_row_column" = "true", - "row_column_page_size" = "8190" - ); - """ - qt_desc_table_page_size("show create table ${dbName1}.test_row_column_page_size2") - sql("use information_schema") qt_views("select TABLE_NAME, VIEW_DEFINITION from views where TABLE_SCHEMA = '${dbName1}'") diff --git a/regression-test/suites/query_p0/system/test_table_options.groovy b/regression-test/suites/query_p0/system/test_table_options.groovy index e53f2ba64e9060..5e2afa13c2d116 100644 --- a/regression-test/suites/query_p0/system/test_table_options.groovy +++ b/regression-test/suites/query_p0/system/test_table_options.groovy @@ -153,6 +153,37 @@ suite("test_table_options") { "replication_allocation" = "tag.location.default: 1" ); """ + + // test row column page size + sql """ + CREATE TABLE IF NOT EXISTS test_row_column_page_size1 ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "store_row_column" = "true" + ); + """ + + sql """ + CREATE TABLE IF NOT EXISTS test_row_column_page_size2 ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "store_row_column" = "true", + "row_column_page_size" = "8190" + ); + """ + qt_select """select * from information_schema.table_options where table_schema=\"${dbName}\" order by TABLE_NAME; """ sql "drop database if exists ${dbName}" } From c32b788d667e328a7fc3ca575c8c3ce3aa8f41a0 Mon Sep 17 00:00:00 2001 From: lxr599 Date: Thu, 11 Jul 2024 22:37:22 +0800 Subject: [PATCH 14/23] change the property name --- .../doris/common/util/PropertyAnalyzer.java | 4 ++-- .../data/query_p0/system/test_table_options.out | 16 ++++++++-------- .../query_p0/system/test_table_options.groovy | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index b1b07c011670f0..5b2534ed156cfc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -95,8 +95,8 @@ public class PropertyAnalyzer { public static final String PROPERTIES_TIMEOUT = "timeout"; public static final String PROPERTIES_COMPRESSION = "compression"; - // row column page size, default 16KB - public static final String PROPERTIES_ROW_COLUMN_PAGE_SIZE = "row_column_page_size"; + // row store page size, default 16KB + public static final String PROPERTIES_ROW_COLUMN_PAGE_SIZE = "row_store_page_size"; public static final long ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE = 16384; public static final String PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE = "light_schema_change"; diff --git a/regression-test/data/query_p0/system/test_table_options.out b/regression-test/data/query_p0/system/test_table_options.out index 31b4a049913541..66856b811fb0b1 100644 --- a/regression-test/data/query_p0/system/test_table_options.out +++ b/regression-test/data/query_p0/system/test_table_options.out @@ -1,11 +1,11 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select -- -aggregate_table internal test_table_options_db AGG user_id,date,city,age,sex user_id HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"5","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","row_column_page_size":"16384","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -duplicate_table internal test_table_options_db DUP timestamp,type,error_code type HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","row_column_page_size":"16384","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -listtable internal test_table_options_db AGG user_id,date,timestamp,city,age,sex user_id HASH 16 3 {"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","row_column_page_size":"16384","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -randomtable internal test_table_options_db DUP user_id,date,timestamp RANDOM RANDOM 16 1 {"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","row_column_page_size":"16384","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -rangetable internal test_table_options_db AGG user_id,date,timestamp,city,age,sex user_id HASH 8 3 {"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","row_column_page_size":"16384","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -test_row_column_page_size1 internal test_table_options_db DUP aaa aaa HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"1","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"true","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","row_column_page_size":"16384","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -test_row_column_page_size2 internal test_table_options_db DUP aaa aaa HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"1","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"true","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","row_column_page_size":"8192","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -unique_table internal test_table_options_db UNI user_id,username user_id HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"2","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"true","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","row_column_page_size":"16384","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +aggregate_table internal test_table_options_db AGG user_id,date,city,age,sex user_id HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"5","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","row_store_page_size":"16384","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +duplicate_table internal test_table_options_db DUP timestamp,type,error_code type HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","row_store_page_size":"16384","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +listtable internal test_table_options_db AGG user_id,date,timestamp,city,age,sex user_id HASH 16 3 {"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","row_store_page_size":"16384","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +randomtable internal test_table_options_db DUP user_id,date,timestamp RANDOM RANDOM 16 1 {"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","row_store_page_size":"16384","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +rangetable internal test_table_options_db AGG user_id,date,timestamp,city,age,sex user_id HASH 8 3 {"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","row_store_page_size":"16384","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +test_row_column_page_size1 internal test_table_options_db DUP aaa aaa HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"1","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"true","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","row_store_page_size":"16384","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +test_row_column_page_size2 internal test_table_options_db DUP aaa aaa HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"1","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"true","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","row_store_page_size":"8192","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +unique_table internal test_table_options_db UNI user_id,username user_id HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"2","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"true","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","row_store_page_size":"16384","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} diff --git a/regression-test/suites/query_p0/system/test_table_options.groovy b/regression-test/suites/query_p0/system/test_table_options.groovy index 5e2afa13c2d116..dd898d4e208346 100644 --- a/regression-test/suites/query_p0/system/test_table_options.groovy +++ b/regression-test/suites/query_p0/system/test_table_options.groovy @@ -180,7 +180,7 @@ suite("test_table_options") { PROPERTIES ( "replication_num" = "1", "store_row_column" = "true", - "row_column_page_size" = "8190" + "row_store_page_size" = "8190" ); """ From 484160aca1041ccb5c80f58dc6e75d2685ee07b3 Mon Sep 17 00:00:00 2001 From: lxr599 Date: Fri, 12 Jul 2024 19:02:41 +0800 Subject: [PATCH 15/23] change names of vars related to row_store_page_size --- be/src/olap/rowset/segment_v2/options.h | 2 +- .../olap/rowset/segment_v2/segment_writer.cpp | 2 +- .../segment_v2/vertical_segment_writer.cpp | 2 +- be/src/olap/tablet_meta.cpp | 4 +-- be/src/olap/tablet_schema.cpp | 8 +++--- be/src/olap/tablet_schema.h | 6 ++--- .../apache/doris/alter/CloudRollupJobV2.java | 2 +- .../doris/alter/CloudSchemaChangeJobV2.java | 2 +- .../org/apache/doris/alter/RollupJobV2.java | 2 +- .../apache/doris/alter/SchemaChangeJobV2.java | 2 +- .../org/apache/doris/backup/RestoreJob.java | 2 +- .../java/org/apache/doris/catalog/Env.java | 6 ++--- .../org/apache/doris/catalog/OlapTable.java | 12 ++++----- .../apache/doris/catalog/TableProperty.java | 14 +++++----- .../datasource/CloudInternalCatalog.java | 4 +-- .../doris/common/util/PropertyAnalyzer.java | 26 +++++++++---------- .../doris/datasource/InternalCatalog.java | 26 +++++++++---------- .../apache/doris/master/ReportHandler.java | 2 +- .../apache/doris/task/CreateReplicaTask.java | 8 +++--- .../org/apache/doris/task/AgentTaskTest.java | 4 +-- gensrc/proto/olap_file.proto | 4 +-- gensrc/thrift/AgentService.thrift | 2 +- 22 files changed, 71 insertions(+), 71 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/options.h b/be/src/olap/rowset/segment_v2/options.h index 5fd5dba9d24c01..93ec03df452b6c 100644 --- a/be/src/olap/rowset/segment_v2/options.h +++ b/be/src/olap/rowset/segment_v2/options.h @@ -24,7 +24,7 @@ namespace segment_v2 { static constexpr size_t DEFAULT_PAGE_SIZE = 1024 * 1024; // default size: 1M -constexpr long ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE = 16384; // default row column page size: 16KB +constexpr long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384; // default row store page size: 16KB struct PageBuilderOptions { size_t data_page_size = DEFAULT_PAGE_SIZE; diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index 836dc050b0fa8a..5588bd79f173f4 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -258,7 +258,7 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co if (column.is_row_store_column()) { // smaller page size for row store column - opts.data_page_size = _tablet_schema->row_column_page_size(); + opts.data_page_size = _tablet_schema->row_store_page_size(); } std::unique_ptr writer; RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer, &writer)); diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp index 59a33d003c3f07..92ac65deb6debb 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp @@ -221,7 +221,7 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo if (column.is_row_store_column()) { // smaller page size for row store column - opts.data_page_size = _tablet_schema->row_column_page_size(); + opts.data_page_size = _tablet_schema->row_store_page_size(); } std::unique_ptr writer; RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer, &writer)); diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index ba6404f26e6fdf..a3526781dddd87 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -317,8 +317,8 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id if (tablet_schema.__isset.store_row_column) { schema->set_store_row_column(tablet_schema.store_row_column); } - if (tablet_schema.__isset.row_column_page_size) { - schema->set_row_column_page_size(tablet_schema.row_column_page_size); + if (tablet_schema.__isset.row_store_page_size) { + schema->set_row_store_page_size(tablet_schema.row_store_page_size); } if (tablet_schema.__isset.skip_write_index_on_load) { schema->set_skip_write_index_on_load(tablet_schema.skip_write_index_on_load); diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index 7e1db892eaa84e..c27db2bd3d9e1b 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -990,7 +990,7 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac _sort_type = schema.sort_type(); _sort_col_num = schema.sort_col_num(); _compression_type = schema.compression_type(); - _row_column_page_size = schema.row_column_page_size(); + _row_store_page_size = schema.row_store_page_size(); _schema_version = schema.schema_version(); // Default to V1 inverted index storage format for backward compatibility if not specified in schema. if (!schema.has_inverted_index_storage_format()) { @@ -1051,7 +1051,7 @@ void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version _skip_write_index_on_load = ori_tablet_schema.skip_write_index_on_load(); _sort_type = ori_tablet_schema.sort_type(); _sort_col_num = ori_tablet_schema.sort_col_num(); - _row_column_page_size = ori_tablet_schema.row_column_page_size(); + _row_store_page_size = ori_tablet_schema.row_store_page_size(); // copy from table_schema_param _schema_version = version; @@ -1205,7 +1205,7 @@ void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const { tablet_schema_pb->set_sort_col_num(_sort_col_num); tablet_schema_pb->set_schema_version(_schema_version); tablet_schema_pb->set_compression_type(_compression_type); - tablet_schema_pb->set_row_column_page_size(_row_column_page_size); + tablet_schema_pb->set_row_store_page_size(_row_store_page_size); tablet_schema_pb->set_version_col_idx(_version_col_idx); tablet_schema_pb->set_inverted_index_storage_format(_inverted_index_storage_format); tablet_schema_pb->mutable_row_store_column_unique_ids()->Assign( @@ -1525,7 +1525,7 @@ bool operator==(const TabletSchema& a, const TabletSchema& b) { if (a._disable_auto_compaction != b._disable_auto_compaction) return false; if (a._enable_single_replica_compaction != b._enable_single_replica_compaction) return false; if (a._store_row_column != b._store_row_column) return false; - if (a._row_column_page_size != b._row_column_page_size) return false; + if (a._row_store_page_size != b._row_store_page_size) return false; if (a._skip_write_index_on_load != b._skip_write_index_on_load) return false; return true; } diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index 8dd3328b492274..8cf6e20208c90f 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -360,8 +360,8 @@ class TabletSchema { void set_version_col_idx(int32_t version_col_idx) { _version_col_idx = version_col_idx; } int32_t version_col_idx() const { return _version_col_idx; } segment_v2::CompressionTypePB compression_type() const { return _compression_type; } - void set_row_column_page_size(long page_size) { _row_column_page_size = page_size; } - long row_column_page_size() const { return _row_column_page_size; } + void set_row_store_page_size(long page_size) { _row_store_page_size = page_size; } + long row_store_page_size() const { return _row_store_page_size; } const std::vector& indexes() const { return _indexes; } bool has_inverted_index() const { @@ -511,7 +511,7 @@ class TabletSchema { size_t _num_rows_per_row_block = 0; CompressKind _compress_kind = COMPRESS_NONE; segment_v2::CompressionTypePB _compression_type = segment_v2::CompressionTypePB::LZ4F; - long _row_column_page_size = segment_v2::ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE; + long _row_store_page_size = segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; size_t _next_column_unique_id = 0; std::string _auto_increment_column; diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java index 6eec5fb1801fd8..1c31d74d98630b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java @@ -211,7 +211,7 @@ private void createRollupReplicaForPartition(OlapTable tbl) throws Exception { tbl.disableAutoCompaction(), tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), tbl.getEnableMowLightDelete(), null, - tbl.rowColumnPageSize()); + tbl.rowStorePageSize()); requestBuilder.addTabletMetas(builder); } // end for rollupTablets ((CloudInternalCatalog) Env.getCurrentInternalCatalog()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java index 7d045131d16a6a..a8bcc546de33e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java @@ -232,7 +232,7 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), tbl.getEnableMowLightDelete(), tbl.getInvertedIndexFileStorageFormat(), - tbl.rowColumnPageSize()); + tbl.rowStorePageSize()); requestBuilder.addTabletMetas(builder); } // end for rollupTablets ((CloudInternalCatalog) Env.getCurrentInternalCatalog()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java index 5d911d24fcdbb2..037139704b6f5b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -266,7 +266,7 @@ protected void createRollupReplica() throws AlterCancelException { binlogConfig, tbl.getRowStoreColumnsUniqueIds(tbl.getTableProperty().getCopiedRowStoreColumns()), objectPool, - tbl.rowColumnPageSize()); + tbl.rowStorePageSize()); createReplicaTask.setBaseTablet(tabletIdMap.get(rollupTabletId), baseSchemaHash); if (this.storageFormat != null) { createReplicaTask.setStorageFormat(this.storageFormat); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 74ffb00fac8e13..d58e23ebbbc59b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -294,7 +294,7 @@ protected void createShadowIndexReplica() throws AlterCancelException { binlogConfig, tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), objectPool, - tbl.rowColumnPageSize()); + tbl.rowStorePageSize()); createReplicaTask.setBaseTablet(partitionIndexTabletMap.get(partitionId, shadowIdxId) .get(shadowTabletId), originSchemaHash); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index a3b008eaf91813..adad0a9c5f44ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -1127,7 +1127,7 @@ private void createReplicas(Database db, AgentBatchTask batchTask, OlapTable loc binlogConfig, localTbl.getRowStoreColumnsUniqueIds(rowStoreColumns), objectPool, - localTbl.rowColumnPageSize()); + localTbl.rowStorePageSize()); task.setInvertedIndexFileStorageFormat(localTbl.getInvertedIndexFileStorageFormat()); task.setInRestoreMode(true); batchTask.addTask(task); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index cbffd7562d09da..66ed5ba94e18e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3511,9 +3511,9 @@ private static void addOlapTablePropertyInfo(OlapTable olapTable, StringBuilder sb.append(olapTable.storeRowColumn()).append("\""); } - // row column page size - sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_ROW_COLUMN_PAGE_SIZE).append("\" = \""); - sb.append(olapTable.rowColumnPageSize()).append("\""); + // row store page size + sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_ROW_STORE_PAGE_SIZE).append("\" = \""); + sb.append(olapTable.rowStorePageSize()).append("\""); } // skip inverted index on load diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 78b53d47f520ed..62b93ca8adc0e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -2505,18 +2505,18 @@ public void setCompressionType(TCompressionType compressionType) { tableProperty.buildCompressionType(); } - public void setRowColumnPageSize(long pageSize) { + public void setRowStorePageSize(long pageSize) { TableProperty tableProperty = getOrCreatTableProperty(); - tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_ROW_COLUMN_PAGE_SIZE, + tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_ROW_STORE_PAGE_SIZE, Long.valueOf(pageSize).toString()); - tableProperty.buildRowColumnPageSize(); + tableProperty.buildRowStorePageSize(); } - public long rowColumnPageSize() { + public long rowStorePageSize() { if (tableProperty != null) { - return tableProperty.rowColumnPageSize(); + return tableProperty.rowStorePageSize(); } - return PropertyAnalyzer.ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE; + return PropertyAnalyzer.ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; } public void setStorageFormat(TStorageFormat storageFormat) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java index f4ba49a029b242..21f96a202f173f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java @@ -102,7 +102,7 @@ public class TableProperty implements Writable, GsonPostProcessable { private boolean skipWriteIndexOnLoad = false; - private long rowColumnPageSize = PropertyAnalyzer.ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE; + private long rowStorePageSize = PropertyAnalyzer.ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; private String compactionPolicy = PropertyAnalyzer.SIZE_BASED_COMPACTION_POLICY; @@ -269,15 +269,15 @@ public boolean storeRowColumn() { return storeRowColumn; } - public TableProperty buildRowColumnPageSize() { - rowColumnPageSize = Long.parseLong( - properties.getOrDefault(PropertyAnalyzer.PROPERTIES_ROW_COLUMN_PAGE_SIZE, - Long.toString(PropertyAnalyzer.ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE))); + public TableProperty buildRowStorePageSize() { + rowStorePageSize = Long.parseLong( + properties.getOrDefault(PropertyAnalyzer.PROPERTIES_ROW_STORE_PAGE_SIZE, + Long.toString(PropertyAnalyzer.ROW_STORE_PAGE_SIZE_DEFAULT_VALUE))); return this; } - public long rowColumnPageSize() { - return rowColumnPageSize; + public long rowStorePageSize() { + return rowStorePageSize; } public TableProperty buildSkipWriteIndexOnLoad() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index 4241822586f9bb..74cdccbb262dd4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -174,7 +174,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), tbl.getEnableMowLightDelete(), tbl.getInvertedIndexFileStorageFormat(), - tbl.rowColumnPageSize()); + tbl.rowStorePageSize()); requestBuilder.addTabletMetas(builder); } if (!storageVaultIdSet && ((CloudEnv) Env.getCurrentEnv()).getEnableStorageVault()) { @@ -348,7 +348,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2); } } - schemaBuilder.setRowColumnPageSize(pageSize); + schemaBuilder.setRowStorePageSize(pageSize); OlapFile.TabletSchemaCloudPB schema = schemaBuilder.build(); builder.setSchema(schema); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 5b2534ed156cfc..e339d8e5e11f47 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -96,8 +96,8 @@ public class PropertyAnalyzer { public static final String PROPERTIES_COMPRESSION = "compression"; // row store page size, default 16KB - public static final String PROPERTIES_ROW_COLUMN_PAGE_SIZE = "row_store_page_size"; - public static final long ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE = 16384; + public static final String PROPERTIES_ROW_STORE_PAGE_SIZE = "row_store_page_size"; + public static final long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384; public static final String PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE = "light_schema_change"; @@ -1020,25 +1020,25 @@ public static long alignTo4K(long size) { return (size + 4095) & ~4095; } - // analyzeRowColumnPageSize will parse the row_column_page_size from properties - public static long analyzeRowColumnPageSize(Map properties) throws AnalysisException { - long rowColumnPageSize = ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE; - if (properties != null && properties.containsKey(PROPERTIES_ROW_COLUMN_PAGE_SIZE)) { - String rowColumnPageSizeStr = properties.get(PROPERTIES_ROW_COLUMN_PAGE_SIZE); + // analyzeRowStorePageSize will parse the row_store_page_size from properties + public static long analyzeRowStorePageSize(Map properties) throws AnalysisException { + long rowStorePageSize = ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; + if (properties != null && properties.containsKey(PROPERTIES_ROW_STORE_PAGE_SIZE)) { + String rowStorePageSizeStr = properties.get(PROPERTIES_ROW_STORE_PAGE_SIZE); try { - rowColumnPageSize = alignTo4K(Long.parseLong(rowColumnPageSizeStr)); + rowStorePageSize = alignTo4K(Long.parseLong(rowStorePageSizeStr)); } catch (NumberFormatException e) { - throw new AnalysisException("Invalid row column page size: " + rowColumnPageSizeStr); + throw new AnalysisException("Invalid row store page size: " + rowStorePageSizeStr); } - if (rowColumnPageSize <= 0) { - throw new AnalysisException("Row column page size should larger than 0."); + if (rowStorePageSize <= 0) { + throw new AnalysisException("Row store page size should larger than 0."); } - properties.remove(PROPERTIES_ROW_COLUMN_PAGE_SIZE); + properties.remove(PROPERTIES_ROW_STORE_PAGE_SIZE); } - return rowColumnPageSize; + return rowStorePageSize; } // analyzeStorageFormat will parse the storage format from properties diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index db6c896ac32d60..dfa84d551efd4e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1559,9 +1559,9 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa properties.put(PropertyAnalyzer.PROPERTIES_STORE_ROW_COLUMN, olapTable.storeRowColumn().toString()); } - if (!properties.containsKey(PropertyAnalyzer.PROPERTIES_ROW_COLUMN_PAGE_SIZE)) { - properties.put(PropertyAnalyzer.PROPERTIES_ROW_COLUMN_PAGE_SIZE, - Long.toString(olapTable.rowColumnPageSize())); + if (!properties.containsKey(PropertyAnalyzer.PROPERTIES_ROW_STORE_PAGE_SIZE)) { + properties.put(PropertyAnalyzer.PROPERTIES_ROW_STORE_PAGE_SIZE, + Long.toString(olapTable.rowStorePageSize())); } if (!properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD)) { properties.put(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD, @@ -1696,7 +1696,7 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa singlePartitionDesc.isInMemory(), singlePartitionDesc.getTabletType(), storagePolicy, idGeneratorBuffer, - binlogConfig, dataProperty.isStorageMediumSpecified(), null, olapTable.rowColumnPageSize()); + binlogConfig, dataProperty.isStorageMediumSpecified(), null, olapTable.rowStorePageSize()); // TODO cluster key ids // check again @@ -1986,7 +1986,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa IdGeneratorBuffer idGeneratorBuffer, BinlogConfig binlogConfig, boolean isStorageMediumSpecified, - List clusterKeyIndexes, long rowColumnPageSize) + List clusterKeyIndexes, long rowStorePageSize) throws DdlException { // create base index first. Preconditions.checkArgument(tbl.getBaseIndexId() != -1); @@ -2070,7 +2070,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa tbl.getTimeSeriesCompactionLevelThreshold(), tbl.storeRowColumn(), binlogConfig, tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), - objectPool, rowColumnPageSize); + objectPool, rowStorePageSize); task.setStorageFormat(tbl.getStorageFormat()); task.setInvertedIndexFileStorageFormat(tbl.getInvertedIndexFileStorageFormat()); @@ -2445,15 +2445,15 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx } olapTable.setCompressionType(compressionType); - // get row_column_page_size - long rowColumnPageSize = PropertyAnalyzer.ROW_COLUMN_PAGE_SIZE_DEFAULT_VALUE; + // get row_store_page_size + long rowStorePageSize = PropertyAnalyzer.ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; try { - rowColumnPageSize = PropertyAnalyzer.analyzeRowColumnPageSize(properties); + rowStorePageSize = PropertyAnalyzer.analyzeRowStorePageSize(properties); } catch (AnalysisException e) { throw new DdlException(e.getMessage()); } - olapTable.setRowColumnPageSize(rowColumnPageSize); + olapTable.setRowStorePageSize(rowStorePageSize); // check data sort properties int keyColumnSize = CollectionUtils.isEmpty(keysDesc.getClusterKeysColumnIds()) ? keysDesc.keysColumnSize() : @@ -2853,7 +2853,7 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx idGeneratorBuffer, binlogConfigForTask, partitionInfo.getDataProperty(partitionId).isStorageMediumSpecified(), - keysDesc.getClusterKeysColumnIds(), olapTable.rowColumnPageSize()); + keysDesc.getClusterKeysColumnIds(), olapTable.rowStorePageSize()); afterCreatePartitions(db.getId(), olapTable.getId(), null, olapTable.getIndexIdList(), true); olapTable.addPartition(partition); @@ -2936,7 +2936,7 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx partionStoragePolicy, idGeneratorBuffer, binlogConfigForTask, dataProperty.isStorageMediumSpecified(), - keysDesc.getClusterKeysColumnIds(), olapTable.rowColumnPageSize()); + keysDesc.getClusterKeysColumnIds(), olapTable.rowStorePageSize()); olapTable.addPartition(partition); olapTable.getPartitionInfo().getDataProperty(partition.getId()) .setStoragePolicy(partionStoragePolicy); @@ -3403,7 +3403,7 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti olapTable.getPartitionInfo().getDataProperty(oldPartitionId).getStoragePolicy(), idGeneratorBuffer, binlogConfig, copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId).isStorageMediumSpecified(), - clusterKeyIdxes, olapTable.rowColumnPageSize()); + clusterKeyIdxes, olapTable.rowStorePageSize()); newPartitions.add(newPartition); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java index 73e50706d44b4a..f2cd9ad5843d5a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java @@ -887,7 +887,7 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta binlogConfig, olapTable.getRowStoreColumnsUniqueIds(rowStoreColumns), objectPool, - olapTable.rowColumnPageSize()); + olapTable.rowStorePageSize()); createReplicaTask.setIsRecoverTask(true); createReplicaTask.setInvertedIndexFileStorageFormat(olapTable diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java index 67583071a12118..7864ce2d4c2f27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -65,7 +65,7 @@ public class CreateReplicaTask extends AgentTask { private TStorageType storageType; private TStorageMedium storageMedium; private TCompressionType compressionType; - private long rowColumnPageSize; + private long rowStorePageSize; private List columns; @@ -153,7 +153,7 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition BinlogConfig binlogConfig, List rowStoreColumnUniqueIds, Map objectPool, - long rowColumnPageSize) { + long rowStorePageSize) { super(null, backendId, TTaskType.CREATE, dbId, tableId, partitionId, indexId, tabletId); this.replicaId = replicaId; @@ -199,7 +199,7 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition this.storeRowColumn = storeRowColumn; this.binlogConfig = binlogConfig; this.objectPool = objectPool; - this.rowColumnPageSize = rowColumnPageSize; + this.rowStorePageSize = rowStorePageSize; } public void setIsRecoverTask(boolean isRecoverTask) { @@ -342,7 +342,7 @@ public TCreateTabletReq toThrift() { tSchema.setEnableSingleReplicaCompaction(enableSingleReplicaCompaction); tSchema.setSkipWriteIndexOnLoad(skipWriteIndexOnLoad); tSchema.setStoreRowColumn(storeRowColumn); - tSchema.setRowColumnPageSize(rowColumnPageSize); + tSchema.setRowStorePageSize(rowStorePageSize); createTabletReq.setTabletSchema(tSchema); createTabletReq.setVersion(version); diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java index 62b6ad2517152b..5eea07e5ffe5d9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java @@ -73,7 +73,7 @@ public class AgentTaskTest { private long version = 1L; private TStorageType storageType = TStorageType.COLUMN; - private long rowColumnPageSize = 16384L; + private long rowStorePageSize = 16384L; private List columns; private MarkedCountDownLatch latch = new MarkedCountDownLatch(3); @@ -108,7 +108,7 @@ public void setUp() throws AnalysisException { createReplicaTask = new CreateReplicaTask(backendId1, dbId, tableId, partitionId, indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType, TStorageMedium.SSD, columns, null, 0, latch, null, false, TTabletType.TABLET_TYPE_DISK, null, - TCompressionType.LZ4F, false, "", false, false, false, "", 0, 0, 0, 0, 0, false, null, null, objectPool, rowColumnPageSize); + TCompressionType.LZ4F, false, "", false, false, false, "", 0, 0, 0, 0, 0, false, null, null, objectPool, rowStorePageSize); // drop dropTask = new DropReplicaTask(backendId1, tabletId1, replicaId1, schemaHash1, false); diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 089530f73c1582..8c1b21fa3cf93b 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -383,7 +383,7 @@ message TabletSchemaPB { optional InvertedIndexStorageFormatPB inverted_index_storage_format = 25 [default=V1]; // column unique ids for row store columns repeated int32 row_store_column_unique_ids = 26; - optional int64 row_column_page_size = 27 [default=16384]; + optional int64 row_store_page_size = 27 [default=16384]; } message TabletSchemaCloudPB { @@ -413,7 +413,7 @@ message TabletSchemaCloudPB { // column unique ids for row store columns repeated int32 row_store_column_unique_ids = 26; optional bool enable_mow_light_delete = 27 [default=false]; - optional int64 row_column_page_size = 28 [default=16384]; + optional int64 row_store_page_size = 28 [default=16384]; optional bool is_dynamic_schema = 100 [default=false]; } diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index 1cff0f77fce1f2..49a0e59b2eebdd 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -47,7 +47,7 @@ struct TTabletSchema { 19: optional list cluster_key_idxes // col unique id for row store column 20: optional list row_store_col_cids - 21: optional i64 row_column_page_size = 16384; + 21: optional i64 row_store_page_size = 16384; } // this enum stands for different storage format in src_backends From 07beb0ce680db02509c72642e4448d77a907b71e Mon Sep 17 00:00:00 2001 From: lxr599 Date: Wed, 17 Jul 2024 20:49:33 +0800 Subject: [PATCH 16/23] compatible with 'branch-3.0' --- gensrc/proto/olap_file.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 8c1b21fa3cf93b..9e10c15a577b7b 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -412,8 +412,8 @@ message TabletSchemaCloudPB { optional InvertedIndexStorageFormatPB inverted_index_storage_format = 25 [default=V1]; // column unique ids for row store columns repeated int32 row_store_column_unique_ids = 26; - optional bool enable_mow_light_delete = 27 [default=false]; - optional int64 row_store_page_size = 28 [default=16384]; + optional int64 row_store_page_size = 27 [default=16384]; + optional bool enable_mow_light_delete = 28 [default=false]; optional bool is_dynamic_schema = 100 [default=false]; } From 66198ca87fe724323ffbce05901d8720f79ead58 Mon Sep 17 00:00:00 2001 From: lxr599 Date: Tue, 23 Jul 2024 13:49:47 +0800 Subject: [PATCH 17/23] get the right value of page size after FE restarts --- .../doris/analysis/ModifyTablePropertiesClause.java | 2 ++ .../java/org/apache/doris/catalog/TableProperty.java | 1 + .../doris/cloud/datasource/CloudInternalCatalog.java | 3 ++- .../apache/doris/common/util/PropertyAnalyzer.java | 2 +- .../org/apache/doris/datasource/InternalCatalog.java | 12 ++++++------ gensrc/thrift/AgentService.thrift | 2 +- 6 files changed, 13 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java index 85b00394326730..e619cc90dbf76a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java @@ -346,6 +346,8 @@ public void analyze(Analyzer analyzer) throws AnalysisException { // do nothing, will be analyzed when creating alter job } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_ROW_STORE_COLUMNS)) { // do nothing, will be analyzed when creating alter job + } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_ROW_STORE_PAGE_SIZE)) { + // do nothing, will be analyzed when creating alter job } else { throw new AnalysisException("Unknown table property: " + properties.keySet()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java index 21f96a202f173f..5ac446411f38e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java @@ -670,6 +670,7 @@ public void gsonPostProcess() throws IOException { buildEnableLightSchemaChange(); buildStoreRowColumn(); buildRowStoreColumns(); + buildRowStorePageSize(); buildSkipWriteIndexOnLoad(); buildCompactionPolicy(); buildTimeSeriesCompactionGoalSizeMbytes(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index 721ec7899f41f7..eba9fb0b0c4a64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -102,7 +102,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa IdGeneratorBuffer idGeneratorBuffer, BinlogConfig binlogConfig, boolean isStorageMediumSpecified, - List clusterKeyIndexes, long pageSize) + List clusterKeyIndexes) throws DdlException { // create base index first. Preconditions.checkArgument(tbl.getBaseIndexId() != -1); @@ -348,6 +348,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2); } } + LOG.info("[lxr] set row store page size: {}", pageSize); schemaBuilder.setRowStorePageSize(pageSize); OlapFile.TabletSchemaCloudPB schema = schemaBuilder.build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index e339d8e5e11f47..3d1ab7c895100d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -97,7 +97,7 @@ public class PropertyAnalyzer { // row store page size, default 16KB public static final String PROPERTIES_ROW_STORE_PAGE_SIZE = "row_store_page_size"; - public static final long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384; + public static final long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384L; public static final String PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE = "light_schema_change"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index c4ebad14425563..0bb64cc02c2736 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1710,7 +1710,7 @@ public PartitionPersistInfo addPartition(Database db, String tableName, AddParti singlePartitionDesc.isInMemory(), singlePartitionDesc.getTabletType(), storagePolicy, idGeneratorBuffer, - binlogConfig, dataProperty.isStorageMediumSpecified(), null, olapTable.rowStorePageSize()); + binlogConfig, dataProperty.isStorageMediumSpecified(), null); // TODO cluster key ids // check again @@ -2006,7 +2006,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa IdGeneratorBuffer idGeneratorBuffer, BinlogConfig binlogConfig, boolean isStorageMediumSpecified, - List clusterKeyIndexes, long rowStorePageSize) + List clusterKeyIndexes) throws DdlException { // create base index first. Preconditions.checkArgument(tbl.getBaseIndexId() != -1); @@ -2090,7 +2090,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa tbl.getTimeSeriesCompactionLevelThreshold(), tbl.storeRowColumn(), binlogConfig, tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), - objectPool, rowStorePageSize); + objectPool, tbl.rowStorePageSize()); task.setStorageFormat(tbl.getStorageFormat()); task.setInvertedIndexFileStorageFormat(tbl.getInvertedIndexFileStorageFormat()); @@ -2880,7 +2880,7 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx idGeneratorBuffer, binlogConfigForTask, partitionInfo.getDataProperty(partitionId).isStorageMediumSpecified(), - keysDesc.getClusterKeysColumnIds(), olapTable.rowStorePageSize()); + keysDesc.getClusterKeysColumnIds()); afterCreatePartitions(db.getId(), olapTable.getId(), null, olapTable.getIndexIdList(), true); olapTable.addPartition(partition); @@ -2963,7 +2963,7 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx partionStoragePolicy, idGeneratorBuffer, binlogConfigForTask, dataProperty.isStorageMediumSpecified(), - keysDesc.getClusterKeysColumnIds(), olapTable.rowStorePageSize()); + keysDesc.getClusterKeysColumnIds()); olapTable.addPartition(partition); olapTable.getPartitionInfo().getDataProperty(partition.getId()) .setStoragePolicy(partionStoragePolicy); @@ -3430,7 +3430,7 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti olapTable.getPartitionInfo().getDataProperty(oldPartitionId).getStoragePolicy(), idGeneratorBuffer, binlogConfig, copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId).isStorageMediumSpecified(), - clusterKeyIdxes, olapTable.rowStorePageSize()); + clusterKeyIdxes); newPartitions.add(newPartition); } diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index 8d24e64c0180a9..767ede9017af46 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -47,7 +47,7 @@ struct TTabletSchema { 19: optional list cluster_key_idxes // col unique id for row store column 20: optional list row_store_col_cids - 21: optional i64 row_store_page_size = 16384; + 21: optional i64 row_store_page_size = 16384 } // this enum stands for different storage format in src_backends From 759ef99c74e45d0f4d6b4784614b9045df1026e8 Mon Sep 17 00:00:00 2001 From: lxr599 Date: Tue, 23 Jul 2024 13:56:49 +0800 Subject: [PATCH 18/23] clear debug logs --- .../org/apache/doris/cloud/datasource/CloudInternalCatalog.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index c1dc1fb9c0364c..075772b929f78b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -348,7 +348,6 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2); } } - LOG.info("[lxr] set row store page size: {}", pageSize); schemaBuilder.setRowStorePageSize(pageSize); OlapFile.TabletSchemaCloudPB schema = schemaBuilder.build(); From cb7548ef242a850ef4931c0b9a152e41dc087614 Mon Sep 17 00:00:00 2001 From: lxr599 Date: Tue, 23 Jul 2024 19:34:20 +0800 Subject: [PATCH 19/23] Ensure page size is larger than 0 in BE; Add more test cases; --- .../olap/rowset/segment_v2/segment_writer.cpp | 7 +- .../segment_v2/vertical_segment_writer.cpp | 7 +- .../analysis/ModifyTablePropertiesClause.java | 2 - .../test_row_store_page_size.out | 16 ++++ .../test_row_store_page_size.groovy | 80 +++++++++++++++++++ 5 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 regression-test/data/cloud_p0/row_store_page_size/test_row_store_page_size.out create mode 100644 regression-test/suites/cloud_p0/row_store_page_size/test_row_store_page_size.groovy diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index 5588bd79f173f4..dfea86ce716f12 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -258,8 +258,13 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co if (column.is_row_store_column()) { // smaller page size for row store column - opts.data_page_size = _tablet_schema->row_store_page_size(); + auto page_size = _tablet_schema->row_store_page_size(); + opts.data_page_size = (page_size > 0) ? page_size : ([this]() { + _tablet_schema->set_row_store_page_size(segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE); + return segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; + })(); } + std::unique_ptr writer; RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer, &writer)); RETURN_IF_ERROR(writer->init()); diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp index 0307a53d740f18..fb4a54b9410041 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp @@ -221,8 +221,13 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo if (column.is_row_store_column()) { // smaller page size for row store column - opts.data_page_size = _tablet_schema->row_store_page_size(); + auto page_size = _tablet_schema->row_store_page_size(); + opts.data_page_size = (page_size > 0) ? page_size : ([this]() { + _tablet_schema->set_row_store_page_size(segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE); + return segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; + })(); } + std::unique_ptr writer; RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer, &writer)); RETURN_IF_ERROR(writer->init()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java index 27052ae8d3e747..c7ea5c2eccad60 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java @@ -350,8 +350,6 @@ public void analyze(Analyzer analyzer) throws AnalysisException { // do nothing, will be analyzed when creating alter job } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_ROW_STORE_COLUMNS)) { // do nothing, will be analyzed when creating alter job - } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_ROW_STORE_PAGE_SIZE)) { - // do nothing, will be analyzed when creating alter job } else { throw new AnalysisException("Unknown table property: " + properties.keySet()); } diff --git a/regression-test/data/cloud_p0/row_store_page_size/test_row_store_page_size.out b/regression-test/data/cloud_p0/row_store_page_size/test_row_store_page_size.out new file mode 100644 index 00000000000000..5ca03a3cfa91e1 --- /dev/null +++ b/regression-test/data/cloud_p0/row_store_page_size/test_row_store_page_size.out @@ -0,0 +1,16 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !show_create -- +ps_table_1 CREATE TABLE `ps_table_1` (\n `k1` int NOT NULL,\n `k2` int NOT NULL,\n `k3` bigint NULL,\n `k4` varchar(100) NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`, `k2`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 3\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"store_row_column" = "true",\n"row_store_page_size" = "16384",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_light_delete" = "false"\n); + +-- !select_star -- +1 1 1 a + +-- !select_star -- +3 3 \N c + +-- !show_create -- +ps_table_2 CREATE TABLE `ps_table_2` (\n `k1` int NOT NULL,\n `k2` int NOT NULL,\n `k3` bigint NULL,\n `k4` varchar(100) NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`, `k2`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 3\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"store_row_column" = "true",\n"row_store_page_size" = "8192",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_light_delete" = "false"\n); + +-- !select_star -- +1 1 1 a + diff --git a/regression-test/suites/cloud_p0/row_store_page_size/test_row_store_page_size.groovy b/regression-test/suites/cloud_p0/row_store_page_size/test_row_store_page_size.groovy new file mode 100644 index 00000000000000..d96f51c51682a1 --- /dev/null +++ b/regression-test/suites/cloud_p0/row_store_page_size/test_row_store_page_size.groovy @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_row_store_page_size_cloud") { + + sql """ DROP TABLE IF EXISTS ps_table_1; """ + + sql """ + create table ps_table_1( + k1 int not null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + unique key (k1,k2) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1", "store_row_column" = "true"); + """ + + qt_show_create "show create table ps_table_1;" + + sql "insert into ps_table_1 select 1,1,1,'a';" + sql "insert into ps_table_1 select 2,2,2,'b';" + sql "insert into ps_table_1 select 3,3,null,'c';" + + explain { + sql("select * from ps_table_1 where k1=1 and k2=1;") + contains("SHORT") + } + + qt_select_star "select * from ps_table_1 where k1=1 and k2=1;" + qt_select_star "select * from ps_table_1 where k1=3 and k2=3;" + + sql """ DROP TABLE IF EXISTS ps_table_1; """ + + sql """ DROP TABLE IF EXISTS ps_table_2; """ + + sql """ + create table ps_table_2( + k1 int not null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + unique key (k1,k2) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1", "store_row_column" = "true", "row_store_page_size" = "8190"); + """ + + qt_show_create "show create table ps_table_2;" + + sql "insert into ps_table_2 select 1,1,1,'a';" + sql "insert into ps_table_2 select 2,2,2,'b';" + sql "insert into ps_table_2 select 3,3,null,'c';" + + explain { + sql("select * from ps_table_2 where k1=1 and k2=1;") + contains("SHORT") + } + + qt_select_star "select * from ps_table_2 where k1=1 and k2=1;" + + sql """ DROP TABLE IF EXISTS ps_table_2; """ +} From 09c384cb98d383ac54b72031a518b0cdf572fc12 Mon Sep 17 00:00:00 2001 From: lxr599 Date: Wed, 24 Jul 2024 00:53:08 +0800 Subject: [PATCH 20/23] modify the test case of cloud mode --- .../test_row_store_page_size.out | 6 ------ .../test_row_store_page_size.groovy | 16 +++++++++++++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/regression-test/data/cloud_p0/row_store_page_size/test_row_store_page_size.out b/regression-test/data/cloud_p0/row_store_page_size/test_row_store_page_size.out index 5ca03a3cfa91e1..95b3ccd49508ac 100644 --- a/regression-test/data/cloud_p0/row_store_page_size/test_row_store_page_size.out +++ b/regression-test/data/cloud_p0/row_store_page_size/test_row_store_page_size.out @@ -1,16 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !show_create -- -ps_table_1 CREATE TABLE `ps_table_1` (\n `k1` int NOT NULL,\n `k2` int NOT NULL,\n `k3` bigint NULL,\n `k4` varchar(100) NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`, `k2`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 3\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"store_row_column" = "true",\n"row_store_page_size" = "16384",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_light_delete" = "false"\n); - -- !select_star -- 1 1 1 a -- !select_star -- 3 3 \N c --- !show_create -- -ps_table_2 CREATE TABLE `ps_table_2` (\n `k1` int NOT NULL,\n `k2` int NOT NULL,\n `k3` bigint NULL,\n `k4` varchar(100) NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`, `k2`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 3\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"store_row_column" = "true",\n"row_store_page_size" = "8192",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_light_delete" = "false"\n); - -- !select_star -- 1 1 1 a diff --git a/regression-test/suites/cloud_p0/row_store_page_size/test_row_store_page_size.groovy b/regression-test/suites/cloud_p0/row_store_page_size/test_row_store_page_size.groovy index d96f51c51682a1..4be53ff17f63c1 100644 --- a/regression-test/suites/cloud_p0/row_store_page_size/test_row_store_page_size.groovy +++ b/regression-test/suites/cloud_p0/row_store_page_size/test_row_store_page_size.groovy @@ -33,7 +33,12 @@ suite ("test_row_store_page_size_cloud") { properties("replication_num" = "1", "store_row_column" = "true"); """ - qt_show_create "show create table ps_table_1;" + test { + sql "show create table ps_table_1;" + check { result, exception, startTime, endTime -> + assertTrue(result[0][1].contains("\"row_store_page_size\" = \"16384\"")) + } + } sql "insert into ps_table_1 select 1,1,1,'a';" sql "insert into ps_table_1 select 2,2,2,'b';" @@ -63,8 +68,13 @@ suite ("test_row_store_page_size_cloud") { properties("replication_num" = "1", "store_row_column" = "true", "row_store_page_size" = "8190"); """ - qt_show_create "show create table ps_table_2;" - + test { + sql "show create table ps_table_2;" + check { result, exception, startTime, endTime -> + assertTrue(result[0][1].contains("\"row_store_page_size\" = \"8192\"")) + } + } + sql "insert into ps_table_2 select 1,1,1,'a';" sql "insert into ps_table_2 select 2,2,2,'b';" sql "insert into ps_table_2 select 3,3,null,'c';" From f6cdadb28783ca626c2656560f1a9a0758aeaede Mon Sep 17 00:00:00 2001 From: lxr599 Date: Wed, 24 Jul 2024 14:44:22 +0800 Subject: [PATCH 21/23] do not set page_size, use default directly --- be/src/olap/rowset/segment_v2/segment_writer.cpp | 5 +---- be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index dfea86ce716f12..4b379d22822efe 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -259,10 +259,7 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co if (column.is_row_store_column()) { // smaller page size for row store column auto page_size = _tablet_schema->row_store_page_size(); - opts.data_page_size = (page_size > 0) ? page_size : ([this]() { - _tablet_schema->set_row_store_page_size(segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE); - return segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; - })(); + opts.data_page_size = (page_size > 0) ? page_size : segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; } std::unique_ptr writer; diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp index fb4a54b9410041..4ea75ffe549511 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp @@ -222,10 +222,7 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo if (column.is_row_store_column()) { // smaller page size for row store column auto page_size = _tablet_schema->row_store_page_size(); - opts.data_page_size = (page_size > 0) ? page_size : ([this]() { - _tablet_schema->set_row_store_page_size(segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE); - return segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; - })(); + opts.data_page_size = (page_size > 0) ? page_size : segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; } std::unique_ptr writer; From 4901a4f41d029698d0171687075ef067bc02bd2b Mon Sep 17 00:00:00 2001 From: lxr599 Date: Wed, 24 Jul 2024 14:51:29 +0800 Subject: [PATCH 22/23] format code --- be/src/olap/rowset/segment_v2/segment_writer.cpp | 3 ++- be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index 4b379d22822efe..c2804cf6325c29 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -259,7 +259,8 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co if (column.is_row_store_column()) { // smaller page size for row store column auto page_size = _tablet_schema->row_store_page_size(); - opts.data_page_size = (page_size > 0) ? page_size : segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; + opts.data_page_size = + (page_size > 0) ? page_size : segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; } std::unique_ptr writer; diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp index 4ea75ffe549511..ba1bfcf353539f 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp @@ -222,7 +222,8 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo if (column.is_row_store_column()) { // smaller page size for row store column auto page_size = _tablet_schema->row_store_page_size(); - opts.data_page_size = (page_size > 0) ? page_size : segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; + opts.data_page_size = + (page_size > 0) ? page_size : segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; } std::unique_ptr writer; From 26e18ac623c12ffa189e17ac6897901ce8faa0c0 Mon Sep 17 00:00:00 2001 From: lxr599 Date: Wed, 24 Jul 2024 14:59:50 +0800 Subject: [PATCH 23/23] format code... --- be/src/olap/rowset/segment_v2/segment_writer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index c2804cf6325c29..bdfcaba8b8eb99 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -259,7 +259,7 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co if (column.is_row_store_column()) { // smaller page size for row store column auto page_size = _tablet_schema->row_store_page_size(); - opts.data_page_size = + opts.data_page_size = (page_size > 0) ? page_size : segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; }