From 4e40a4d5e533a5e53224b50e78fc32ba4bb1fb5d Mon Sep 17 00:00:00 2001 From: qiye Date: Thu, 20 Mar 2025 10:45:15 +0800 Subject: [PATCH] [fix](Index)Make column unique ids in index dynamically computed (#48988) Issue Number: close #48989 Related PR: #46648 #42882 Problem Summary: We introduced `Index.columnUniqueIds` in 3.0.3, and make sure `Index.columnUniqueIds` is not empty in 3.0.4. But when we upgrade Doris from 3.0.2 and before to 3.0.4, the `Index.columnUniqueIds` will be `null`. Versions 302 and earlier do not have this variable, and gson deserialization will assign it to null. We need to initialize the column unique ids associated with the index correctly. Key Changes Removed stored `columnUniqueIds` from `Index` class: Previously, column unique IDs were stored as a field in the `Index` class Now they're dynamically computed when needed rather than stored redundantly Added dynamic computation method: New method `getColumnUniqueIds(List schema)` computes IDs at runtime Column IDs are looked up by matching index column names with schema columns Updated method signatures: Changed `toThrift()` to `toThrift(List indexColumnUniqueIds)` Changed `toPb()` to `toPb(Map columnMap, List indexColumnUniqueIds)` Updated all callers to pass column IDs as parameters --- .../doris/alter/SchemaChangeHandler.java | 2 - .../doris/analysis/BuildIndexClause.java | 2 +- .../doris/analysis/CreateIndexClause.java | 2 +- .../doris/analysis/CreateTableStmt.java | 3 +- .../org/apache/doris/analysis/IndexDef.java | 5 - .../java/org/apache/doris/catalog/Index.java | 40 +++-- .../doris/catalog/MaterializedIndexMeta.java | 7 - .../datasource/CloudInternalCatalog.java | 5 +- .../plans/commands/info/IndexDefinition.java | 2 +- .../apache/doris/planner/OlapScanNode.java | 2 +- .../apache/doris/planner/OlapTableSink.java | 2 +- .../doris/task/AlterInvertedIndexTask.java | 4 +- .../apache/doris/task/CreateReplicaTask.java | 2 +- .../org/apache/doris/catalog/IndexTest.java | 164 ++++++++++++++++++ .../apache/doris/catalog/OlapTableTest.java | 2 +- .../common/proc/IndexesProcNodeTest.java | 8 +- .../TableAddOrDropColumnsInfoTest.java | 2 +- 17 files changed, 202 insertions(+), 52 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 55f4d7255c79e5..ac3f8ad40be769 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -2712,7 +2712,6 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab indexDef.checkColumn(column, olapTable.getKeysType(), olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite(), disableInvertedIndexV1ForVariant); - indexDef.getColumnUniqueIds().add(column.getUniqueId()); } else { throw new DdlException("index column does not exist in table. invalid column: " + col); } @@ -2723,7 +2722,6 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab // so here update column name in CreateIndexClause after checkColumn for indexDef, // there will use the column name in olapTable instead of the column name in CreateIndexClause. alterIndex.setColumns(indexDef.getColumns()); - alterIndex.setColumnUniqueIds(indexDef.getColumnUniqueIds()); newIndexes.add(alterIndex); return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java index 46520162db7a00..d04d24e86e403a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java @@ -77,7 +77,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException { indexDef.analyze(); this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(), indexDef.getColumns(), indexDef.getIndexType(), - indexDef.getProperties(), indexDef.getComment(), indexDef.getColumnUniqueIds()); + indexDef.getProperties(), indexDef.getComment()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateIndexClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateIndexClause.java index 62d051ca318d31..a3193943cacf9b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateIndexClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateIndexClause.java @@ -73,7 +73,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException { indexDef.analyze(); this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(), indexDef.getColumns(), indexDef.getIndexType(), - indexDef.getProperties(), indexDef.getComment(), indexDef.getColumnUniqueIds()); + indexDef.getProperties(), indexDef.getComment()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index 6afee3ad843cb8..431f0dff8ff499 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -592,8 +592,7 @@ public void analyze(Analyzer analyzer) throws UserException { } } indexes.add(new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(), indexDef.getColumns(), - indexDef.getIndexType(), indexDef.getProperties(), indexDef.getComment(), - indexDef.getColumnUniqueIds())); + indexDef.getIndexType(), indexDef.getProperties(), indexDef.getComment())); distinct.add(indexDef.getIndexName()); distinctCol.add(Pair.of(indexDef.getIndexType(), indexDef.getColumns().stream().map(String::toUpperCase).collect(Collectors.toList()))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java index 3e44d7c76ad521..0f07a953b9eeed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java @@ -42,7 +42,6 @@ public class IndexDef { private Map properties; private boolean isBuildDeferred = false; private PartitionNames partitionNames; - private List columnUniqueIds = Lists.newArrayList(); public static final int MIN_NGRAM_SIZE = 1; public static final int MAX_NGRAM_SIZE = 255; public static final int MIN_BF_SIZE = 64; @@ -202,10 +201,6 @@ public List getPartitionNames() { return partitionNames == null ? Lists.newArrayList() : partitionNames.getPartitionNames(); } - public List getColumnUniqueIds() { - return columnUniqueIds; - } - public enum IndexType { BITMAP, INVERTED, diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index f770de9176f0db..1718c26bdb4566 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -65,19 +65,15 @@ public class Index implements Writable { private Map properties; @SerializedName(value = "ct", alternate = {"comment"}) private String comment; - @SerializedName(value = "cui", alternate = {"columnUniqueIds"}) - private List columnUniqueIds; public Index(long indexId, String indexName, List columns, - IndexDef.IndexType indexType, Map properties, String comment, - List columnUniqueIds) { + IndexDef.IndexType indexType, Map properties, String comment) { this.indexId = indexId; this.indexName = indexName; this.columns = columns == null ? Lists.newArrayList() : Lists.newArrayList(columns); this.indexType = indexType; this.properties = properties == null ? Maps.newHashMap() : Maps.newHashMap(properties); this.comment = comment; - this.columnUniqueIds = columnUniqueIds == null ? Lists.newArrayList() : Lists.newArrayList(columnUniqueIds); if (indexType == IndexDef.IndexType.INVERTED) { if (this.properties != null && !this.properties.isEmpty()) { if (this.properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)) { @@ -101,7 +97,6 @@ public Index() { this.indexType = null; this.properties = null; this.comment = null; - this.columnUniqueIds = null; } public long getIndexId() { @@ -191,14 +186,6 @@ public void setComment(String comment) { this.comment = comment; } - public List getColumnUniqueIds() { - return columnUniqueIds; - } - - public void setColumnUniqueIds(List columnUniqueIds) { - this.columnUniqueIds = columnUniqueIds; - } - @Override public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); @@ -216,7 +203,7 @@ public int hashCode() { public Index clone() { return new Index(indexId, indexName, new ArrayList<>(columns), - indexType, new HashMap<>(properties), comment, columnUniqueIds); + indexType, new HashMap<>(properties), comment); } @Override @@ -251,7 +238,21 @@ public String toSql() { return sb.toString(); } - public TOlapTableIndex toThrift() { + public List getColumnUniqueIds(List schema) { + List columnUniqueIds = new ArrayList<>(); + if (schema != null) { + for (String columnName : columns) { + for (Column column : schema) { + if (columnName.equalsIgnoreCase(column.getName())) { + columnUniqueIds.add(column.getUniqueId()); + } + } + } + } + return columnUniqueIds; + } + + public TOlapTableIndex toThrift(List indexColumnUniqueIds) { TOlapTableIndex tIndex = new TOlapTableIndex(); tIndex.setIndexId(indexId); tIndex.setIndexName(indexName); @@ -260,15 +261,16 @@ public TOlapTableIndex toThrift() { if (properties != null) { tIndex.setProperties(properties); } - tIndex.setColumnUniqueIds(columnUniqueIds); + tIndex.setColumnUniqueIds(indexColumnUniqueIds); return tIndex; } - public OlapFile.TabletIndexPB toPb(Map columnMap) { + public OlapFile.TabletIndexPB toPb(Map columnMap, List indexColumnUniqueIds) { OlapFile.TabletIndexPB.Builder builder = OlapFile.TabletIndexPB.newBuilder(); builder.setIndexId(indexId); builder.setIndexName(indexName); - for (Integer columnUniqueId : columnUniqueIds) { + + for (Integer columnUniqueId : indexColumnUniqueIds) { Column column = columnMap.get(columnUniqueId); if (column != null) { builder.addColUniqueId(column.getUniqueId()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java index b0d483751d7282..6a30e771a58bc8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java @@ -391,13 +391,6 @@ public void initSchemaColumnUniqueId() { maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE; this.schema.forEach(column -> { column.setUniqueId(incAndGetMaxColUniqueId()); - this.indexes.forEach(index -> { - index.getColumns().forEach(col -> { - if (col.equalsIgnoreCase(column.getName())) { - index.getColumnUniqueIds().add(column.getUniqueId()); - } - }); - }); if (LOG.isDebugEnabled()) { LOG.debug("indexId: {}, column:{}, uniqueId:{}", indexId, column, column.getUniqueId()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index 17bc301c050d83..7867931f071f88 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -321,9 +321,8 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, columnMap.put(column.getUniqueId(), column); } if (indexes != null) { - for (int i = 0; i < indexes.size(); i++) { - Index index = indexes.get(i); - schemaBuilder.addIndex(index.toPb(columnMap)); + for (Index index : indexes) { + schemaBuilder.addIndex(index.toPb(columnMap, index.getColumnUniqueIds(schemaColumns))); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java index 9ee52e5b7a5f28..f3ad6e81ec148f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java @@ -209,6 +209,6 @@ public IndexType getIndexType() { public Index translateToCatalogStyle() { return new Index(Env.getCurrentEnv().getNextId(), name, cols, indexType, properties, - comment, null); + comment); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 74d7de03bab1d5..a6a5607443299c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -1518,7 +1518,7 @@ protected void toThrift(TPlanNode msg) { } for (Index index : olapTable.getIndexes()) { - TOlapTableIndex tIndex = index.toThrift(); + TOlapTableIndex tIndex = index.toThrift(index.getColumnUniqueIds(olapTable.getBaseSchema())); indexDesc.add(tIndex); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index b63da388d20c70..31c15d8a2c1bb5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -316,7 +316,7 @@ public TOlapTableSchemaParam createSchema(long dbId, OlapTable table, Analyzer a indexes = table.getIndexes(); } for (Index index : indexes) { - TOlapTableIndex tIndex = index.toThrift(); + TOlapTableIndex tIndex = index.toThrift(index.getColumnUniqueIds(table.getBaseSchema())); indexDesc.add(tIndex); } TOlapTableIndexSchema indexSchema = new TOlapTableIndexSchema(pair.getKey(), columns, diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java index caf7733165e99a..c11d0e67497dfb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java @@ -103,7 +103,7 @@ public TAlterInvertedIndexReq toThrift() { if (!alterInvertedIndexes.isEmpty()) { List tIndexes = new ArrayList<>(); for (Index index : alterInvertedIndexes) { - tIndexes.add(index.toThrift()); + tIndexes.add(index.toThrift(index.getColumnUniqueIds(schemaColumns))); } req.setAlterInvertedIndexes(tIndexes); } @@ -111,7 +111,7 @@ public TAlterInvertedIndexReq toThrift() { if (existIndexes != null) { List indexDesc = new ArrayList(); for (Index index : existIndexes) { - TOlapTableIndex tIndex = index.toThrift(); + TOlapTableIndex tIndex = index.toThrift(index.getColumnUniqueIds(schemaColumns)); indexDesc.add(tIndex); } req.setIndexesDesc(indexDesc); diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java index 94b73ef286475a..6328bf2f2a0485 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -351,7 +351,7 @@ public TCreateTabletReq toThrift() { } else { tIndexes = new ArrayList<>(); for (Index index : indexes) { - tIndexes.add(index.toThrift()); + tIndexes.add(index.toThrift(index.getColumnUniqueIds(columns))); } } tSchema.setIndexes(tIndexes); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java new file mode 100644 index 00000000000000..776c97ca76b934 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.analysis.IndexDef; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class IndexTest { + + @Test + public void testGetColumnUniqueIds() { + // Create test columns with unique IDs + List schema = new ArrayList<>(); + Column col1 = new Column("col1", Type.INT); + col1.setUniqueId(101); + Column col2 = new Column("col2", Type.VARCHAR); + col2.setUniqueId(102); + Column col3 = new Column("col3", Type.DOUBLE); + col3.setUniqueId(103); + Column specialCol = new Column("special-name!@#", Type.STRING); + specialCol.setUniqueId(104); + Column mixedCaseCol = new Column("MiXeD_CaSe", Type.BIGINT); + mixedCaseCol.setUniqueId(105); + + schema.add(col1); + schema.add(col2); + schema.add(col3); + schema.add(specialCol); + schema.add(mixedCaseCol); + + // Test case 1: Basic column matching + List indexColumns1 = new ArrayList<>(); + indexColumns1.add("col1"); + indexColumns1.add("col3"); + Index index1 = new Index(1, "test_index1", indexColumns1, IndexDef.IndexType.BITMAP, null, null); + + List uniqueIds1 = index1.getColumnUniqueIds(schema); + Assert.assertEquals(2, uniqueIds1.size()); + Assert.assertEquals(Integer.valueOf(101), uniqueIds1.get(0)); + Assert.assertEquals(Integer.valueOf(103), uniqueIds1.get(1)); + + // Test case 2: Case-insensitive matching + List indexColumns2 = new ArrayList<>(); + indexColumns2.add("CoL1"); + indexColumns2.add("COL3"); + Index index2 = new Index(2, "test_index2", indexColumns2, IndexDef.IndexType.BITMAP, null, null); + + List uniqueIds2 = index2.getColumnUniqueIds(schema); + Assert.assertEquals(2, uniqueIds2.size()); + Assert.assertEquals(Integer.valueOf(101), uniqueIds2.get(0)); + Assert.assertEquals(Integer.valueOf(103), uniqueIds2.get(1)); + + // Test case 3: Non-existent column name + List indexColumns3 = new ArrayList<>(); + indexColumns3.add("col1"); + indexColumns3.add("non_existent_column"); + Index index3 = new Index(3, "test_index3", indexColumns3, IndexDef.IndexType.BITMAP, null, null); + + List uniqueIds3 = index3.getColumnUniqueIds(schema); + Assert.assertEquals(1, uniqueIds3.size()); + Assert.assertEquals(Integer.valueOf(101), uniqueIds3.get(0)); + + // Test case 4: Null schema + List uniqueIds4 = index1.getColumnUniqueIds(null); + Assert.assertEquals(0, uniqueIds4.size()); + + // Test case 5: Empty column list + Index emptyColIndex = new Index(5, "empty_col_index", new ArrayList<>(), + IndexDef.IndexType.BITMAP, null, null); + List emptyColUniqueIds = emptyColIndex.getColumnUniqueIds(schema); + Assert.assertEquals(0, emptyColUniqueIds.size()); + + // Test case 6: Empty schema (non-null) + List emptySchemaUniqueIds = index1.getColumnUniqueIds(new ArrayList<>()); + Assert.assertEquals(0, emptySchemaUniqueIds.size()); + + // Test case 7: Duplicate column names + List dupColumns = new ArrayList<>(); + dupColumns.add("col1"); + dupColumns.add("col1"); // Duplicated + dupColumns.add("col2"); + Index dupIndex = new Index(7, "dup_index", dupColumns, IndexDef.IndexType.BITMAP, null, null); + + List dupUniqueIds = dupIndex.getColumnUniqueIds(schema); + Assert.assertEquals(3, dupUniqueIds.size()); + Assert.assertEquals(Integer.valueOf(101), dupUniqueIds.get(0)); + Assert.assertEquals(Integer.valueOf(101), dupUniqueIds.get(1)); + Assert.assertEquals(Integer.valueOf(102), dupUniqueIds.get(2)); + + // Test case 8: Special characters in column names + List specialColList = new ArrayList<>(); + specialColList.add("special-name!@#"); + Index specialIndex = new Index(8, "special_index", specialColList, IndexDef.IndexType.BITMAP, null, null); + + List specialUniqueIds = specialIndex.getColumnUniqueIds(schema); + Assert.assertEquals(1, specialUniqueIds.size()); + Assert.assertEquals(Integer.valueOf(104), specialUniqueIds.get(0)); + + // Test case 9: Mixed case column name + List mixedCaseList = new ArrayList<>(); + mixedCaseList.add("mixed_case"); // Testing case insensitivity with underscores + Index mixedCaseIndex = new Index(9, "mixed_case_index", mixedCaseList, IndexDef.IndexType.BITMAP, null, null); + + List mixedCaseUniqueIds = mixedCaseIndex.getColumnUniqueIds(schema); + Assert.assertEquals(1, mixedCaseUniqueIds.size()); + Assert.assertEquals(Integer.valueOf(105), mixedCaseUniqueIds.get(0)); + + // Test case 10: Large number of columns + List largeColumnList = new ArrayList<>(); + List largeSchema = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + Column tempCol = new Column("col" + i, Type.INT); + tempCol.setUniqueId(1000 + i); + largeSchema.add(tempCol); + + // Add every other column to the index + if (i % 2 == 0) { + largeColumnList.add("col" + i); + } + } + + Index largeIndex = new Index(10, "large_index", largeColumnList, IndexDef.IndexType.BITMAP, null, null); + List largeUniqueIds = largeIndex.getColumnUniqueIds(largeSchema); + + Assert.assertEquals(500, largeUniqueIds.size()); + // Check first and last elements + Assert.assertEquals(Integer.valueOf(1000), largeUniqueIds.get(0)); + Assert.assertEquals(Integer.valueOf(1000 + 998), largeUniqueIds.get(499)); + + // Test case 11: Order preservation - ensure column order in index is preserved in IDs + List reverseOrderColumns = new ArrayList<>(); + reverseOrderColumns.add("col3"); + reverseOrderColumns.add("col2"); + reverseOrderColumns.add("col1"); + + Index reverseIndex = new Index(11, "reverse_index", reverseOrderColumns, IndexDef.IndexType.BITMAP, null, null); + List reverseUniqueIds = reverseIndex.getColumnUniqueIds(schema); + + Assert.assertEquals(3, reverseUniqueIds.size()); + Assert.assertEquals(Integer.valueOf(103), reverseUniqueIds.get(0)); + Assert.assertEquals(Integer.valueOf(102), reverseUniqueIds.get(1)); + Assert.assertEquals(Integer.valueOf(101), reverseUniqueIds.get(2)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java index 84b8c6062d2351..950371de3034bd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java @@ -59,7 +59,7 @@ int getCurrentEnvJournalVersion() { } OlapTable tbl = (OlapTable) table; tbl.setIndexes(Lists.newArrayList(new Index(0, "index", Lists.newArrayList("col"), - IndexDef.IndexType.BITMAP, null, "xxxxxx", Lists.newArrayList(1)))); + IndexDef.IndexType.BITMAP, null, "xxxxxx"))); System.out.println("orig table id: " + tbl.getId()); FastByteArrayOutputStream byteArrayOutputStream = new FastByteArrayOutputStream(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexesProcNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexesProcNodeTest.java index 273915a2d20317..c19bb0fecb967f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexesProcNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexesProcNodeTest.java @@ -42,18 +42,18 @@ public class IndexesProcNodeTest { public void testFetchResult() throws AnalysisException { List indexes = new ArrayList<>(); Index indexBitmap = new Index(1, "bitmap_index", Lists.newArrayList("col_1"), - IndexType.BITMAP, null, "bitmap index on col_1", Lists.newArrayList(1)); + IndexType.BITMAP, null, "bitmap index on col_1"); Map invertedProperties = new HashMap<>(); invertedProperties.put("parser", "unicode"); Index indexInverted = new Index(2, "inverted_index", Lists.newArrayList("col_2"), - IndexType.INVERTED, invertedProperties, "inverted index on col_2", Lists.newArrayList(2)); + IndexType.INVERTED, invertedProperties, "inverted index on col_2"); Index indexBf = new Index(3, "bloomfilter_index", Lists.newArrayList("col_3"), - IndexType.BLOOMFILTER, null, "bloomfilter index on col_3", Lists.newArrayList(3)); + IndexType.BLOOMFILTER, null, "bloomfilter index on col_3"); Map ngramProperties = new HashMap<>(); ngramProperties.put("gram_size", "3"); ngramProperties.put("bf_size", "256"); Index indexNgramBf = new Index(4, "ngram_bf_index", Lists.newArrayList("col_4"), - IndexType.NGRAM_BF, ngramProperties, "ngram_bf index on col_4", Lists.newArrayList(4)); + IndexType.NGRAM_BF, ngramProperties, "ngram_bf index on col_4"); indexes.add(indexBitmap); indexes.add(indexInverted); indexes.add(indexBf); diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java index da48a24c79f321..e3c55f70c0756b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java @@ -71,7 +71,7 @@ public void testSerialization() throws IOException { indexNameToId.put("index", 1L); List indexes = Lists.newArrayList( - new Index(0, "index", Lists.newArrayList("testCol1"), IndexDef.IndexType.INVERTED, null, "xxxxxx", Lists.newArrayList(1))); + new Index(0, "index", Lists.newArrayList("testCol1"), IndexDef.IndexType.INVERTED, null, "xxxxxx")); TableAddOrDropColumnsInfo tableAddOrDropColumnsInfo1 = new TableAddOrDropColumnsInfo( "", dbId, tableId, tableId,