From a40c15696e7f5d3007ae8875f40cb42d63a741f3 Mon Sep 17 00:00:00 2001 From: qidaye Date: Wed, 12 Mar 2025 20:31:44 +0800 Subject: [PATCH 01/12] [fix](Index)Add null check before using columnUniqueIds in Index --- .../java/org/apache/doris/catalog/Index.java | 24 ++++++++++++++----- .../doris/catalog/MaterializedIndexMeta.java | 12 ++++++++++ .../org/apache/doris/catalog/OlapTable.java | 7 ++++++ .../doris/datasource/InternalCatalog.java | 4 ++++ 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index f770de9176f0db..40dcf372879343 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -24,6 +24,7 @@ import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.SqlUtils; +import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.proto.OlapFile; import org.apache.doris.thrift.TIndexType; @@ -50,7 +51,7 @@ * Internal representation of index, including index type, name, columns and comments. * This class will be used in olap table */ -public class Index implements Writable { +public class Index implements Writable, GsonPostProcessable { public static final int INDEX_ID_INIT_VALUE = -1; @SerializedName(value = "i", alternate = {"indexId"}) @@ -209,6 +210,13 @@ public static Index read(DataInput in) throws IOException { return GsonUtils.GSON.fromJson(json, Index.class); } + @Override + public void gsonPostProcess() throws IOException { + if (columnUniqueIds == null) { + columnUniqueIds = Lists.newArrayList(); + } + } + @Override public int hashCode() { return 31 * (indexName.hashCode() + columns.hashCode() + indexType.hashCode()); @@ -260,7 +268,9 @@ public TOlapTableIndex toThrift() { if (properties != null) { tIndex.setProperties(properties); } - tIndex.setColumnUniqueIds(columnUniqueIds); + if (columnUniqueIds != null) { + tIndex.setColumnUniqueIds(columnUniqueIds); + } return tIndex; } @@ -268,10 +278,12 @@ public OlapFile.TabletIndexPB toPb(Map columnMap) { OlapFile.TabletIndexPB.Builder builder = OlapFile.TabletIndexPB.newBuilder(); builder.setIndexId(indexId); builder.setIndexName(indexName); - for (Integer columnUniqueId : columnUniqueIds) { - Column column = columnMap.get(columnUniqueId); - if (column != null) { - builder.addColUniqueId(column.getUniqueId()); + if (columnUniqueIds != null) { + for (Integer columnUniqueId : columnUniqueIds) { + Column column = columnMap.get(columnUniqueId); + if (column != null) { + builder.addColUniqueId(column.getUniqueId()); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java index 5dd5776c761902..6d3febfd8897cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java @@ -401,6 +401,18 @@ public void initSchemaColumnUniqueId() { }); } + public void initIndexColumnUniqueId() { + this.schema.forEach(column -> { + this.indexes.forEach(index -> { + index.getColumns().forEach(col -> { + if (col.equalsIgnoreCase(column.getName())) { + index.getColumnUniqueIds().add(column.getUniqueId()); + } + }); + }); + }); + } + public void initColumnNameMap() { // case insensitive nameToColumn = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index d31a3c3806b3cc..9329fa86aae100 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -3042,6 +3042,13 @@ public void initSchemaColumnUniqueId() { } } + // for older version index without column unique id + public void initIndexColumnUniqueId() { + for (MaterializedIndexMeta indexMeta : indexIdToMeta.values()) { + indexMeta.initIndexColumnUniqueId(); + } + } + public Set getPartitionKeys() { return idToPartition.keySet(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 1795c50d43b39c..9ec993ffd333d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1766,6 +1766,8 @@ public PartitionPersistInfo addPartition(Database db, String tableName, AddParti if (!isCreateTable) { beforeCreatePartitions(db.getId(), olapTable.getId(), partitionIds, indexIds, isCreateTable); } + // init column unique id for indexes + olapTable.initIndexColumnUniqueId(); Partition partition = createPartitionWithIndices(db.getId(), olapTable, partitionId, partitionName, indexIdToMeta, distributionInfo, dataProperty, singlePartitionDesc.getReplicaAlloc(), @@ -3611,6 +3613,8 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti // which is the right behavior. long oldPartitionId = entry.getValue(); long newPartitionId = oldToNewPartitionId.get(oldPartitionId); + // init column unique id for indexes + copiedTbl.initIndexColumnUniqueId(); Partition newPartition = createPartitionWithIndices(db.getId(), copiedTbl, newPartitionId, entry.getKey(), copiedTbl.getIndexIdToMeta(), partitionsDistributionInfo.get(oldPartitionId), From 0400781b0ca6f23abfed85115cdad0fe1c9e9371 Mon Sep 17 00:00:00 2001 From: qidaye Date: Thu, 13 Mar 2025 11:12:09 +0800 Subject: [PATCH 02/12] Add UT --- .../org/apache/doris/catalog/IndexTest.java | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java new file mode 100644 index 00000000000000..bfc04d13281b54 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.analysis.IndexDef; +import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.proto.OlapFile; +import org.apache.doris.thrift.TOlapTableIndex; +import org.apache.doris.thrift.TStorageType; + +import com.google.common.collect.Lists; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class IndexTest { + + @Test + public void testConstructorWithNullColumnUniqueIds() { + // Test constructor with null columnUniqueIds + Index index = new Index(1, "test_index", Lists.newArrayList("col1", "col2"), + IndexDef.IndexType.BITMAP, null, "test comment", null); + + // Should initialize columnUniqueIds as empty list, not null + Assert.assertNotNull(index.getColumnUniqueIds()); + Assert.assertTrue(index.getColumnUniqueIds().isEmpty()); + } + + @Test + public void testGsonPostProcess() throws IOException { + // Create JSON without columnUniqueIds field + String json = "{\"i\":1,\"in\":\"test_index\",\"c\":[\"col1\",\"col2\"],\"it\":\"BITMAP\",\"pt\":{},\"ct\":\"test comment\"}"; + + // Deserialize + Index index = GsonUtils.GSON.fromJson(json, Index.class); + + // Manually call gsonPostProcess to simulate deserialization + index.gsonPostProcess(); + + // columnUniqueIds should be initialized by gsonPostProcess + Assert.assertNotNull(index.getColumnUniqueIds()); + Assert.assertTrue(index.getColumnUniqueIds().isEmpty()); + } + + @Test + public void testToThriftWithNullColumnUniqueIds() { + // Create an index with null columnUniqueIds + Index index = new Index(); + index.setIndexId(1); + index.setIndexName("test_index"); + index.setColumns(Lists.newArrayList("col1", "col2")); + index.setIndexType(IndexDef.IndexType.BITMAP); + index.setProperties(new HashMap<>()); + index.setColumnUniqueIds(null); // explicitly set to null + + // Should not throw NPE when calling toThrift + TOlapTableIndex tIndex = index.toThrift(); + + // Verify basic fields + Assert.assertEquals(1, tIndex.getIndexId()); + Assert.assertEquals("test_index", tIndex.getIndexName()); + Assert.assertEquals(2, tIndex.getColumns().size()); + + // ColumnUniqueIds should not be set when columnUniqueIds is null + Assert.assertFalse(tIndex.isSetColumnUniqueIds()); + } + + @Test + public void testToPbWithNullColumnUniqueIds() { + // Create an index with null columnUniqueIds + Index index = new Index(); + index.setIndexId(1); + index.setIndexName("test_index"); + index.setColumns(Lists.newArrayList("col1", "col2")); + index.setIndexType(IndexDef.IndexType.BITMAP); + index.setProperties(new HashMap<>()); + index.setColumnUniqueIds(null); // explicitly set to null + + Map columnMap = new HashMap<>(); + columnMap.put(100, new Column("col1", Type.INT)); + columnMap.put(101, new Column("col2", Type.STRING)); + + // Should not throw NPE when calling toPb + OlapFile.TabletIndexPB pb = index.toPb(columnMap); + + // Verify basic fields + Assert.assertEquals(1, pb.getIndexId()); + Assert.assertEquals("test_index", pb.getIndexName()); + + // No column unique IDs should be added when columnUniqueIds is null + Assert.assertEquals(0, pb.getColUniqueIdCount()); + } + + @Test + public void testInitIndexColumnUniqueId() { + // Create columns with unique IDs + List columns = new ArrayList<>(); + Column col1 = new Column("col1", Type.INT); + col1.setUniqueId(100); + Column col2 = new Column("col2", Type.STRING); + col2.setUniqueId(101); + columns.add(col1); + columns.add(col2); + + // Create indexes referencing those columns + List indexes = new ArrayList<>(); + Index index1 = new Index(1, "idx1", Lists.newArrayList("col1"), + IndexDef.IndexType.BITMAP, null, null, null); + Index index2 = new Index(2, "idx2", Lists.newArrayList("col2"), + IndexDef.IndexType.INVERTED, null, null, null); + indexes.add(index1); + indexes.add(index2); + + MaterializedIndexMeta indexMeta = new MaterializedIndexMeta(1, columns, 1, 1, (short) 1, + TStorageType.COLUMN, KeysType.DUP_KEYS, null, indexes, "testDb"); + + // Verify columnUniqueIds is initially empty + Assert.assertTrue(index1.getColumnUniqueIds().isEmpty()); + Assert.assertTrue(index2.getColumnUniqueIds().isEmpty()); + + // Initialize column unique IDs + indexMeta.initIndexColumnUniqueId(); + + // Verify columnUniqueIds is populated correctly + Assert.assertEquals(1, index1.getColumnUniqueIds().size()); + Assert.assertEquals(Integer.valueOf(100), index1.getColumnUniqueIds().get(0)); + + Assert.assertEquals(1, index2.getColumnUniqueIds().size()); + Assert.assertEquals(Integer.valueOf(101), index2.getColumnUniqueIds().get(0)); + } + + @Test + public void testOlapTableInitIndexColumnUniqueId() { + // Create columns with unique IDs + List columns = new ArrayList<>(); + Column col1 = new Column("col1", Type.INT); + col1.setUniqueId(100); + Column col2 = new Column("col2", Type.STRING); + col2.setUniqueId(101); + columns.add(col1); + columns.add(col2); + + // Create indexes referencing those columns + List indexes = new ArrayList<>(); + Index index1 = new Index(1, "idx1", Lists.newArrayList("col1"), + IndexDef.IndexType.BITMAP, null, null, null); + Index index2 = new Index(2, "idx2", Lists.newArrayList("col2"), + IndexDef.IndexType.INVERTED, null, null, null); + indexes.add(index1); + indexes.add(index2); + + // Create OlapTable with materialized index using these indexes + OlapTable table = new OlapTable(1, "test_table", columns, KeysType.DUP_KEYS, + new SinglePartitionInfo(), new RandomDistributionInfo()); + + table.setIndexMeta(1, "base_index", columns, 1, 1, (short) 1, + TStorageType.COLUMN, KeysType.DUP_KEYS, indexes); + + // Verify columnUniqueIds is initially empty + Assert.assertTrue(index1.getColumnUniqueIds().isEmpty()); + Assert.assertTrue(index2.getColumnUniqueIds().isEmpty()); + + // Initialize column unique IDs at table level + table.initIndexColumnUniqueId(); + + // Verify columnUniqueIds is populated correctly + Assert.assertEquals(1, index1.getColumnUniqueIds().size()); + Assert.assertEquals(Integer.valueOf(100), index1.getColumnUniqueIds().get(0)); + + Assert.assertEquals(1, index2.getColumnUniqueIds().size()); + Assert.assertEquals(Integer.valueOf(101), index2.getColumnUniqueIds().get(0)); + } +} From 590631bdc501f2363ceefc467003e0feccd6eb39 Mon Sep 17 00:00:00 2001 From: qidaye Date: Thu, 13 Mar 2025 11:26:39 +0800 Subject: [PATCH 03/12] check duplicated column unique id --- .../java/org/apache/doris/catalog/MaterializedIndexMeta.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java index 6d3febfd8897cc..eef555dd0c02c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java @@ -406,7 +406,10 @@ public void initIndexColumnUniqueId() { this.indexes.forEach(index -> { index.getColumns().forEach(col -> { if (col.equalsIgnoreCase(column.getName())) { - index.getColumnUniqueIds().add(column.getUniqueId()); + // Check if this column ID is already added to avoid duplicates + if (!index.getColumnUniqueIds().contains(column.getUniqueId())) { + index.getColumnUniqueIds().add(column.getUniqueId()); + } } }); }); From 3c126ae58c0ab5f9215c679a97bfdc03db5fcd93 Mon Sep 17 00:00:00 2001 From: qidaye Date: Thu, 13 Mar 2025 11:31:35 +0800 Subject: [PATCH 04/12] opt function name --- .../java/org/apache/doris/catalog/MaterializedIndexMeta.java | 2 +- .../src/main/java/org/apache/doris/catalog/OlapTable.java | 4 ++-- .../java/org/apache/doris/datasource/InternalCatalog.java | 4 ++-- .../src/test/java/org/apache/doris/catalog/IndexTest.java | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java index eef555dd0c02c3..307874a42e7c67 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java @@ -401,7 +401,7 @@ public void initSchemaColumnUniqueId() { }); } - public void initIndexColumnUniqueId() { + public void makeSureIndexColumnUniqueIdInitialized() { this.schema.forEach(column -> { this.indexes.forEach(index -> { index.getColumns().forEach(col -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 9329fa86aae100..a0cc0265bd3dc5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -3043,9 +3043,9 @@ public void initSchemaColumnUniqueId() { } // for older version index without column unique id - public void initIndexColumnUniqueId() { + public void makeSureIndexColumnUniqueIdInitialized() { for (MaterializedIndexMeta indexMeta : indexIdToMeta.values()) { - indexMeta.initIndexColumnUniqueId(); + indexMeta.makeSureIndexColumnUniqueIdInitialized(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 9ec993ffd333d4..0996f8c4c969d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1767,7 +1767,7 @@ public PartitionPersistInfo addPartition(Database db, String tableName, AddParti beforeCreatePartitions(db.getId(), olapTable.getId(), partitionIds, indexIds, isCreateTable); } // init column unique id for indexes - olapTable.initIndexColumnUniqueId(); + olapTable.makeSureIndexColumnUniqueIdInitialized(); Partition partition = createPartitionWithIndices(db.getId(), olapTable, partitionId, partitionName, indexIdToMeta, distributionInfo, dataProperty, singlePartitionDesc.getReplicaAlloc(), @@ -3614,7 +3614,7 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti long oldPartitionId = entry.getValue(); long newPartitionId = oldToNewPartitionId.get(oldPartitionId); // init column unique id for indexes - copiedTbl.initIndexColumnUniqueId(); + copiedTbl.makeSureIndexColumnUniqueIdInitialized(); Partition newPartition = createPartitionWithIndices(db.getId(), copiedTbl, newPartitionId, entry.getKey(), copiedTbl.getIndexIdToMeta(), partitionsDistributionInfo.get(oldPartitionId), diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java index bfc04d13281b54..f03cea1b9b7ee8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java @@ -139,7 +139,7 @@ public void testInitIndexColumnUniqueId() { Assert.assertTrue(index2.getColumnUniqueIds().isEmpty()); // Initialize column unique IDs - indexMeta.initIndexColumnUniqueId(); + indexMeta.makeSureIndexColumnUniqueIdInitialized(); // Verify columnUniqueIds is populated correctly Assert.assertEquals(1, index1.getColumnUniqueIds().size()); @@ -181,7 +181,7 @@ public void testOlapTableInitIndexColumnUniqueId() { Assert.assertTrue(index2.getColumnUniqueIds().isEmpty()); // Initialize column unique IDs at table level - table.initIndexColumnUniqueId(); + table.makeSureIndexColumnUniqueIdInitialized(); // Verify columnUniqueIds is populated correctly Assert.assertEquals(1, index1.getColumnUniqueIds().size()); From 4b0bb6d1ad3d4bf4404f40b7caf6678c0928462b Mon Sep 17 00:00:00 2001 From: qidaye Date: Thu, 13 Mar 2025 16:20:06 +0800 Subject: [PATCH 05/12] Get index columnUniqueIds when using --- .../doris/alter/SchemaChangeHandler.java | 2 - .../doris/analysis/BuildIndexClause.java | 2 +- .../doris/analysis/CreateIndexClause.java | 2 +- .../doris/analysis/CreateTableStmt.java | 3 +- .../org/apache/doris/analysis/IndexDef.java | 5 - .../java/org/apache/doris/catalog/Index.java | 51 ++--- .../doris/catalog/MaterializedIndexMeta.java | 22 -- .../org/apache/doris/catalog/OlapTable.java | 13 +- .../datasource/CloudInternalCatalog.java | 2 +- .../doris/datasource/InternalCatalog.java | 4 - .../plans/commands/info/IndexDefinition.java | 2 +- .../apache/doris/planner/OlapScanNode.java | 2 +- .../apache/doris/planner/OlapTableSink.java | 4 +- .../doris/task/AlterInvertedIndexTask.java | 4 +- .../apache/doris/task/CreateReplicaTask.java | 2 +- .../org/apache/doris/catalog/IndexTest.java | 193 ------------------ .../apache/doris/catalog/OlapTableTest.java | 2 +- .../common/proc/IndexesProcNodeTest.java | 8 +- .../TableAddOrDropColumnsInfoTest.java | 2 +- 19 files changed, 42 insertions(+), 283 deletions(-) delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 51671f4aa2926a..14d58bed42665c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -2720,7 +2720,6 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite(), olapTable.getInvertedIndexFileStorageFormat(), disableInvertedIndexV1ForVariant); - indexDef.getColumnUniqueIds().add(column.getUniqueId()); } else { throw new DdlException("index column does not exist in table. invalid column: " + col); } @@ -2731,7 +2730,6 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab // so here update column name in CreateIndexClause after checkColumn for indexDef, // there will use the column name in olapTable instead of the column name in CreateIndexClause. alterIndex.setColumns(indexDef.getColumns()); - alterIndex.setColumnUniqueIds(indexDef.getColumnUniqueIds()); newIndexes.add(alterIndex); return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java index 46520162db7a00..d04d24e86e403a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java @@ -77,7 +77,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException { indexDef.analyze(); this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(), indexDef.getColumns(), indexDef.getIndexType(), - indexDef.getProperties(), indexDef.getComment(), indexDef.getColumnUniqueIds()); + indexDef.getProperties(), indexDef.getComment()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateIndexClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateIndexClause.java index daf4aef214ae82..cb831c267e4a2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateIndexClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateIndexClause.java @@ -82,7 +82,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException { indexDef.analyze(); this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(), indexDef.getColumns(), indexDef.getIndexType(), - indexDef.getProperties(), indexDef.getComment(), indexDef.getColumnUniqueIds()); + indexDef.getProperties(), indexDef.getComment()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index 9232d66050a30e..4838abab81ab7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -623,8 +623,7 @@ public void analyze(Analyzer analyzer) throws UserException { } } indexes.add(new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(), indexDef.getColumns(), - indexDef.getIndexType(), indexDef.getProperties(), indexDef.getComment(), - indexDef.getColumnUniqueIds())); + indexDef.getIndexType(), indexDef.getProperties(), indexDef.getComment())); distinct.add(indexDef.getIndexName()); distinctCol.add(Pair.of(indexDef.getIndexType(), indexDef.getColumns().stream().map(String::toUpperCase).collect(Collectors.toList()))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java index e486ded71049c4..72c921365e1e6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java @@ -43,7 +43,6 @@ public class IndexDef { private Map properties; private boolean isBuildDeferred = false; private PartitionNames partitionNames; - private List columnUniqueIds = Lists.newArrayList(); public static final int MIN_NGRAM_SIZE = 1; public static final int MAX_NGRAM_SIZE = 255; public static final int MIN_BF_SIZE = 64; @@ -203,10 +202,6 @@ public List getPartitionNames() { return partitionNames == null ? Lists.newArrayList() : partitionNames.getPartitionNames(); } - public List getColumnUniqueIds() { - return columnUniqueIds; - } - public enum IndexType { BITMAP, INVERTED, diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index 40dcf372879343..459bfabba463d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -24,7 +24,6 @@ import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.SqlUtils; -import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.proto.OlapFile; import org.apache.doris.thrift.TIndexType; @@ -51,7 +50,7 @@ * Internal representation of index, including index type, name, columns and comments. * This class will be used in olap table */ -public class Index implements Writable, GsonPostProcessable { +public class Index implements Writable { public static final int INDEX_ID_INIT_VALUE = -1; @SerializedName(value = "i", alternate = {"indexId"}) @@ -66,19 +65,15 @@ public class Index implements Writable, GsonPostProcessable { private Map properties; @SerializedName(value = "ct", alternate = {"comment"}) private String comment; - @SerializedName(value = "cui", alternate = {"columnUniqueIds"}) - private List columnUniqueIds; public Index(long indexId, String indexName, List columns, - IndexDef.IndexType indexType, Map properties, String comment, - List columnUniqueIds) { + IndexDef.IndexType indexType, Map properties, String comment) { this.indexId = indexId; this.indexName = indexName; this.columns = columns == null ? Lists.newArrayList() : Lists.newArrayList(columns); this.indexType = indexType; this.properties = properties == null ? Maps.newHashMap() : Maps.newHashMap(properties); this.comment = comment; - this.columnUniqueIds = columnUniqueIds == null ? Lists.newArrayList() : Lists.newArrayList(columnUniqueIds); if (indexType == IndexDef.IndexType.INVERTED) { if (this.properties != null && !this.properties.isEmpty()) { if (this.properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)) { @@ -102,7 +97,6 @@ public Index() { this.indexType = null; this.properties = null; this.comment = null; - this.columnUniqueIds = null; } public long getIndexId() { @@ -192,14 +186,6 @@ public void setComment(String comment) { this.comment = comment; } - public List getColumnUniqueIds() { - return columnUniqueIds; - } - - public void setColumnUniqueIds(List columnUniqueIds) { - this.columnUniqueIds = columnUniqueIds; - } - @Override public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); @@ -210,13 +196,6 @@ public static Index read(DataInput in) throws IOException { return GsonUtils.GSON.fromJson(json, Index.class); } - @Override - public void gsonPostProcess() throws IOException { - if (columnUniqueIds == null) { - columnUniqueIds = Lists.newArrayList(); - } - } - @Override public int hashCode() { return 31 * (indexName.hashCode() + columns.hashCode() + indexType.hashCode()); @@ -224,7 +203,7 @@ public int hashCode() { public Index clone() { return new Index(indexId, indexName, new ArrayList<>(columns), - indexType, new HashMap<>(properties), comment, columnUniqueIds); + indexType, new HashMap<>(properties), comment); } @Override @@ -259,7 +238,12 @@ public String toSql() { return sb.toString(); } - public TOlapTableIndex toThrift() { + private List getIndexColumnUniqueIds(long tableId) { + OlapTable olapTable = (OlapTable) Env.getCurrentInternalCatalog().getTableByTableId(tableId); + return olapTable.getIndexColumnIds(columns); + } + + public TOlapTableIndex toThrift(long tableId) { TOlapTableIndex tIndex = new TOlapTableIndex(); tIndex.setIndexId(indexId); tIndex.setIndexName(indexName); @@ -268,22 +252,19 @@ public TOlapTableIndex toThrift() { if (properties != null) { tIndex.setProperties(properties); } - if (columnUniqueIds != null) { - tIndex.setColumnUniqueIds(columnUniqueIds); - } + tIndex.setColumnUniqueIds(getIndexColumnUniqueIds(tableId)); return tIndex; } - public OlapFile.TabletIndexPB toPb(Map columnMap) { + public OlapFile.TabletIndexPB toPb(Map columnMap, long tableId) { OlapFile.TabletIndexPB.Builder builder = OlapFile.TabletIndexPB.newBuilder(); builder.setIndexId(indexId); builder.setIndexName(indexName); - if (columnUniqueIds != null) { - for (Integer columnUniqueId : columnUniqueIds) { - Column column = columnMap.get(columnUniqueId); - if (column != null) { - builder.addColUniqueId(column.getUniqueId()); - } + + for (Integer columnUniqueId : getIndexColumnUniqueIds(tableId)) { + Column column = columnMap.get(columnUniqueId); + if (column != null) { + builder.addColUniqueId(column.getUniqueId()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java index 307874a42e7c67..6125e0334003c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java @@ -387,13 +387,6 @@ public void initSchemaColumnUniqueId() { maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE; this.schema.forEach(column -> { column.setUniqueId(incAndGetMaxColUniqueId()); - this.indexes.forEach(index -> { - index.getColumns().forEach(col -> { - if (col.equalsIgnoreCase(column.getName())) { - index.getColumnUniqueIds().add(column.getUniqueId()); - } - }); - }); if (LOG.isDebugEnabled()) { LOG.debug("indexId: {}, column:{}, uniqueId:{}", indexId, column, column.getUniqueId()); @@ -401,21 +394,6 @@ public void initSchemaColumnUniqueId() { }); } - public void makeSureIndexColumnUniqueIdInitialized() { - this.schema.forEach(column -> { - this.indexes.forEach(index -> { - index.getColumns().forEach(col -> { - if (col.equalsIgnoreCase(column.getName())) { - // Check if this column ID is already added to avoid duplicates - if (!index.getColumnUniqueIds().contains(column.getUniqueId())) { - index.getColumnUniqueIds().add(column.getUniqueId()); - } - } - }); - }); - }); - } - public void initColumnNameMap() { // case insensitive nameToColumn = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index a0cc0265bd3dc5..dbc346d0b8bf44 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -3042,11 +3042,16 @@ public void initSchemaColumnUniqueId() { } } - // for older version index without column unique id - public void makeSureIndexColumnUniqueIdInitialized() { - for (MaterializedIndexMeta indexMeta : indexIdToMeta.values()) { - indexMeta.makeSureIndexColumnUniqueIdInitialized(); + public List getIndexColumnIds(List columnNames) { + List columnUniqueIds = new ArrayList<>(); + for (Column column : getBaseSchema()) { + for (String columnName : columnNames) { + if (column.getName().equalsIgnoreCase(columnName)) { + columnUniqueIds.add(column.getUniqueId()); + } + } } + return columnUniqueIds; } public Set getPartitionKeys() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index ed7ce0dc4b591b..692f22b9cf625a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -330,7 +330,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, if (indexes != null) { for (int i = 0; i < indexes.size(); i++) { Index index = indexes.get(i); - schemaBuilder.addIndex(index.toPb(columnMap)); + schemaBuilder.addIndex(index.toPb(columnMap, tableId)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 0996f8c4c969d6..1795c50d43b39c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1766,8 +1766,6 @@ public PartitionPersistInfo addPartition(Database db, String tableName, AddParti if (!isCreateTable) { beforeCreatePartitions(db.getId(), olapTable.getId(), partitionIds, indexIds, isCreateTable); } - // init column unique id for indexes - olapTable.makeSureIndexColumnUniqueIdInitialized(); Partition partition = createPartitionWithIndices(db.getId(), olapTable, partitionId, partitionName, indexIdToMeta, distributionInfo, dataProperty, singlePartitionDesc.getReplicaAlloc(), @@ -3613,8 +3611,6 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti // which is the right behavior. long oldPartitionId = entry.getValue(); long newPartitionId = oldToNewPartitionId.get(oldPartitionId); - // init column unique id for indexes - copiedTbl.makeSureIndexColumnUniqueIdInitialized(); Partition newPartition = createPartitionWithIndices(db.getId(), copiedTbl, newPartitionId, entry.getKey(), copiedTbl.getIndexIdToMeta(), partitionsDistributionInfo.get(oldPartitionId), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java index 449f7ee8261090..906b6e1d10ad63 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java @@ -216,7 +216,7 @@ public IndexType getIndexType() { public Index translateToCatalogStyle() { return new Index(Env.getCurrentEnv().getNextId(), name, cols, indexType, properties, - comment, null); + comment); } public IndexDef translateToLegacyIndexDef() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 92175523f227a6..9ebc53ad462f0f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -1560,7 +1560,7 @@ protected void toThrift(TPlanNode msg) { } for (Index index : olapTable.getIndexes()) { - TOlapTableIndex tIndex = index.toThrift(); + TOlapTableIndex tIndex = index.toThrift(olapTable.getId()); indexDesc.add(tIndex); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index 885bc687c27282..6e5b5f3aabef7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -386,7 +386,7 @@ public TOlapTableSchemaParam createSchema(long dbId, OlapTable table, Analyzer a indexes = table.getIndexes(); } for (Index index : indexes) { - TOlapTableIndex tIndex = index.toThrift(); + TOlapTableIndex tIndex = index.toThrift(table.getId()); indexDesc.add(tIndex); } TOlapTableIndexSchema indexSchema = new TOlapTableIndexSchema(pair.getKey(), columns, @@ -472,7 +472,7 @@ private TOlapTableSchemaParam createSchema(long dbId, OlapTable table) throws An indexes = table.getIndexes(); } for (Index index : indexes) { - TOlapTableIndex tIndex = index.toThrift(); + TOlapTableIndex tIndex = index.toThrift(table.getId()); indexDesc.add(tIndex); } TOlapTableIndexSchema indexSchema = new TOlapTableIndexSchema(pair.getKey(), columns, diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java index caf7733165e99a..fa372d0c0ba734 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java @@ -103,7 +103,7 @@ public TAlterInvertedIndexReq toThrift() { if (!alterInvertedIndexes.isEmpty()) { List tIndexes = new ArrayList<>(); for (Index index : alterInvertedIndexes) { - tIndexes.add(index.toThrift()); + tIndexes.add(index.toThrift(tableId)); } req.setAlterInvertedIndexes(tIndexes); } @@ -111,7 +111,7 @@ public TAlterInvertedIndexReq toThrift() { if (existIndexes != null) { List indexDesc = new ArrayList(); for (Index index : existIndexes) { - TOlapTableIndex tIndex = index.toThrift(); + TOlapTableIndex tIndex = index.toThrift(tableId); indexDesc.add(tIndex); } req.setIndexesDesc(indexDesc); diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java index 90e709c96c3e5a..e42eeae266066c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -351,7 +351,7 @@ public TCreateTabletReq toThrift() { } else { tIndexes = new ArrayList<>(); for (Index index : indexes) { - tIndexes.add(index.toThrift()); + tIndexes.add(index.toThrift(tableId)); } } tSchema.setIndexes(tIndexes); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java deleted file mode 100644 index f03cea1b9b7ee8..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java +++ /dev/null @@ -1,193 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.catalog; - -import org.apache.doris.analysis.IndexDef; -import org.apache.doris.persist.gson.GsonUtils; -import org.apache.doris.proto.OlapFile; -import org.apache.doris.thrift.TOlapTableIndex; -import org.apache.doris.thrift.TStorageType; - -import com.google.common.collect.Lists; -import org.junit.Assert; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class IndexTest { - - @Test - public void testConstructorWithNullColumnUniqueIds() { - // Test constructor with null columnUniqueIds - Index index = new Index(1, "test_index", Lists.newArrayList("col1", "col2"), - IndexDef.IndexType.BITMAP, null, "test comment", null); - - // Should initialize columnUniqueIds as empty list, not null - Assert.assertNotNull(index.getColumnUniqueIds()); - Assert.assertTrue(index.getColumnUniqueIds().isEmpty()); - } - - @Test - public void testGsonPostProcess() throws IOException { - // Create JSON without columnUniqueIds field - String json = "{\"i\":1,\"in\":\"test_index\",\"c\":[\"col1\",\"col2\"],\"it\":\"BITMAP\",\"pt\":{},\"ct\":\"test comment\"}"; - - // Deserialize - Index index = GsonUtils.GSON.fromJson(json, Index.class); - - // Manually call gsonPostProcess to simulate deserialization - index.gsonPostProcess(); - - // columnUniqueIds should be initialized by gsonPostProcess - Assert.assertNotNull(index.getColumnUniqueIds()); - Assert.assertTrue(index.getColumnUniqueIds().isEmpty()); - } - - @Test - public void testToThriftWithNullColumnUniqueIds() { - // Create an index with null columnUniqueIds - Index index = new Index(); - index.setIndexId(1); - index.setIndexName("test_index"); - index.setColumns(Lists.newArrayList("col1", "col2")); - index.setIndexType(IndexDef.IndexType.BITMAP); - index.setProperties(new HashMap<>()); - index.setColumnUniqueIds(null); // explicitly set to null - - // Should not throw NPE when calling toThrift - TOlapTableIndex tIndex = index.toThrift(); - - // Verify basic fields - Assert.assertEquals(1, tIndex.getIndexId()); - Assert.assertEquals("test_index", tIndex.getIndexName()); - Assert.assertEquals(2, tIndex.getColumns().size()); - - // ColumnUniqueIds should not be set when columnUniqueIds is null - Assert.assertFalse(tIndex.isSetColumnUniqueIds()); - } - - @Test - public void testToPbWithNullColumnUniqueIds() { - // Create an index with null columnUniqueIds - Index index = new Index(); - index.setIndexId(1); - index.setIndexName("test_index"); - index.setColumns(Lists.newArrayList("col1", "col2")); - index.setIndexType(IndexDef.IndexType.BITMAP); - index.setProperties(new HashMap<>()); - index.setColumnUniqueIds(null); // explicitly set to null - - Map columnMap = new HashMap<>(); - columnMap.put(100, new Column("col1", Type.INT)); - columnMap.put(101, new Column("col2", Type.STRING)); - - // Should not throw NPE when calling toPb - OlapFile.TabletIndexPB pb = index.toPb(columnMap); - - // Verify basic fields - Assert.assertEquals(1, pb.getIndexId()); - Assert.assertEquals("test_index", pb.getIndexName()); - - // No column unique IDs should be added when columnUniqueIds is null - Assert.assertEquals(0, pb.getColUniqueIdCount()); - } - - @Test - public void testInitIndexColumnUniqueId() { - // Create columns with unique IDs - List columns = new ArrayList<>(); - Column col1 = new Column("col1", Type.INT); - col1.setUniqueId(100); - Column col2 = new Column("col2", Type.STRING); - col2.setUniqueId(101); - columns.add(col1); - columns.add(col2); - - // Create indexes referencing those columns - List indexes = new ArrayList<>(); - Index index1 = new Index(1, "idx1", Lists.newArrayList("col1"), - IndexDef.IndexType.BITMAP, null, null, null); - Index index2 = new Index(2, "idx2", Lists.newArrayList("col2"), - IndexDef.IndexType.INVERTED, null, null, null); - indexes.add(index1); - indexes.add(index2); - - MaterializedIndexMeta indexMeta = new MaterializedIndexMeta(1, columns, 1, 1, (short) 1, - TStorageType.COLUMN, KeysType.DUP_KEYS, null, indexes, "testDb"); - - // Verify columnUniqueIds is initially empty - Assert.assertTrue(index1.getColumnUniqueIds().isEmpty()); - Assert.assertTrue(index2.getColumnUniqueIds().isEmpty()); - - // Initialize column unique IDs - indexMeta.makeSureIndexColumnUniqueIdInitialized(); - - // Verify columnUniqueIds is populated correctly - Assert.assertEquals(1, index1.getColumnUniqueIds().size()); - Assert.assertEquals(Integer.valueOf(100), index1.getColumnUniqueIds().get(0)); - - Assert.assertEquals(1, index2.getColumnUniqueIds().size()); - Assert.assertEquals(Integer.valueOf(101), index2.getColumnUniqueIds().get(0)); - } - - @Test - public void testOlapTableInitIndexColumnUniqueId() { - // Create columns with unique IDs - List columns = new ArrayList<>(); - Column col1 = new Column("col1", Type.INT); - col1.setUniqueId(100); - Column col2 = new Column("col2", Type.STRING); - col2.setUniqueId(101); - columns.add(col1); - columns.add(col2); - - // Create indexes referencing those columns - List indexes = new ArrayList<>(); - Index index1 = new Index(1, "idx1", Lists.newArrayList("col1"), - IndexDef.IndexType.BITMAP, null, null, null); - Index index2 = new Index(2, "idx2", Lists.newArrayList("col2"), - IndexDef.IndexType.INVERTED, null, null, null); - indexes.add(index1); - indexes.add(index2); - - // Create OlapTable with materialized index using these indexes - OlapTable table = new OlapTable(1, "test_table", columns, KeysType.DUP_KEYS, - new SinglePartitionInfo(), new RandomDistributionInfo()); - - table.setIndexMeta(1, "base_index", columns, 1, 1, (short) 1, - TStorageType.COLUMN, KeysType.DUP_KEYS, indexes); - - // Verify columnUniqueIds is initially empty - Assert.assertTrue(index1.getColumnUniqueIds().isEmpty()); - Assert.assertTrue(index2.getColumnUniqueIds().isEmpty()); - - // Initialize column unique IDs at table level - table.makeSureIndexColumnUniqueIdInitialized(); - - // Verify columnUniqueIds is populated correctly - Assert.assertEquals(1, index1.getColumnUniqueIds().size()); - Assert.assertEquals(Integer.valueOf(100), index1.getColumnUniqueIds().get(0)); - - Assert.assertEquals(1, index2.getColumnUniqueIds().size()); - Assert.assertEquals(Integer.valueOf(101), index2.getColumnUniqueIds().get(0)); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java index 218042e3aa2872..d5da34f74cf85a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java @@ -65,7 +65,7 @@ int getCurrentEnvJournalVersion() { } OlapTable tbl = (OlapTable) table; tbl.setIndexes(Lists.newArrayList(new Index(0, "index", Lists.newArrayList("col"), - IndexDef.IndexType.BITMAP, null, "xxxxxx", Lists.newArrayList(1)))); + IndexDef.IndexType.BITMAP, null, "xxxxxx"))); System.out.println("orig table id: " + tbl.getId()); FastByteArrayOutputStream byteArrayOutputStream = new FastByteArrayOutputStream(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexesProcNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexesProcNodeTest.java index 273915a2d20317..c19bb0fecb967f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexesProcNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexesProcNodeTest.java @@ -42,18 +42,18 @@ public class IndexesProcNodeTest { public void testFetchResult() throws AnalysisException { List indexes = new ArrayList<>(); Index indexBitmap = new Index(1, "bitmap_index", Lists.newArrayList("col_1"), - IndexType.BITMAP, null, "bitmap index on col_1", Lists.newArrayList(1)); + IndexType.BITMAP, null, "bitmap index on col_1"); Map invertedProperties = new HashMap<>(); invertedProperties.put("parser", "unicode"); Index indexInverted = new Index(2, "inverted_index", Lists.newArrayList("col_2"), - IndexType.INVERTED, invertedProperties, "inverted index on col_2", Lists.newArrayList(2)); + IndexType.INVERTED, invertedProperties, "inverted index on col_2"); Index indexBf = new Index(3, "bloomfilter_index", Lists.newArrayList("col_3"), - IndexType.BLOOMFILTER, null, "bloomfilter index on col_3", Lists.newArrayList(3)); + IndexType.BLOOMFILTER, null, "bloomfilter index on col_3"); Map ngramProperties = new HashMap<>(); ngramProperties.put("gram_size", "3"); ngramProperties.put("bf_size", "256"); Index indexNgramBf = new Index(4, "ngram_bf_index", Lists.newArrayList("col_4"), - IndexType.NGRAM_BF, ngramProperties, "ngram_bf index on col_4", Lists.newArrayList(4)); + IndexType.NGRAM_BF, ngramProperties, "ngram_bf index on col_4"); indexes.add(indexBitmap); indexes.add(indexInverted); indexes.add(indexBf); diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java index 4df2759e9df92a..958f01c4e25f83 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java @@ -68,7 +68,7 @@ public void testSerialization() throws IOException { oldIndexSchemaMap.put(tableId, fullSchema); List indexes = Lists.newArrayList( - new Index(0, "index", Lists.newArrayList("testCol1"), IndexDef.IndexType.INVERTED, null, "xxxxxx", Lists.newArrayList(1))); + new Index(0, "index", Lists.newArrayList("testCol1"), IndexDef.IndexType.INVERTED, null, "xxxxxx")); TableAddOrDropColumnsInfo tableAddOrDropColumnsInfo1 = new TableAddOrDropColumnsInfo("", dbId, tableId, indexSchemaMap, oldIndexSchemaMap, indexes, jobId); From f8a9c9d77aa3b02b81185a0e6848ab1c5f5166ef Mon Sep 17 00:00:00 2001 From: qidaye Date: Thu, 13 Mar 2025 18:34:25 +0800 Subject: [PATCH 06/12] opt --- .../apache/doris/alter/IndexChangeJob.java | 14 ++++++++++++-- .../apache/doris/alter/SchemaChangeJobV2.java | 9 ++++++++- .../org/apache/doris/backup/RestoreJob.java | 9 ++++++++- .../java/org/apache/doris/catalog/Index.java | 4 ++-- .../doris/datasource/InternalCatalog.java | 9 ++++++++- .../apache/doris/master/ReportHandler.java | 9 ++++++++- .../apache/doris/planner/OlapScanNode.java | 2 +- .../apache/doris/planner/OlapTableSink.java | 4 ++-- .../doris/task/AlterInvertedIndexTask.java | 19 ++++++++++--------- .../apache/doris/task/CreateReplicaTask.java | 13 +++++++------ 10 files changed, 66 insertions(+), 26 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java b/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java index e00587bee16d98..4dc264d35fd01b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java @@ -55,7 +55,9 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class IndexChangeJob implements Writable { @@ -318,6 +320,14 @@ protected void runWaitingTxnJob() throws AlterCancelException { int originSchemaHash = olapTable.getSchemaHashByIndexId(originIndexId); Partition partition = olapTable.getPartition(partitionId); MaterializedIndex origIdx = partition.getIndex(originIndexId); + Map> existIndexToColumnUniqueIds = new HashMap<>(); + Map> alterIndexToColumnUniqueIds = new HashMap<>(); + for (Index idx : olapTable.getIndexes()) { + existIndexToColumnUniqueIds.put(idx, olapTable.getIndexColumnIds(idx.getColumns())); + } + for (Index idx : alterInvertedIndexes) { + alterIndexToColumnUniqueIds.put(idx, olapTable.getIndexColumnIds(idx.getColumns())); + } for (Tablet originTablet : origIdx.getTablets()) { long taskSignature = Env.getCurrentEnv().getNextId(); long originTabletId = originTablet.getId(); @@ -332,8 +342,8 @@ protected void runWaitingTxnJob() throws AlterCancelException { AlterInvertedIndexTask alterInvertedIndexTask = new AlterInvertedIndexTask( originReplica.getBackendIdWithoutException(), db.getId(), olapTable.getId(), partitionId, originIndexId, originTabletId, - originSchemaHash, olapTable.getIndexes(), - alterInvertedIndexes, originSchemaColumns, + originSchemaHash, existIndexToColumnUniqueIds, + alterIndexToColumnUniqueIds, originSchemaColumns, isDropOp, taskSignature, jobId); invertedIndexBatchTask.addTask(alterInvertedIndexTask); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index a37ffc863d75f4..ce31d664a2ffe2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -287,6 +287,13 @@ protected void createShadowIndexReplica() throws AlterCancelException { KeysType originKeysType = tbl.getKeysTypeByIndexId(originIndexId); List tabletIndexes = originIndexId == tbl.getBaseIndexId() ? indexes : null; + Map> indexListMap = null; + if (tabletIndexes != null) { + indexListMap = new HashMap<>(); + for (Index idx : tabletIndexes) { + indexListMap.put(idx, tbl.getIndexColumnIds(idx.getColumns())); + } + } for (Tablet shadowTablet : shadowIdx.getTablets()) { long shadowTabletId = shadowTablet.getId(); @@ -300,7 +307,7 @@ protected void createShadowIndexReplica() throws AlterCancelException { shadowReplicaId, shadowShortKeyColumnCount, shadowSchemaHash, Partition.PARTITION_INIT_VERSION, originKeysType, TStorageType.COLUMN, storageMedium, - shadowSchema, bfColumns, bfFpp, countDownLatch, tabletIndexes, + shadowSchema, bfColumns, bfFpp, countDownLatch, indexListMap, tbl.isInMemory(), tbl.getPartitionInfo().getTabletType(partitionId), null, diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 2353c61ab8d4e5..8a3c2ca71b8606 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -1342,6 +1342,13 @@ private void createReplicas(Database db, AgentBatchTask batchTask, OlapTable loc MaterializedIndexMeta indexMeta = localTbl.getIndexMetaByIndexId(restoredIdx.getId()); List indexes = restoredIdx.getId() == localTbl.getBaseIndexId() ? localTbl.getCopiedIndexes() : null; + Map> indexListMap = null; + if (indexes != null) { + indexListMap = new HashMap<>(); + for (Index idx : indexes) { + indexListMap.put(idx, localTbl.getIndexColumnIds(idx.getColumns())); + } + } List clusterKeyUids = null; if (indexMeta.getIndexId() == localTbl.getBaseIndexId() || localTbl.isShadowIndex(indexMeta.getIndexId())) { clusterKeyUids = OlapTable.getClusterKeyUids(indexMeta.getSchema()); @@ -1363,7 +1370,7 @@ private void createReplicas(Database db, AgentBatchTask batchTask, OlapTable loc indexMeta.getKeysType(), TStorageType.COLUMN, storageMedium, indexMeta.getSchema(), bfColumns, bfFpp, null, - indexes, + indexListMap, localTbl.isInMemory(), localTbl.getPartitionInfo().getTabletType(restorePart.getId()), null, diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index 459bfabba463d9..e8dccbb8f57e65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -243,7 +243,7 @@ private List getIndexColumnUniqueIds(long tableId) { return olapTable.getIndexColumnIds(columns); } - public TOlapTableIndex toThrift(long tableId) { + public TOlapTableIndex toThrift(List indexColumnUniqueIds) { TOlapTableIndex tIndex = new TOlapTableIndex(); tIndex.setIndexId(indexId); tIndex.setIndexName(indexName); @@ -252,7 +252,7 @@ public TOlapTableIndex toThrift(long tableId) { if (properties != null) { tIndex.setProperties(properties); } - tIndex.setColumnUniqueIds(getIndexColumnUniqueIds(tableId)); + tIndex.setColumnUniqueIds(indexColumnUniqueIds); return tIndex; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 1795c50d43b39c..a7f135a2cd9dec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2153,6 +2153,13 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa } KeysType keysType = indexMeta.getKeysType(); List indexes = indexId == tbl.getBaseIndexId() ? tbl.getCopiedIndexes() : null; + Map> indexListMap = null; + if (indexes != null) { + indexListMap = new HashMap<>(); + for (Index idx : indexes) { + indexListMap.put(idx, tbl.getIndexColumnIds(idx.getColumns())); + } + } int totalTaskNum = index.getTablets().size() * totalReplicaNum; MarkedCountDownLatch countDownLatch = new MarkedCountDownLatch(totalTaskNum); AgentBatchTask batchTask = new AgentBatchTask(); @@ -2166,7 +2173,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa CreateReplicaTask task = new CreateReplicaTask(backendId, dbId, tbl.getId(), partitionId, indexId, tabletId, replicaId, shortKeyColumnCount, schemaHash, version, keysType, storageType, realStorageMedium, schema, bfColumns, tbl.getBfFpp(), countDownLatch, - indexes, tbl.isInMemory(), tabletType, + indexListMap, tbl.isInMemory(), tabletType, tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getEnableUniqueKeyMergeOnWrite(), storagePolicy, tbl.disableAutoCompaction(), tbl.enableSingleReplicaCompaction(), tbl.skipWriteIndexOnLoad(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java index cae089f38dd285..725264c8ea25eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java @@ -949,6 +949,13 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta double bfFpp = olapTable.getBfFpp(); List indexes = indexId == olapTable.getBaseIndexId() ? olapTable.getCopiedIndexes() : null; + Map> indexListMap = null; + if (indexes != null) { + indexListMap = new HashMap<>(); + for (Index idx : indexes) { + indexListMap.put(idx, olapTable.getIndexColumnIds(idx.getColumns())); + } + } List rowStoreColumns = olapTable.getTableProperty().getCopiedRowStoreColumns(); CreateReplicaTask createReplicaTask = new CreateReplicaTask(backendId, dbId, @@ -958,7 +965,7 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta indexMeta.getKeysType(), TStorageType.COLUMN, TStorageMedium.HDD, indexMeta.getSchema(), bfColumns, bfFpp, null, - indexes, + indexListMap, olapTable.isInMemory(), olapTable.getPartitionInfo().getTabletType(partitionId), null, diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 9ebc53ad462f0f..b6ec4557a24171 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -1560,7 +1560,7 @@ protected void toThrift(TPlanNode msg) { } for (Index index : olapTable.getIndexes()) { - TOlapTableIndex tIndex = index.toThrift(olapTable.getId()); + TOlapTableIndex tIndex = index.toThrift(olapTable.getIndexColumnIds(index.getColumns())); indexDesc.add(tIndex); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index 6e5b5f3aabef7f..40a8b23c2771e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -386,7 +386,7 @@ public TOlapTableSchemaParam createSchema(long dbId, OlapTable table, Analyzer a indexes = table.getIndexes(); } for (Index index : indexes) { - TOlapTableIndex tIndex = index.toThrift(table.getId()); + TOlapTableIndex tIndex = index.toThrift(table.getIndexColumnIds(index.getColumns())); indexDesc.add(tIndex); } TOlapTableIndexSchema indexSchema = new TOlapTableIndexSchema(pair.getKey(), columns, @@ -472,7 +472,7 @@ private TOlapTableSchemaParam createSchema(long dbId, OlapTable table) throws An indexes = table.getIndexes(); } for (Index index : indexes) { - TOlapTableIndex tIndex = index.toThrift(table.getId()); + TOlapTableIndex tIndex = index.toThrift(table.getIndexColumnIds(index.getColumns())); indexDesc.add(tIndex); } TOlapTableIndexSchema indexSchema = new TOlapTableIndexSchema(pair.getKey(), columns, diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java index fa372d0c0ba734..8a9649c368917a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; /* * This task is used for alter table process, such as rollup and schema change @@ -40,15 +41,15 @@ public class AlterInvertedIndexTask extends AgentTask { private static final Logger LOG = LogManager.getLogger(AlterInvertedIndexTask.class); private long tabletId; private int schemaHash; - private List alterInvertedIndexes; + private Map> alterInvertedIndexes; private List schemaColumns; - private List existIndexes; + private Map> existIndexes; private boolean isDropOp = false; private long jobId; public AlterInvertedIndexTask(long backendId, long dbId, long tableId, long partitionId, long indexId, long tabletId, int schemaHash, - List existIndexes, List alterInvertedIndexes, + Map> existIndexes, Map> alterInvertedIndexes, List schemaColumns, boolean isDropOp, long taskSignature, long jobId) { super(null, backendId, TTaskType.ALTER_INVERTED_INDEX, dbId, tableId, @@ -70,7 +71,7 @@ public int getSchemaHash() { return schemaHash; } - public List getAlterInvertedIndexes() { + public Map> getAlterInvertedIndexes() { return alterInvertedIndexes; } @@ -82,7 +83,7 @@ public String toString() { sb.append("ADD"); } sb.append(" ("); - for (Index alterIndex : alterInvertedIndexes) { + for (Index alterIndex : alterInvertedIndexes.keySet()) { sb.append(alterIndex.getIndexId()); sb.append(": "); sb.append(alterIndex.toString()); @@ -102,16 +103,16 @@ public TAlterInvertedIndexReq toThrift() { if (!alterInvertedIndexes.isEmpty()) { List tIndexes = new ArrayList<>(); - for (Index index : alterInvertedIndexes) { - tIndexes.add(index.toThrift(tableId)); + for (Map.Entry> index : alterInvertedIndexes.entrySet()) { + tIndexes.add(index.getKey().toThrift(index.getValue())); } req.setAlterInvertedIndexes(tIndexes); } if (existIndexes != null) { List indexDesc = new ArrayList(); - for (Index index : existIndexes) { - TOlapTableIndex tIndex = index.toThrift(tableId); + for (Map.Entry> index : existIndexes.entrySet()) { + TOlapTableIndex tIndex = index.getKey().toThrift(index.getValue()); indexDesc.add(tIndex); } req.setIndexesDesc(indexDesc); diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java index e42eeae266066c..40592c05015b17 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -75,7 +75,8 @@ public class CreateReplicaTask extends AgentTask { private double bfFpp; // indexes - private List indexes; + // index -> index column unique ids + private Map> indexes; private boolean isInMemory; @@ -137,7 +138,7 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition KeysType keysType, TStorageType storageType, TStorageMedium storageMedium, List columns, Set bfColumns, double bfFpp, MarkedCountDownLatch latch, - List indexes, + Map> indexes, boolean isInMemory, TTabletType tabletType, DataSortInfo dataSortInfo, @@ -343,15 +344,15 @@ public TCreateTabletReq toThrift() { LOG.debug("cluster key uids={}, table_id={}, tablet_id={}", clusterKeyUids, tableId, tabletId); } } - if (CollectionUtils.isNotEmpty(indexes)) { + if (indexes != null && !indexes.isEmpty()) { List tIndexes = null; - Object value = objectPool.get(indexes); + Object value = objectPool.get(indexes.keySet()); if (value != null) { tIndexes = (List) value; } else { tIndexes = new ArrayList<>(); - for (Index index : indexes) { - tIndexes.add(index.toThrift(tableId)); + for (Map.Entry> index : indexes.entrySet()) { + tIndexes.add(index.getKey().toThrift(index.getValue())); } } tSchema.setIndexes(tIndexes); From 7f335e1698405fffe39366c9ff580e313c2cfe1f Mon Sep 17 00:00:00 2001 From: qidaye Date: Thu, 13 Mar 2025 18:58:38 +0800 Subject: [PATCH 07/12] opt --- .../doris/alter/CloudSchemaChangeJobV2.java | 10 +++++++++- .../java/org/apache/doris/catalog/Index.java | 9 ++------- .../cloud/datasource/CloudInternalCatalog.java | 17 +++++++++++------ 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java index f765dd1bb7fdca..253c0dfe982037 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java @@ -52,6 +52,7 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -251,6 +252,13 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio long originIndexId = indexIdMap.get(shadowIdxId); KeysType originKeysType = tbl.getKeysTypeByIndexId(originIndexId); List tabletIndexes = originIndexId == tbl.getBaseIndexId() ? indexes : null; + Map> indexListMap = null; + if (tabletIndexes != null) { + indexListMap = new HashMap<>(); + for (Index idx : tabletIndexes) { + indexListMap.put(idx, tbl.getIndexColumnIds(idx.getColumns())); + } + } Cloud.CreateTabletsRequest.Builder requestBuilder = Cloud.CreateTabletsRequest.newBuilder(); @@ -261,7 +269,7 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio partitionId, shadowTablet, tbl.getPartitionInfo().getTabletType(partitionId), shadowSchemaHash, originKeysType, shadowShortKeyColumnCount, bfColumns, - bfFpp, tabletIndexes, shadowSchema, tbl.getDataSortInfo(), + bfFpp, indexListMap, shadowSchema, tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getStoragePolicy(), tbl.isInMemory(), true, tbl.getName(), tbl.getTTLSeconds(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index e8dccbb8f57e65..b3cb51a2654b46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -238,11 +238,6 @@ public String toSql() { return sb.toString(); } - private List getIndexColumnUniqueIds(long tableId) { - OlapTable olapTable = (OlapTable) Env.getCurrentInternalCatalog().getTableByTableId(tableId); - return olapTable.getIndexColumnIds(columns); - } - public TOlapTableIndex toThrift(List indexColumnUniqueIds) { TOlapTableIndex tIndex = new TOlapTableIndex(); tIndex.setIndexId(indexId); @@ -256,12 +251,12 @@ public TOlapTableIndex toThrift(List indexColumnUniqueIds) { return tIndex; } - public OlapFile.TabletIndexPB toPb(Map columnMap, long tableId) { + public OlapFile.TabletIndexPB toPb(Map columnMap, List indexColumnUniqueIds) { OlapFile.TabletIndexPB.Builder builder = OlapFile.TabletIndexPB.newBuilder(); builder.setIndexId(indexId); builder.setIndexName(indexName); - for (Integer columnUniqueId : getIndexColumnUniqueIds(tableId)) { + for (Integer columnUniqueId : indexColumnUniqueIds) { Column column = columnMap.get(columnUniqueId); if (column != null) { builder.addColUniqueId(column.getUniqueId()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index 692f22b9cf625a..5cc024e40ad57c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -75,6 +75,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -158,6 +159,10 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa } else { indexes = Lists.newArrayList(); } + Map> indexListMap = new HashMap<>(); + for (Index idx : indexes) { + indexListMap.put(idx, tbl.getIndexColumnIds(idx.getColumns())); + } List clusterKeyUids = null; if (indexId == tbl.getBaseIndexId()) { // only base and shadow index need cluster key unique column ids @@ -169,7 +174,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa for (Tablet tablet : index.getTablets()) { OlapFile.TabletMetaCloudPB.Builder builder = createTabletMetaBuilder(tbl.getId(), indexId, partitionId, tablet, tabletType, schemaHash, keysType, shortKeyColumnCount, - bfColumns, tbl.getBfFpp(), indexes, columns, tbl.getDataSortInfo(), + bfColumns, tbl.getBfFpp(), indexListMap, columns, tbl.getDataSortInfo(), tbl.getCompressionType(), storagePolicy, isInMemory, false, tbl.getName(), tbl.getTTLSeconds(), tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(), indexMeta.getSchemaVersion(), tbl.getCompactionPolicy(), tbl.getTimeSeriesCompactionGoalSizeMbytes(), @@ -205,7 +210,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, long indexId, long partitionId, Tablet tablet, TTabletType tabletType, int schemaHash, KeysType keysType, - short shortKeyColumnCount, Set bfColumns, double bfFpp, List indexes, + short shortKeyColumnCount, Set bfColumns, double bfFpp, Map> indexes, List schemaColumns, DataSortInfo dataSortInfo, TCompressionType compressionType, String storagePolicy, boolean isInMemory, boolean isShadow, String tableName, long ttlSeconds, boolean enableUniqueKeyMergeOnWrite, @@ -320,7 +325,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, schemaBuilder.setSortColNum(dataSortInfo.getColNum()); for (int i = 0; i < schemaColumns.size(); i++) { Column column = schemaColumns.get(i); - schemaBuilder.addColumn(column.toPb(bfColumns, indexes)); + schemaBuilder.addColumn(column.toPb(bfColumns, (List) indexes.keySet())); } Map columnMap = Maps.newHashMap(); @@ -328,9 +333,9 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, columnMap.put(column.getUniqueId(), column); } if (indexes != null) { - for (int i = 0; i < indexes.size(); i++) { - Index index = indexes.get(i); - schemaBuilder.addIndex(index.toPb(columnMap, tableId)); + for (Map.Entry> index : indexes.entrySet()) { + Index idx = index.getKey(); + schemaBuilder.addIndex(idx.toPb(columnMap, index.getValue())); } } From b5097dbef5c3865d6bc80d79f02f91b0eca8621a Mon Sep 17 00:00:00 2001 From: qidaye Date: Thu, 13 Mar 2025 21:21:55 +0800 Subject: [PATCH 08/12] fix --- .../doris/alter/CloudSchemaChangeJobV2.java | 10 +--------- .../apache/doris/alter/IndexChangeJob.java | 14 ++------------ .../apache/doris/alter/SchemaChangeJobV2.java | 9 +-------- .../org/apache/doris/backup/RestoreJob.java | 9 +-------- .../java/org/apache/doris/catalog/Index.java | 12 ++++++++++++ .../datasource/CloudInternalCatalog.java | 16 +++++----------- .../doris/datasource/InternalCatalog.java | 9 +-------- .../apache/doris/master/ReportHandler.java | 9 +-------- .../apache/doris/planner/OlapScanNode.java | 2 +- .../apache/doris/planner/OlapTableSink.java | 4 ++-- .../doris/task/AlterInvertedIndexTask.java | 19 +++++++++---------- .../apache/doris/task/CreateReplicaTask.java | 13 ++++++------- 12 files changed, 42 insertions(+), 84 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java index 253c0dfe982037..f765dd1bb7fdca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java @@ -52,7 +52,6 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -252,13 +251,6 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio long originIndexId = indexIdMap.get(shadowIdxId); KeysType originKeysType = tbl.getKeysTypeByIndexId(originIndexId); List tabletIndexes = originIndexId == tbl.getBaseIndexId() ? indexes : null; - Map> indexListMap = null; - if (tabletIndexes != null) { - indexListMap = new HashMap<>(); - for (Index idx : tabletIndexes) { - indexListMap.put(idx, tbl.getIndexColumnIds(idx.getColumns())); - } - } Cloud.CreateTabletsRequest.Builder requestBuilder = Cloud.CreateTabletsRequest.newBuilder(); @@ -269,7 +261,7 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio partitionId, shadowTablet, tbl.getPartitionInfo().getTabletType(partitionId), shadowSchemaHash, originKeysType, shadowShortKeyColumnCount, bfColumns, - bfFpp, indexListMap, shadowSchema, tbl.getDataSortInfo(), + bfFpp, tabletIndexes, shadowSchema, tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getStoragePolicy(), tbl.isInMemory(), true, tbl.getName(), tbl.getTTLSeconds(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java b/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java index 4dc264d35fd01b..e00587bee16d98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java @@ -55,9 +55,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.HashMap; import java.util.List; -import java.util.Map; public class IndexChangeJob implements Writable { @@ -320,14 +318,6 @@ protected void runWaitingTxnJob() throws AlterCancelException { int originSchemaHash = olapTable.getSchemaHashByIndexId(originIndexId); Partition partition = olapTable.getPartition(partitionId); MaterializedIndex origIdx = partition.getIndex(originIndexId); - Map> existIndexToColumnUniqueIds = new HashMap<>(); - Map> alterIndexToColumnUniqueIds = new HashMap<>(); - for (Index idx : olapTable.getIndexes()) { - existIndexToColumnUniqueIds.put(idx, olapTable.getIndexColumnIds(idx.getColumns())); - } - for (Index idx : alterInvertedIndexes) { - alterIndexToColumnUniqueIds.put(idx, olapTable.getIndexColumnIds(idx.getColumns())); - } for (Tablet originTablet : origIdx.getTablets()) { long taskSignature = Env.getCurrentEnv().getNextId(); long originTabletId = originTablet.getId(); @@ -342,8 +332,8 @@ protected void runWaitingTxnJob() throws AlterCancelException { AlterInvertedIndexTask alterInvertedIndexTask = new AlterInvertedIndexTask( originReplica.getBackendIdWithoutException(), db.getId(), olapTable.getId(), partitionId, originIndexId, originTabletId, - originSchemaHash, existIndexToColumnUniqueIds, - alterIndexToColumnUniqueIds, originSchemaColumns, + originSchemaHash, olapTable.getIndexes(), + alterInvertedIndexes, originSchemaColumns, isDropOp, taskSignature, jobId); invertedIndexBatchTask.addTask(alterInvertedIndexTask); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index ce31d664a2ffe2..a37ffc863d75f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -287,13 +287,6 @@ protected void createShadowIndexReplica() throws AlterCancelException { KeysType originKeysType = tbl.getKeysTypeByIndexId(originIndexId); List tabletIndexes = originIndexId == tbl.getBaseIndexId() ? indexes : null; - Map> indexListMap = null; - if (tabletIndexes != null) { - indexListMap = new HashMap<>(); - for (Index idx : tabletIndexes) { - indexListMap.put(idx, tbl.getIndexColumnIds(idx.getColumns())); - } - } for (Tablet shadowTablet : shadowIdx.getTablets()) { long shadowTabletId = shadowTablet.getId(); @@ -307,7 +300,7 @@ protected void createShadowIndexReplica() throws AlterCancelException { shadowReplicaId, shadowShortKeyColumnCount, shadowSchemaHash, Partition.PARTITION_INIT_VERSION, originKeysType, TStorageType.COLUMN, storageMedium, - shadowSchema, bfColumns, bfFpp, countDownLatch, indexListMap, + shadowSchema, bfColumns, bfFpp, countDownLatch, tabletIndexes, tbl.isInMemory(), tbl.getPartitionInfo().getTabletType(partitionId), null, diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 8a3c2ca71b8606..2353c61ab8d4e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -1342,13 +1342,6 @@ private void createReplicas(Database db, AgentBatchTask batchTask, OlapTable loc MaterializedIndexMeta indexMeta = localTbl.getIndexMetaByIndexId(restoredIdx.getId()); List indexes = restoredIdx.getId() == localTbl.getBaseIndexId() ? localTbl.getCopiedIndexes() : null; - Map> indexListMap = null; - if (indexes != null) { - indexListMap = new HashMap<>(); - for (Index idx : indexes) { - indexListMap.put(idx, localTbl.getIndexColumnIds(idx.getColumns())); - } - } List clusterKeyUids = null; if (indexMeta.getIndexId() == localTbl.getBaseIndexId() || localTbl.isShadowIndex(indexMeta.getIndexId())) { clusterKeyUids = OlapTable.getClusterKeyUids(indexMeta.getSchema()); @@ -1370,7 +1363,7 @@ private void createReplicas(Database db, AgentBatchTask batchTask, OlapTable loc indexMeta.getKeysType(), TStorageType.COLUMN, storageMedium, indexMeta.getSchema(), bfColumns, bfFpp, null, - indexListMap, + indexes, localTbl.isInMemory(), localTbl.getPartitionInfo().getTabletType(restorePart.getId()), null, diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index b3cb51a2654b46..dd24a84e9b2300 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -238,6 +238,18 @@ public String toSql() { return sb.toString(); } + public List getColumnUniqueIds(List schema) { + List columnUniqueIds = new ArrayList<>(); + for (String columnName : columns) { + for (Column column : schema) { + if (columnName.equalsIgnoreCase(column.getName())) { + columnUniqueIds.add(column.getUniqueId()); + } + } + } + return columnUniqueIds; + } + public TOlapTableIndex toThrift(List indexColumnUniqueIds) { TOlapTableIndex tIndex = new TOlapTableIndex(); tIndex.setIndexId(indexId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index 5cc024e40ad57c..fa05947b07a448 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -75,7 +75,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -159,10 +158,6 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa } else { indexes = Lists.newArrayList(); } - Map> indexListMap = new HashMap<>(); - for (Index idx : indexes) { - indexListMap.put(idx, tbl.getIndexColumnIds(idx.getColumns())); - } List clusterKeyUids = null; if (indexId == tbl.getBaseIndexId()) { // only base and shadow index need cluster key unique column ids @@ -174,7 +169,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa for (Tablet tablet : index.getTablets()) { OlapFile.TabletMetaCloudPB.Builder builder = createTabletMetaBuilder(tbl.getId(), indexId, partitionId, tablet, tabletType, schemaHash, keysType, shortKeyColumnCount, - bfColumns, tbl.getBfFpp(), indexListMap, columns, tbl.getDataSortInfo(), + bfColumns, tbl.getBfFpp(), indexes, columns, tbl.getDataSortInfo(), tbl.getCompressionType(), storagePolicy, isInMemory, false, tbl.getName(), tbl.getTTLSeconds(), tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(), indexMeta.getSchemaVersion(), tbl.getCompactionPolicy(), tbl.getTimeSeriesCompactionGoalSizeMbytes(), @@ -210,7 +205,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, long indexId, long partitionId, Tablet tablet, TTabletType tabletType, int schemaHash, KeysType keysType, - short shortKeyColumnCount, Set bfColumns, double bfFpp, Map> indexes, + short shortKeyColumnCount, Set bfColumns, double bfFpp, List indexes, List schemaColumns, DataSortInfo dataSortInfo, TCompressionType compressionType, String storagePolicy, boolean isInMemory, boolean isShadow, String tableName, long ttlSeconds, boolean enableUniqueKeyMergeOnWrite, @@ -325,7 +320,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, schemaBuilder.setSortColNum(dataSortInfo.getColNum()); for (int i = 0; i < schemaColumns.size(); i++) { Column column = schemaColumns.get(i); - schemaBuilder.addColumn(column.toPb(bfColumns, (List) indexes.keySet())); + schemaBuilder.addColumn(column.toPb(bfColumns, (List) indexes)); } Map columnMap = Maps.newHashMap(); @@ -333,9 +328,8 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, columnMap.put(column.getUniqueId(), column); } if (indexes != null) { - for (Map.Entry> index : indexes.entrySet()) { - Index idx = index.getKey(); - schemaBuilder.addIndex(idx.toPb(columnMap, index.getValue())); + for (Index index : indexes) { + schemaBuilder.addIndex(index.toPb(columnMap, index.getColumnUniqueIds(schemaColumns))); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index a7f135a2cd9dec..1795c50d43b39c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2153,13 +2153,6 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa } KeysType keysType = indexMeta.getKeysType(); List indexes = indexId == tbl.getBaseIndexId() ? tbl.getCopiedIndexes() : null; - Map> indexListMap = null; - if (indexes != null) { - indexListMap = new HashMap<>(); - for (Index idx : indexes) { - indexListMap.put(idx, tbl.getIndexColumnIds(idx.getColumns())); - } - } int totalTaskNum = index.getTablets().size() * totalReplicaNum; MarkedCountDownLatch countDownLatch = new MarkedCountDownLatch(totalTaskNum); AgentBatchTask batchTask = new AgentBatchTask(); @@ -2173,7 +2166,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa CreateReplicaTask task = new CreateReplicaTask(backendId, dbId, tbl.getId(), partitionId, indexId, tabletId, replicaId, shortKeyColumnCount, schemaHash, version, keysType, storageType, realStorageMedium, schema, bfColumns, tbl.getBfFpp(), countDownLatch, - indexListMap, tbl.isInMemory(), tabletType, + indexes, tbl.isInMemory(), tabletType, tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getEnableUniqueKeyMergeOnWrite(), storagePolicy, tbl.disableAutoCompaction(), tbl.enableSingleReplicaCompaction(), tbl.skipWriteIndexOnLoad(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java index 725264c8ea25eb..cae089f38dd285 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java @@ -949,13 +949,6 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta double bfFpp = olapTable.getBfFpp(); List indexes = indexId == olapTable.getBaseIndexId() ? olapTable.getCopiedIndexes() : null; - Map> indexListMap = null; - if (indexes != null) { - indexListMap = new HashMap<>(); - for (Index idx : indexes) { - indexListMap.put(idx, olapTable.getIndexColumnIds(idx.getColumns())); - } - } List rowStoreColumns = olapTable.getTableProperty().getCopiedRowStoreColumns(); CreateReplicaTask createReplicaTask = new CreateReplicaTask(backendId, dbId, @@ -965,7 +958,7 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta indexMeta.getKeysType(), TStorageType.COLUMN, TStorageMedium.HDD, indexMeta.getSchema(), bfColumns, bfFpp, null, - indexListMap, + indexes, olapTable.isInMemory(), olapTable.getPartitionInfo().getTabletType(partitionId), null, diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index b6ec4557a24171..19c4aa3863223a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -1560,7 +1560,7 @@ protected void toThrift(TPlanNode msg) { } for (Index index : olapTable.getIndexes()) { - TOlapTableIndex tIndex = index.toThrift(olapTable.getIndexColumnIds(index.getColumns())); + TOlapTableIndex tIndex = index.toThrift(index.getColumnUniqueIds(olapTable.getBaseSchema())); indexDesc.add(tIndex); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index 40a8b23c2771e9..5bd95264c55ea1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -386,7 +386,7 @@ public TOlapTableSchemaParam createSchema(long dbId, OlapTable table, Analyzer a indexes = table.getIndexes(); } for (Index index : indexes) { - TOlapTableIndex tIndex = index.toThrift(table.getIndexColumnIds(index.getColumns())); + TOlapTableIndex tIndex = index.toThrift(index.getColumnUniqueIds(table.getBaseSchema())); indexDesc.add(tIndex); } TOlapTableIndexSchema indexSchema = new TOlapTableIndexSchema(pair.getKey(), columns, @@ -472,7 +472,7 @@ private TOlapTableSchemaParam createSchema(long dbId, OlapTable table) throws An indexes = table.getIndexes(); } for (Index index : indexes) { - TOlapTableIndex tIndex = index.toThrift(table.getIndexColumnIds(index.getColumns())); + TOlapTableIndex tIndex = index.toThrift(index.getColumnUniqueIds(table.getBaseSchema())); indexDesc.add(tIndex); } TOlapTableIndexSchema indexSchema = new TOlapTableIndexSchema(pair.getKey(), columns, diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java index 8a9649c368917a..c11d0e67497dfb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java @@ -29,7 +29,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Map; /* * This task is used for alter table process, such as rollup and schema change @@ -41,15 +40,15 @@ public class AlterInvertedIndexTask extends AgentTask { private static final Logger LOG = LogManager.getLogger(AlterInvertedIndexTask.class); private long tabletId; private int schemaHash; - private Map> alterInvertedIndexes; + private List alterInvertedIndexes; private List schemaColumns; - private Map> existIndexes; + private List existIndexes; private boolean isDropOp = false; private long jobId; public AlterInvertedIndexTask(long backendId, long dbId, long tableId, long partitionId, long indexId, long tabletId, int schemaHash, - Map> existIndexes, Map> alterInvertedIndexes, + List existIndexes, List alterInvertedIndexes, List schemaColumns, boolean isDropOp, long taskSignature, long jobId) { super(null, backendId, TTaskType.ALTER_INVERTED_INDEX, dbId, tableId, @@ -71,7 +70,7 @@ public int getSchemaHash() { return schemaHash; } - public Map> getAlterInvertedIndexes() { + public List getAlterInvertedIndexes() { return alterInvertedIndexes; } @@ -83,7 +82,7 @@ public String toString() { sb.append("ADD"); } sb.append(" ("); - for (Index alterIndex : alterInvertedIndexes.keySet()) { + for (Index alterIndex : alterInvertedIndexes) { sb.append(alterIndex.getIndexId()); sb.append(": "); sb.append(alterIndex.toString()); @@ -103,16 +102,16 @@ public TAlterInvertedIndexReq toThrift() { if (!alterInvertedIndexes.isEmpty()) { List tIndexes = new ArrayList<>(); - for (Map.Entry> index : alterInvertedIndexes.entrySet()) { - tIndexes.add(index.getKey().toThrift(index.getValue())); + for (Index index : alterInvertedIndexes) { + tIndexes.add(index.toThrift(index.getColumnUniqueIds(schemaColumns))); } req.setAlterInvertedIndexes(tIndexes); } if (existIndexes != null) { List indexDesc = new ArrayList(); - for (Map.Entry> index : existIndexes.entrySet()) { - TOlapTableIndex tIndex = index.getKey().toThrift(index.getValue()); + for (Index index : existIndexes) { + TOlapTableIndex tIndex = index.toThrift(index.getColumnUniqueIds(schemaColumns)); indexDesc.add(tIndex); } req.setIndexesDesc(indexDesc); diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java index 40592c05015b17..17c99435879a91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -75,8 +75,7 @@ public class CreateReplicaTask extends AgentTask { private double bfFpp; // indexes - // index -> index column unique ids - private Map> indexes; + private List indexes; private boolean isInMemory; @@ -138,7 +137,7 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition KeysType keysType, TStorageType storageType, TStorageMedium storageMedium, List columns, Set bfColumns, double bfFpp, MarkedCountDownLatch latch, - Map> indexes, + List indexes, boolean isInMemory, TTabletType tabletType, DataSortInfo dataSortInfo, @@ -344,15 +343,15 @@ public TCreateTabletReq toThrift() { LOG.debug("cluster key uids={}, table_id={}, tablet_id={}", clusterKeyUids, tableId, tabletId); } } - if (indexes != null && !indexes.isEmpty()) { + if (CollectionUtils.isNotEmpty(indexes)) { List tIndexes = null; - Object value = objectPool.get(indexes.keySet()); + Object value = objectPool.get(indexes); if (value != null) { tIndexes = (List) value; } else { tIndexes = new ArrayList<>(); - for (Map.Entry> index : indexes.entrySet()) { - tIndexes.add(index.getKey().toThrift(index.getValue())); + for (Index index : indexes) { + tIndexes.add(index.toThrift(index.getColumnUniqueIds(columns))); } } tSchema.setIndexes(tIndexes); From 0fafddf00c79a3fc4f02ac10567ab97b4a4935bb Mon Sep 17 00:00:00 2001 From: qidaye Date: Thu, 13 Mar 2025 21:22:53 +0800 Subject: [PATCH 09/12] opt function name --- .../java/org/apache/doris/catalog/OlapTable.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index dbc346d0b8bf44..d31a3c3806b3cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -3042,18 +3042,6 @@ public void initSchemaColumnUniqueId() { } } - public List getIndexColumnIds(List columnNames) { - List columnUniqueIds = new ArrayList<>(); - for (Column column : getBaseSchema()) { - for (String columnName : columnNames) { - if (column.getName().equalsIgnoreCase(columnName)) { - columnUniqueIds.add(column.getUniqueId()); - } - } - } - return columnUniqueIds; - } - public Set getPartitionKeys() { return idToPartition.keySet(); } From a304bb22292b7bb50dd07db070a4d49e0d018593 Mon Sep 17 00:00:00 2001 From: qidaye Date: Thu, 13 Mar 2025 21:27:12 +0800 Subject: [PATCH 10/12] opt function name --- .../src/main/java/org/apache/doris/catalog/Index.java | 10 ++++++---- .../doris/cloud/datasource/CloudInternalCatalog.java | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index dd24a84e9b2300..5642f109650ab1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -240,10 +240,12 @@ public String toSql() { public List getColumnUniqueIds(List schema) { List columnUniqueIds = new ArrayList<>(); - for (String columnName : columns) { - for (Column column : schema) { - if (columnName.equalsIgnoreCase(column.getName())) { - columnUniqueIds.add(column.getUniqueId()); + if (columns != null) { + for (String columnName : columns) { + for (Column column : schema) { + if (columnName.equalsIgnoreCase(column.getName())) { + columnUniqueIds.add(column.getUniqueId()); + } } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index fa05947b07a448..f645c60fe373b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -320,7 +320,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, schemaBuilder.setSortColNum(dataSortInfo.getColNum()); for (int i = 0; i < schemaColumns.size(); i++) { Column column = schemaColumns.get(i); - schemaBuilder.addColumn(column.toPb(bfColumns, (List) indexes)); + schemaBuilder.addColumn(column.toPb(bfColumns, indexes)); } Map columnMap = Maps.newHashMap(); From 2019b9096d95ee8120d7a288257535d5030e33b5 Mon Sep 17 00:00:00 2001 From: qidaye Date: Thu, 13 Mar 2025 21:29:24 +0800 Subject: [PATCH 11/12] opt function name --- fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index 5642f109650ab1..1718c26bdb4566 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -240,7 +240,7 @@ public String toSql() { public List getColumnUniqueIds(List schema) { List columnUniqueIds = new ArrayList<>(); - if (columns != null) { + if (schema != null) { for (String columnName : columns) { for (Column column : schema) { if (columnName.equalsIgnoreCase(column.getName())) { From 61e7fa159b52f2a061c23b7b640641a202c3012a Mon Sep 17 00:00:00 2001 From: qidaye Date: Thu, 13 Mar 2025 21:54:54 +0800 Subject: [PATCH 12/12] add ut --- .../org/apache/doris/catalog/IndexTest.java | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java new file mode 100644 index 00000000000000..776c97ca76b934 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.analysis.IndexDef; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class IndexTest { + + @Test + public void testGetColumnUniqueIds() { + // Create test columns with unique IDs + List schema = new ArrayList<>(); + Column col1 = new Column("col1", Type.INT); + col1.setUniqueId(101); + Column col2 = new Column("col2", Type.VARCHAR); + col2.setUniqueId(102); + Column col3 = new Column("col3", Type.DOUBLE); + col3.setUniqueId(103); + Column specialCol = new Column("special-name!@#", Type.STRING); + specialCol.setUniqueId(104); + Column mixedCaseCol = new Column("MiXeD_CaSe", Type.BIGINT); + mixedCaseCol.setUniqueId(105); + + schema.add(col1); + schema.add(col2); + schema.add(col3); + schema.add(specialCol); + schema.add(mixedCaseCol); + + // Test case 1: Basic column matching + List indexColumns1 = new ArrayList<>(); + indexColumns1.add("col1"); + indexColumns1.add("col3"); + Index index1 = new Index(1, "test_index1", indexColumns1, IndexDef.IndexType.BITMAP, null, null); + + List uniqueIds1 = index1.getColumnUniqueIds(schema); + Assert.assertEquals(2, uniqueIds1.size()); + Assert.assertEquals(Integer.valueOf(101), uniqueIds1.get(0)); + Assert.assertEquals(Integer.valueOf(103), uniqueIds1.get(1)); + + // Test case 2: Case-insensitive matching + List indexColumns2 = new ArrayList<>(); + indexColumns2.add("CoL1"); + indexColumns2.add("COL3"); + Index index2 = new Index(2, "test_index2", indexColumns2, IndexDef.IndexType.BITMAP, null, null); + + List uniqueIds2 = index2.getColumnUniqueIds(schema); + Assert.assertEquals(2, uniqueIds2.size()); + Assert.assertEquals(Integer.valueOf(101), uniqueIds2.get(0)); + Assert.assertEquals(Integer.valueOf(103), uniqueIds2.get(1)); + + // Test case 3: Non-existent column name + List indexColumns3 = new ArrayList<>(); + indexColumns3.add("col1"); + indexColumns3.add("non_existent_column"); + Index index3 = new Index(3, "test_index3", indexColumns3, IndexDef.IndexType.BITMAP, null, null); + + List uniqueIds3 = index3.getColumnUniqueIds(schema); + Assert.assertEquals(1, uniqueIds3.size()); + Assert.assertEquals(Integer.valueOf(101), uniqueIds3.get(0)); + + // Test case 4: Null schema + List uniqueIds4 = index1.getColumnUniqueIds(null); + Assert.assertEquals(0, uniqueIds4.size()); + + // Test case 5: Empty column list + Index emptyColIndex = new Index(5, "empty_col_index", new ArrayList<>(), + IndexDef.IndexType.BITMAP, null, null); + List emptyColUniqueIds = emptyColIndex.getColumnUniqueIds(schema); + Assert.assertEquals(0, emptyColUniqueIds.size()); + + // Test case 6: Empty schema (non-null) + List emptySchemaUniqueIds = index1.getColumnUniqueIds(new ArrayList<>()); + Assert.assertEquals(0, emptySchemaUniqueIds.size()); + + // Test case 7: Duplicate column names + List dupColumns = new ArrayList<>(); + dupColumns.add("col1"); + dupColumns.add("col1"); // Duplicated + dupColumns.add("col2"); + Index dupIndex = new Index(7, "dup_index", dupColumns, IndexDef.IndexType.BITMAP, null, null); + + List dupUniqueIds = dupIndex.getColumnUniqueIds(schema); + Assert.assertEquals(3, dupUniqueIds.size()); + Assert.assertEquals(Integer.valueOf(101), dupUniqueIds.get(0)); + Assert.assertEquals(Integer.valueOf(101), dupUniqueIds.get(1)); + Assert.assertEquals(Integer.valueOf(102), dupUniqueIds.get(2)); + + // Test case 8: Special characters in column names + List specialColList = new ArrayList<>(); + specialColList.add("special-name!@#"); + Index specialIndex = new Index(8, "special_index", specialColList, IndexDef.IndexType.BITMAP, null, null); + + List specialUniqueIds = specialIndex.getColumnUniqueIds(schema); + Assert.assertEquals(1, specialUniqueIds.size()); + Assert.assertEquals(Integer.valueOf(104), specialUniqueIds.get(0)); + + // Test case 9: Mixed case column name + List mixedCaseList = new ArrayList<>(); + mixedCaseList.add("mixed_case"); // Testing case insensitivity with underscores + Index mixedCaseIndex = new Index(9, "mixed_case_index", mixedCaseList, IndexDef.IndexType.BITMAP, null, null); + + List mixedCaseUniqueIds = mixedCaseIndex.getColumnUniqueIds(schema); + Assert.assertEquals(1, mixedCaseUniqueIds.size()); + Assert.assertEquals(Integer.valueOf(105), mixedCaseUniqueIds.get(0)); + + // Test case 10: Large number of columns + List largeColumnList = new ArrayList<>(); + List largeSchema = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + Column tempCol = new Column("col" + i, Type.INT); + tempCol.setUniqueId(1000 + i); + largeSchema.add(tempCol); + + // Add every other column to the index + if (i % 2 == 0) { + largeColumnList.add("col" + i); + } + } + + Index largeIndex = new Index(10, "large_index", largeColumnList, IndexDef.IndexType.BITMAP, null, null); + List largeUniqueIds = largeIndex.getColumnUniqueIds(largeSchema); + + Assert.assertEquals(500, largeUniqueIds.size()); + // Check first and last elements + Assert.assertEquals(Integer.valueOf(1000), largeUniqueIds.get(0)); + Assert.assertEquals(Integer.valueOf(1000 + 998), largeUniqueIds.get(499)); + + // Test case 11: Order preservation - ensure column order in index is preserved in IDs + List reverseOrderColumns = new ArrayList<>(); + reverseOrderColumns.add("col3"); + reverseOrderColumns.add("col2"); + reverseOrderColumns.add("col1"); + + Index reverseIndex = new Index(11, "reverse_index", reverseOrderColumns, IndexDef.IndexType.BITMAP, null, null); + List reverseUniqueIds = reverseIndex.getColumnUniqueIds(schema); + + Assert.assertEquals(3, reverseUniqueIds.size()); + Assert.assertEquals(Integer.valueOf(103), reverseUniqueIds.get(0)); + Assert.assertEquals(Integer.valueOf(102), reverseUniqueIds.get(1)); + Assert.assertEquals(Integer.valueOf(101), reverseUniqueIds.get(2)); + } +}