[fix](Index)Make column unique ids in index dynamically computed (#48988)

fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java

@@ -2712,7 +2712,6 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab
                 indexDef.checkColumn(column, olapTable.getKeysType(),
                         olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite(),
                                                                         disableInvertedIndexV1ForVariant);
-                indexDef.getColumnUniqueIds().add(column.getUniqueId());
             } else {
                 throw new DdlException("index column does not exist in table. invalid column: " + col);
             }
@@ -2723,7 +2722,6 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab
         // so here update column name in CreateIndexClause after checkColumn for indexDef,
         // there will use the column name in olapTable instead of the column name in CreateIndexClause.
         alterIndex.setColumns(indexDef.getColumns());
-        alterIndex.setColumnUniqueIds(indexDef.getColumnUniqueIds());
         newIndexes.add(alterIndex);
         return false;
     }

fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java

@@ -77,7 +77,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
         indexDef.analyze();
         this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(),
                 indexDef.getColumns(), indexDef.getIndexType(),
-                indexDef.getProperties(), indexDef.getComment(), indexDef.getColumnUniqueIds());
+                indexDef.getProperties(), indexDef.getComment());
     }
 
     @Override

fe/fe-core/src/main/java/org/apache/doris/analysis/CreateIndexClause.java

@@ -73,7 +73,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
         indexDef.analyze();
         this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(),
                 indexDef.getColumns(), indexDef.getIndexType(),
-                indexDef.getProperties(), indexDef.getComment(), indexDef.getColumnUniqueIds());
+                indexDef.getProperties(), indexDef.getComment());
     }
 
     @Override

fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java

@@ -592,8 +592,7 @@ public void analyze(Analyzer analyzer) throws UserException {
                     }
                 }
                 indexes.add(new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(), indexDef.getColumns(),
-                        indexDef.getIndexType(), indexDef.getProperties(), indexDef.getComment(),
-                        indexDef.getColumnUniqueIds()));
+                        indexDef.getIndexType(), indexDef.getProperties(), indexDef.getComment()));
                 distinct.add(indexDef.getIndexName());
                 distinctCol.add(Pair.of(indexDef.getIndexType(),
                         indexDef.getColumns().stream().map(String::toUpperCase).collect(Collectors.toList())));

fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java

@@ -42,7 +42,6 @@ public class IndexDef {
     private Map<String, String> properties;
     private boolean isBuildDeferred = false;
     private PartitionNames partitionNames;
-    private List<Integer> columnUniqueIds = Lists.newArrayList();
     public static final int MIN_NGRAM_SIZE = 1;
     public static final int MAX_NGRAM_SIZE = 255;
     public static final int MIN_BF_SIZE = 64;
@@ -202,10 +201,6 @@ public List<String> getPartitionNames() {
         return partitionNames == null ? Lists.newArrayList() : partitionNames.getPartitionNames();
     }
 
-    public List<Integer> getColumnUniqueIds() {
-        return columnUniqueIds;
-    }
-
     public enum IndexType {
         BITMAP,
         INVERTED,

fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java

@@ -65,19 +65,15 @@ public class Index implements Writable {
     private Map<String, String> properties;
     @SerializedName(value = "ct", alternate = {"comment"})
     private String comment;
-    @SerializedName(value = "cui", alternate = {"columnUniqueIds"})
-    private List<Integer> columnUniqueIds;
 
     public Index(long indexId, String indexName, List<String> columns,
-            IndexDef.IndexType indexType, Map<String, String> properties, String comment,
-            List<Integer> columnUniqueIds) {
+            IndexDef.IndexType indexType, Map<String, String> properties, String comment) {
         this.indexId = indexId;
         this.indexName = indexName;
         this.columns = columns == null ? Lists.newArrayList() : Lists.newArrayList(columns);
         this.indexType = indexType;
         this.properties = properties == null ? Maps.newHashMap() : Maps.newHashMap(properties);
         this.comment = comment;
-        this.columnUniqueIds = columnUniqueIds == null ? Lists.newArrayList() : Lists.newArrayList(columnUniqueIds);
         if (indexType == IndexDef.IndexType.INVERTED) {
             if (this.properties != null && !this.properties.isEmpty()) {
                 if (this.properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)) {
@@ -101,7 +97,6 @@ public Index() {
         this.indexType = null;
         this.properties = null;
         this.comment = null;
-        this.columnUniqueIds = null;
     }
 
     public long getIndexId() {
@@ -191,14 +186,6 @@ public void setComment(String comment) {
         this.comment = comment;
     }
 
-    public List<Integer> getColumnUniqueIds() {
-        return columnUniqueIds;
-    }
-
-    public void setColumnUniqueIds(List<Integer> columnUniqueIds) {
-        this.columnUniqueIds = columnUniqueIds;
-    }
-
     @Override
     public void write(DataOutput out) throws IOException {
         Text.writeString(out, GsonUtils.GSON.toJson(this));
@@ -216,7 +203,7 @@ public int hashCode() {
 
     public Index clone() {
         return new Index(indexId, indexName, new ArrayList<>(columns),
-                indexType, new HashMap<>(properties), comment, columnUniqueIds);
+                indexType, new HashMap<>(properties), comment);
     }
 
     @Override
@@ -251,7 +238,21 @@ public String toSql() {
         return sb.toString();
     }
 
-    public TOlapTableIndex toThrift() {
+    public List<Integer> getColumnUniqueIds(List<Column> schema) {
+        List<Integer> columnUniqueIds = new ArrayList<>();
+        if (schema != null) {
+            for (String columnName : columns) {
+                for (Column column : schema) {
+                    if (columnName.equalsIgnoreCase(column.getName())) {
+                        columnUniqueIds.add(column.getUniqueId());
+                    }
+                }
+            }
+        }
+        return columnUniqueIds;
+    }
+
+    public TOlapTableIndex toThrift(List<Integer> indexColumnUniqueIds) {
         TOlapTableIndex tIndex = new TOlapTableIndex();
         tIndex.setIndexId(indexId);
         tIndex.setIndexName(indexName);
@@ -260,15 +261,16 @@ public TOlapTableIndex toThrift() {
         if (properties != null) {
             tIndex.setProperties(properties);
         }
-        tIndex.setColumnUniqueIds(columnUniqueIds);
+        tIndex.setColumnUniqueIds(indexColumnUniqueIds);
         return tIndex;
     }
 
-    public OlapFile.TabletIndexPB toPb(Map<Integer, Column> columnMap) {
+    public OlapFile.TabletIndexPB toPb(Map<Integer, Column> columnMap, List<Integer> indexColumnUniqueIds) {
         OlapFile.TabletIndexPB.Builder builder = OlapFile.TabletIndexPB.newBuilder();
         builder.setIndexId(indexId);
         builder.setIndexName(indexName);
-        for (Integer columnUniqueId : columnUniqueIds) {
+
+        for (Integer columnUniqueId : indexColumnUniqueIds) {
             Column column = columnMap.get(columnUniqueId);
             if (column != null) {
                 builder.addColUniqueId(column.getUniqueId());

fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java

@@ -391,13 +391,6 @@ public void initSchemaColumnUniqueId() {
         maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE;
         this.schema.forEach(column -> {
             column.setUniqueId(incAndGetMaxColUniqueId());
-            this.indexes.forEach(index -> {
-                index.getColumns().forEach(col -> {
-                    if (col.equalsIgnoreCase(column.getName())) {
-                        index.getColumnUniqueIds().add(column.getUniqueId());
-                    }
-                });
-            });
             if (LOG.isDebugEnabled()) {
                 LOG.debug("indexId: {},  column:{}, uniqueId:{}",
                         indexId, column, column.getUniqueId());

fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java

@@ -321,9 +321,8 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId,
             columnMap.put(column.getUniqueId(), column);
         }
         if (indexes != null) {
-            for (int i = 0; i < indexes.size(); i++) {
-                Index index = indexes.get(i);
-                schemaBuilder.addIndex(index.toPb(columnMap));
+            for (Index index : indexes) {
+                schemaBuilder.addIndex(index.toPb(columnMap, index.getColumnUniqueIds(schemaColumns)));
             }
         }
 

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java

@@ -209,6 +209,6 @@ public IndexType getIndexType() {
 
     public Index translateToCatalogStyle() {
         return new Index(Env.getCurrentEnv().getNextId(), name, cols, indexType, properties,
-                comment, null);
+                comment);
     }
 }

fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java

@@ -1518,7 +1518,7 @@ protected void toThrift(TPlanNode msg) {
         }
 
         for (Index index : olapTable.getIndexes()) {
-            TOlapTableIndex tIndex = index.toThrift();
+            TOlapTableIndex tIndex = index.toThrift(index.getColumnUniqueIds(olapTable.getBaseSchema()));
             indexDesc.add(tIndex);
         }
 

fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java

@@ -316,7 +316,7 @@ public TOlapTableSchemaParam createSchema(long dbId, OlapTable table, Analyzer a
                 indexes = table.getIndexes();
             }
             for (Index index : indexes) {
-                TOlapTableIndex tIndex = index.toThrift();
+                TOlapTableIndex tIndex = index.toThrift(index.getColumnUniqueIds(table.getBaseSchema()));
                 indexDesc.add(tIndex);
             }
             TOlapTableIndexSchema indexSchema = new TOlapTableIndexSchema(pair.getKey(), columns,

fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java

@@ -103,15 +103,15 @@ public TAlterInvertedIndexReq toThrift() {
         if (!alterInvertedIndexes.isEmpty()) {
             List<TOlapTableIndex> tIndexes = new ArrayList<>();
             for (Index index : alterInvertedIndexes) {
-                tIndexes.add(index.toThrift());
+                tIndexes.add(index.toThrift(index.getColumnUniqueIds(schemaColumns)));
             }
             req.setAlterInvertedIndexes(tIndexes);
         }
 
         if (existIndexes != null) {
             List<TOlapTableIndex> indexDesc = new ArrayList<TOlapTableIndex>();
             for (Index index : existIndexes) {
-                TOlapTableIndex tIndex = index.toThrift();
+                TOlapTableIndex tIndex = index.toThrift(index.getColumnUniqueIds(schemaColumns));
                 indexDesc.add(tIndex);
             }
             req.setIndexesDesc(indexDesc);

fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java

@@ -351,7 +351,7 @@ public TCreateTabletReq toThrift() {
             } else {
                 tIndexes = new ArrayList<>();
                 for (Index index : indexes) {
-                    tIndexes.add(index.toThrift());
+                    tIndexes.add(index.toThrift(index.getColumnUniqueIds(columns)));
                 }
             }
             tSchema.setIndexes(tIndexes);