From 5f501432f17a77d5619f61f58a82fcc5edf33f2b Mon Sep 17 00:00:00 2001 From: qiye Date: Thu, 8 May 2025 11:30:01 +0800 Subject: [PATCH 1/7] [feature](index change)Support light index change for ngram bf index (#48461) Currently, NGram bloom filter index only supports directly schema change, and users need to build indexes incrementally when using it. The design goal is that ngrambf supports light_index_change, including local and cloud mode, which can incrementally add indexes or build indexes on stock data. Inverted indexes are currently only supported in local mode for light_schema_change, cloud mode is still a directly SC, this time it does not involve inverted indexes, and its functionality remains unchanged. After the completion of the function, the NGram BF index construction can be used in the following way, following the existing syntax, does not involve changes or additions. ```sql alter table t1 add index idx_ngram_k2 (`k2`) using ngram_bf properties("bf_size" = "1024", "gram_size" = "3"); create index idx_ngram_k2 (`k2`) on t1 using ngram_bf properties("bf_size" = "1024", "gram_size" = "3"); build index idx_ngram_k2 on t1; show alter table column; cancel build index on t1; ``` **NOTE:** Currently, building an index by partition is not supported. If you want to build an index for stock data, you need to build it for all data, including new data written after the Add index has been added. Build index by partition will be supported in next stage. Support light index change for NGram bf index --- fe/fe-core/src/main/cup/sql_parser.cup | 10 +- .../doris/alter/SchemaChangeHandler.java | 111 +++---- .../apache/doris/alter/SchemaChangeJobV2.java | 15 +- .../doris/analysis/BuildIndexClause.java | 63 +++- .../org/apache/doris/analysis/IndexDef.java | 4 +- .../java/org/apache/doris/catalog/Index.java | 12 +- .../org/apache/doris/catalog/OlapTable.java | 14 + .../doris/alter/IndexChangeJobTest.java | 156 +++++++++- .../doris/alter/SchemaChangeHandlerTest.java | 9 +- .../test_ngram_bloomfilter_index_change.out | 69 +++++ .../pipeline/cloud_p1/conf/fe_custom.conf | 1 - ...test_ngram_bloomfilter_index_change.groovy | 286 ++++++++++++++++++ 12 files changed, 638 insertions(+), 112 deletions(-) create mode 100644 regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out create mode 100644 regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 95d77384a81231..5340fd10a8412e 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -900,7 +900,6 @@ nonterminal Map key_value_map, opt_key_value_map, opt_key_value_ opt_ext_properties, opt_enable_feature_properties, properties; nonterminal ColumnDef column_definition; nonterminal IndexDef index_definition; -nonterminal IndexDef build_index_definition; nonterminal ArrayList column_definition_list; nonterminal ArrayList index_definition_list; nonterminal AggregateType opt_agg_type; @@ -2194,7 +2193,7 @@ create_stmt ::= :} | KW_BUILD KW_INDEX ident:indexName KW_ON table_name:tableName opt_partition_names:partitionNames {: - RESULT = new AlterTableStmt(tableName, Lists.newArrayList(new BuildIndexClause(tableName, new IndexDef(indexName, partitionNames, true), false))); + RESULT = new AlterTableStmt(tableName, Lists.newArrayList(new BuildIndexClause(tableName, indexName, partitionNames, false))); :} /* stage */ | KW_CREATE KW_STAGE opt_if_not_exists:ifNotExists ident:stageName KW_PROPERTIES opt_key_value_map:properties @@ -4121,13 +4120,6 @@ index_definition ::= :} ; -build_index_definition ::= - KW_INDEX ident:indexName opt_partition_names:partitionNames - {: - RESULT = new IndexDef(indexName, partitionNames, true); - :} - ; - opt_nullable_type ::= {: RESULT = ColumnNullableType.DEFAULT; diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 7751cbac9f1002..a5e678eb779897 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1278,7 +1278,12 @@ private void checkAssignedTargetIndexName(String baseIndexName, String targetInd } private void createJob(String rawSql, long dbId, OlapTable olapTable, Map> indexSchemaMap, - Map propertyMap, List indexes) throws UserException { + Map propertyMap, List indexes, + boolean isBuildIndex) throws UserException { + if (isBuildIndex) { + // remove the index which is not the base index, only base index can be built index + indexSchemaMap.entrySet().removeIf(entry -> !entry.getKey().equals(olapTable.getBaseIndexId())); + } checkReplicaCount(olapTable); // process properties first @@ -1314,7 +1319,7 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map newSet = new HashSet<>(indexes); Set oriSet = new HashSet<>(olapTable.getIndexes()); - if (!newSet.equals(oriSet)) { + if (!newSet.equals(oriSet) || isBuildIndex) { hasIndexChange = true; } @@ -1330,7 +1335,7 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map oriBfColumns = olapTable.getCopiedBfColumns(); double oriBfFpp = olapTable.getBfFpp(); @@ -1991,7 +1996,7 @@ public int getAsInt() { List newIndexes = olapTable.getCopiedIndexes(); List alterIndexes = new ArrayList<>(); - Map> invertedIndexOnPartitions = new HashMap<>(); + Map> indexOnPartitions = new HashMap<>(); boolean isDropIndex = false; Map propertyMap = new HashMap<>(); for (AlterClause alterClause : alterClauses) { @@ -2145,68 +2150,29 @@ public int getAsInt() { } lightSchemaChange = false; - if (index.isLightIndexChangeSupported() && !Config.isCloudMode()) { + // ngram_bf index can do light_schema_change in both local and cloud mode + // inverted index can only do light_schema_change in local mode + if (index.isLightIndexChangeSupported()) { alterIndexes.add(index); isDropIndex = false; - // now only support light index change for inverted index lightIndexChange = true; } } else if (alterClause instanceof BuildIndexClause) { BuildIndexClause buildIndexClause = (BuildIndexClause) alterClause; IndexDef indexDef = buildIndexClause.getIndexDef(); Index index = buildIndexClause.getIndex(); - if (Config.isCloudMode()) { + if (Config.isCloudMode() && index.getIndexType() == IndexDef.IndexType.INVERTED) { throw new DdlException("BUILD INDEX operation failed: No need to do it in cloud mode."); } - if (!olapTable.isPartitionedTable()) { - List specifiedPartitions = indexDef.getPartitionNames(); - if (!specifiedPartitions.isEmpty()) { - throw new DdlException("table " + olapTable.getName() - + " is not partitioned, cannot build index with partitions."); - } - } - List existedIndexes = olapTable.getIndexes(); - boolean found = false; - for (Index existedIdx : existedIndexes) { - if (existedIdx.getIndexName().equalsIgnoreCase(indexDef.getIndexName())) { - found = true; - if (!existedIdx.isLightIndexChangeSupported()) { - throw new DdlException("BUILD INDEX operation failed: The index " - + existedIdx.getIndexName() + " of type " + existedIdx.getIndexType() - + " does not support lightweight index changes."); - } - for (Column column : olapTable.getBaseSchema()) { - if (!column.getType().isVariantType()) { - continue; - } - // variant type column can not support for building index - for (String indexColumn : existedIdx.getColumns()) { - if (column.getName().equalsIgnoreCase(indexColumn)) { - throw new DdlException("BUILD INDEX operation failed: The " - + indexDef.getIndexName() + " index can not be built on the " - + indexColumn + " column, because it is a variant type column."); - } - } - } - index = existedIdx.clone(); - if (indexDef.getPartitionNames().isEmpty()) { - invertedIndexOnPartitions.put(index.getIndexId(), olapTable.getPartitionNames()); - } else { - invertedIndexOnPartitions.put( - index.getIndexId(), new HashSet<>(indexDef.getPartitionNames())); - } - break; - } - } - if (!found) { - throw new DdlException("index " + indexDef.getIndexName() - + " not exist, cannot build it with defferred."); + if (indexDef.getPartitionNames().isEmpty()) { + indexOnPartitions.put(index.getIndexId(), olapTable.getPartitionNames()); + } else { + indexOnPartitions.put( + index.getIndexId(), new HashSet<>(indexDef.getPartitionNames())); } - if (indexDef.isInvertedIndex()) { - alterIndexes.add(index); - } + alterIndexes.add(index); buildIndexChange = true; lightSchemaChange = false; } else if (alterClause instanceof DropIndexClause) { @@ -2224,7 +2190,9 @@ public int getAsInt() { break; } } - if (found.isLightIndexChangeSupported() && !Config.isCloudMode()) { + // only inverted index with local mode can do light drop index change + if (found != null && found.getIndexType() == IndexDef.IndexType.INVERTED + && Config.isNotCloudMode()) { alterIndexes.add(found); isDropIndex = true; lightIndexChange = true; @@ -2245,19 +2213,27 @@ public int getAsInt() { long jobId = Env.getCurrentEnv().getNextId(); //for schema change add/drop value column optimize, direct modify table meta. modifyTableLightSchemaChange(rawSql, db, olapTable, indexSchemaMap, newIndexes, - null, isDropIndex, jobId, false); + null, isDropIndex, jobId, false, propertyMap); } else if (Config.enable_light_index_change && lightIndexChange) { long jobId = Env.getCurrentEnv().getNextId(); - //for schema change add/drop inverted index optimize, direct modify table meta firstly. + //for schema change add/drop inverted index and ngram_bf optimize, direct modify table meta firstly. modifyTableLightSchemaChange(rawSql, db, olapTable, indexSchemaMap, newIndexes, - alterIndexes, isDropIndex, jobId, false); + alterIndexes, isDropIndex, jobId, false, propertyMap); } else if (buildIndexChange) { + if (alterIndexes.isEmpty()) { + throw new DdlException("Altered index is empty. please check your alter stmt."); + } + IndexDef.IndexType indexType = alterIndexes.get(0).getIndexType(); if (Config.enable_light_index_change) { - buildOrDeleteTableInvertedIndices(db, olapTable, indexSchemaMap, - alterIndexes, invertedIndexOnPartitions, false); + if (indexType == IndexDef.IndexType.INVERTED) { + buildOrDeleteTableInvertedIndices(db, olapTable, indexSchemaMap, + alterIndexes, indexOnPartitions, false); + } else { + createJob(rawSql, db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes, true); + } } } else { - createJob(rawSql, db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes); + createJob(rawSql, db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes, false); } } finally { olapTable.writeUnlock(); @@ -2747,6 +2723,8 @@ private void cancelIndexJob(CancelAlterTableStmt cancelAlterTableStmt) throws Dd olapTable.writeUnlock(); } + // if this table has ngram_bf index, we must run cancel for schema change job + boolean hasNGramBFIndex = ((OlapTable) olapTable).hasIndexOfType(IndexDef.IndexType.NGRAM_BF); // alter job v2's cancel must be called outside the table lock if (jobList.size() > 0) { for (IndexChangeJob job : jobList) { @@ -2761,6 +2739,8 @@ private void cancelIndexJob(CancelAlterTableStmt cancelAlterTableStmt) throws Dd LOG.info("cancel build index job {} on table {} success", jobId, tableName); } } + } else if (hasNGramBFIndex) { + cancelColumnJob(cancelAlterTableStmt); } else { throw new DdlException("No job to cancel for Table[" + tableName + "]"); } @@ -2800,7 +2780,7 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab Column column = olapTable.getColumn(col); if (column != null) { indexDef.checkColumn(column, olapTable.getKeysType(), - olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite(), + olapTable.getEnableUniqueKeyMergeOnWrite(), olapTable.getInvertedIndexFileStorageFormat()); if (!InvertedIndexUtil.getInvertedIndexFieldPattern(indexDef.getProperties()).isEmpty()) { throw new DdlException("Can not create index with field pattern"); @@ -2961,7 +2941,7 @@ public void replayAlterJobV2(AlterJobV2 alterJob) throws AnalysisException { public void modifyTableLightSchemaChange(String rawSql, Database db, OlapTable olapTable, Map> indexSchemaMap, List indexes, List alterIndexes, boolean isDropIndex, - long jobId, boolean isReplay) + long jobId, boolean isReplay, Map propertyMap) throws DdlException, AnalysisException { if (LOG.isDebugEnabled()) { @@ -3040,7 +3020,7 @@ public void modifyTableLightSchemaChange(String rawSql, Database db, OlapTable o } try { buildOrDeleteTableInvertedIndices(db, olapTable, indexSchemaMap, - alterIndexes, invertedIndexOnPartitions, true); + alterIndexes, invertedIndexOnPartitions, true); } catch (Exception e) { throw new DdlException(e.getMessage()); } @@ -3101,7 +3081,8 @@ public void replayModifyTableLightSchemaChange(TableAddOrDropColumnsInfo info) OlapTable olapTable = (OlapTable) db.getTableOrMetaException(tableId, TableType.OLAP); olapTable.writeLock(); try { - modifyTableLightSchemaChange("", db, olapTable, indexSchemaMap, indexes, null, false, jobId, true); + modifyTableLightSchemaChange("", db, olapTable, indexSchemaMap, indexes, null, false, jobId, + true, new HashMap<>()); } catch (DdlException e) { // should not happen LOG.warn("failed to replay modify table add or drop or modify columns", e); @@ -3252,7 +3233,7 @@ public void replayModifyTableAddOrDropInvertedIndices(TableAddOrDropInvertedIndi olapTable.writeLock(); try { modifyTableLightSchemaChange("", db, olapTable, indexSchemaMap, newIndexes, - alterIndexes, isDropIndex, jobId, true); + alterIndexes, isDropIndex, jobId, true, new HashMap<>()); } catch (UserException e) { // should not happen LOG.warn("failed to replay modify table add or drop indexes", e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 2e4ca2c6a70ee4..ad4a6ef60d1229 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -151,7 +151,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 { private boolean hasEnableUniqueKeySkipBitmapChanged = false; // save all schema change tasks - private AgentBatchTask schemaChangeBatchTask = new AgentBatchTask(); + AgentBatchTask schemaChangeBatchTask = new AgentBatchTask(); protected SchemaChangeJobV2() { super(JobType.SCHEMA_CHANGE); @@ -678,14 +678,13 @@ protected void runRunningJob() throws AlterCancelException { healthyReplicaNum++; } } - if (!FeConstants.runningUnitTest) { - if (healthyReplicaNum < expectReplicationNum / 2 + 1) { - LOG.warn("shadow tablet {} has few healthy replicas: {}, schema change job: {}" - + " healthyReplicaNum {} expectReplicationNum {}", - shadowTablet.getId(), replicas, jobId, healthyReplicaNum, expectReplicationNum); - throw new AlterCancelException( + + if ((healthyReplicaNum < expectReplicationNum / 2 + 1) && !FeConstants.runningUnitTest) { + LOG.warn("shadow tablet {} has few healthy replicas: {}, schema change job: {}" + + " healthyReplicaNum {} expectReplicationNum {}", + shadowTablet.getId(), replicas, jobId, healthyReplicaNum, expectReplicationNum); + throw new AlterCancelException( "shadow tablet " + shadowTablet.getId() + " has few healthy replicas"); - } } } // end for tablets } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java index d04d24e86e403a..355b043e3d57c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java @@ -18,12 +18,17 @@ package org.apache.doris.analysis; import org.apache.doris.alter.AlterOpType; +import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Index; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import com.google.common.collect.Maps; +import java.util.List; import java.util.Map; public class BuildIndexClause extends AlterTableClause { @@ -36,11 +41,14 @@ public class BuildIndexClause extends AlterTableClause { private boolean alter; // index internal class private Index index; + private String indexName; + private PartitionNames partitionNames; - public BuildIndexClause(TableName tableName, IndexDef indexDef, boolean alter) { + public BuildIndexClause(TableName tableName, String indexName, PartitionNames partitionNames, boolean alter) { super(AlterOpType.SCHEMA_CHANGE); this.tableName = tableName; - this.indexDef = indexDef; + this.indexName = indexName; + this.partitionNames = partitionNames; this.alter = alter; } @@ -67,17 +75,52 @@ public TableName getTableName() { @Override public void analyze(Analyzer analyzer) throws AnalysisException { - if (indexDef == null) { - throw new AnalysisException("index definition expected."); + tableName.analyze(analyzer); + DatabaseIf db = Env.getCurrentEnv().getCatalogMgr().getInternalCatalog() + .getDb(tableName.getDb()).orElse(null); + if (db == null) { + throw new AnalysisException("Database[" + tableName.getDb() + "] is not exist"); } - if (indexDef.getIndexType() == IndexDef.IndexType.NGRAM_BF - || indexDef.getIndexType() == IndexDef.IndexType.BLOOMFILTER) { - throw new AnalysisException("ngram bloomfilter or bloomfilter index is not needed to build."); + + TableIf table = db.getTable(tableName.getTbl()).orElse(null); + if (table == null) { + throw new AnalysisException("Table[" + tableName.getTbl() + "] is not exist"); + } + if (!(table instanceof OlapTable)) { + throw new AnalysisException("Only olap table support build index"); + } + + Index existedIdx = null; + for (Index index : table.getTableIndexes().getIndexes()) { + if (index.getIndexName().equalsIgnoreCase(indexName)) { + existedIdx = index; + if (!existedIdx.isLightIndexChangeSupported()) { + throw new AnalysisException("BUILD INDEX operation failed: The index " + + existedIdx.getIndexName() + " of type " + existedIdx.getIndexType() + + " does not support lightweight index changes."); + } + break; + } + } + if (existedIdx == null) { + throw new AnalysisException("Index[" + indexName + "] is not exist in table[" + tableName.getTbl() + "]"); + } + + IndexDef.IndexType indexType = existedIdx.getIndexType(); + if (!existedIdx.isLightIndexChangeSupported()) { + throw new AnalysisException(indexType.toString() + " index is not needed to build."); + } + + indexDef = new IndexDef(indexName, partitionNames, indexType, true); + if (!table.isPartitionedTable()) { + List specifiedPartitions = indexDef.getPartitionNames(); + if (!specifiedPartitions.isEmpty()) { + throw new AnalysisException("table " + table.getName() + + " is not partitioned, cannot build index with partitions."); + } } indexDef.analyze(); - this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(), - indexDef.getColumns(), indexDef.getIndexType(), - indexDef.getProperties(), indexDef.getComment()); + this.index = existedIdx.clone(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java index 8d1f6a5bfd6a74..a1df981898e665 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java @@ -83,9 +83,9 @@ public IndexDef(String indexName, boolean ifNotExists, List columns, Ind } } - public IndexDef(String indexName, PartitionNames partitionNames, boolean isBuildDeferred) { + public IndexDef(String indexName, PartitionNames partitionNames, IndexType indexType, boolean isBuildDeferred) { this.indexName = indexName; - this.indexType = IndexType.INVERTED; + this.indexType = indexType; this.partitionNames = partitionNames; this.isBuildDeferred = isBuildDeferred; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index 312e3194ed3eab..1b6197524cf431 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.IndexDef; import org.apache.doris.analysis.InvertedIndexUtil; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.PrintableMap; @@ -168,8 +169,17 @@ public String getInvertedIndexParserStopwords() { return InvertedIndexUtil.getInvertedIndexParserStopwords(properties); } + // Whether the index can be changed in light mode + // cloud mode only supports light change for ngram_bf index + // local mode supports light change for both inverted index and ngram_bf index + // the rest of the index types do not support light change public boolean isLightIndexChangeSupported() { - return indexType == IndexDef.IndexType.INVERTED; + if (Config.isCloudMode()) { + return indexType == IndexDef.IndexType.NGRAM_BF; + } else { + return indexType == IndexDef.IndexType.INVERTED + || indexType == IndexDef.IndexType.NGRAM_BF; + } } public String getInvertedIndexCustomAnalyzer() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index d4dca760bfef3d..e39cc1ec0d06a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -24,6 +24,7 @@ import org.apache.doris.analysis.CreateMaterializedViewStmt; import org.apache.doris.analysis.DataSortInfo; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.IndexDef; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotRef; import org.apache.doris.backup.Status; @@ -356,6 +357,19 @@ public List getIndexIds() { return indexes.getIndexIds(); } + /** + * Checks if the table contains at least one index of the specified type. + * @param indexType The index type to check for + * @return true if the table has at least one index of the specified type, false otherwise + */ + public boolean hasIndexOfType(IndexDef.IndexType indexType) { + if (indexes == null) { + return false; + } + return indexes.getIndexes().stream() + .anyMatch(index -> index.getIndexType() == indexType); + } + @Override public TableIndexes getTableIndexes() { return indexes; diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java index 6b0943c306f851..20bd26fa92db50 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java @@ -55,6 +55,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import mockit.Mock; +import mockit.MockUp; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -85,6 +87,8 @@ public class IndexChangeJobTest { private static BuildIndexClause buildIndexClause; private static DropIndexClause dropIndexClause; private static CancelAlterTableStmt cancelAlterTableStmt; + private static TableName tableName; + private static String indexName; @Rule public ExpectedException expectedEx = ExpectedException.none(); @@ -108,25 +112,31 @@ public void setUp() db = masterEnv.getInternalCatalog().getDbOrDdlException(CatalogTestUtil.testDbId1); olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId1); + new MockUp() { + @Mock + public Env getCurrentEnv() { + return masterEnv; + } + }; + // set mow table property Map properties = Maps.newHashMap(); properties.put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE, "false"); TableProperty tableProperty = new TableProperty(properties); olapTable.setTableProperty(tableProperty); - TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), olapTable.getName()); - IndexDef indexDef = new IndexDef("index1", false, + indexName = "index1"; + IndexDef indexDef = new IndexDef(indexName, false, Lists.newArrayList(olapTable.getBaseSchema().get(1).getName()), IndexDef.IndexType.INVERTED, Maps.newHashMap(), "balabala"); + FakeEnv.setEnv(masterEnv); createIndexClause = new CreateIndexClause(tableName, indexDef, false); createIndexClause.analyze(analyzer); - buildIndexClause = new BuildIndexClause(tableName, indexDef, false); - buildIndexClause.analyze(analyzer); - - dropIndexClause = new DropIndexClause("index1", false, tableName, false); + dropIndexClause = new DropIndexClause(indexName, false, tableName, false); dropIndexClause.analyze(analyzer); cancelAlterTableStmt = new CancelAlterTableStmt(ShowAlterStmt.AlterType.INDEX, tableName); @@ -163,6 +173,8 @@ public void testBuildIndexIndexChange() throws UserException { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -203,6 +215,8 @@ public void testBuildIndexIndexChangeNormal() throws UserException { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -292,6 +306,8 @@ public void testCancelBuildIndexIndexChangeNormal() throws UserException { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -339,6 +355,8 @@ public void testBuildIndexIndexChangeWhileTableNotStable() throws Exception { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -467,6 +485,8 @@ public void testBuildIndexFailedWithMinFailedNum() throws Exception { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -519,6 +539,8 @@ public void testBuildIndexFailedWithMaxFailedNum() throws Exception { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -526,13 +548,13 @@ public void testBuildIndexFailedWithMaxFailedNum() throws Exception { Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState()); IndexChangeJob indexChangejob = indexChangeJobMap.values().stream().findAny().get(); - Assert.assertEquals(indexChangejob.invertedIndexBatchTask.getTaskNum(), 0); + Assert.assertEquals(0, indexChangejob.invertedIndexBatchTask.getTaskNum()); Assert.assertEquals(IndexChangeJob.JobState.WAITING_TXN, indexChangejob.getJobState()); // run waiting txn job schemaChangeHandler.runAfterCatalogReady(); Assert.assertEquals(IndexChangeJob.JobState.RUNNING, indexChangejob.getJobState()); - Assert.assertEquals(indexChangejob.invertedIndexBatchTask.getTaskNum(), 3); + Assert.assertEquals(3, indexChangejob.invertedIndexBatchTask.getTaskNum()); // run running job schemaChangeHandler.runAfterCatalogReady(); Assert.assertEquals(IndexChangeJob.JobState.RUNNING, indexChangejob.getJobState()); @@ -565,17 +587,123 @@ public void testNgramBfBuildIndex() throws UserException { fakeEditLog = new FakeEditLog(); FakeEnv.setEnv(masterEnv); - IndexDef indexDef = new IndexDef("ngram_bf_index", false, - Lists.newArrayList(olapTable.getBaseSchema().get(1).getName()), + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + String indexName = "ngram_bf_index"; + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, Maps.newHashMap(), "ngram bf index"); TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), - olapTable.getName()); + table.getName()); createIndexClause = new CreateIndexClause(tableName, indexDef, false); createIndexClause.analyze(analyzer); + SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); + + long jobId = indexChangeJobMap.values().stream().findAny().get().jobId; + + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); + alterClauses.clear(); + alterClauses.add(buildIndexClause); + + schemaChangeHandler.process(alterClauses, db, table); + Assert.assertEquals(2, indexChangeJobMap.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + + SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() + .filter(job -> job.jobId != jobId) + .findFirst() + .orElse(null); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, jobV2.getJobState()); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } - buildIndexClause = new BuildIndexClause(tableName, indexDef, false); - org.junit.jupiter.api.Assertions.assertThrows(org.apache.doris.common.AnalysisException.class, - () -> buildIndexClause.analyze(analyzer)); + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); + } + + @Test + public void testCancelNgramBfBuildIndex() throws UserException { + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + String indexName = "ngram_bf_index"; + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + Maps.newHashMap(), "ngram bf index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); + + long jobId = indexChangeJobMap.values().stream().findAny().get().jobId; + + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); + alterClauses.clear(); + alterClauses.add(buildIndexClause); + + schemaChangeHandler.process(alterClauses, db, table); + Assert.assertEquals(2, indexChangeJobMap.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + + SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() + .filter(job -> job.jobId != jobId) + .findFirst() + .orElse(null); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, jobV2.getJobState()); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + cancelAlterTableStmt = new CancelAlterTableStmt(ShowAlterStmt.AlterType.INDEX, tableName); + cancelAlterTableStmt.analyze(analyzer); + schemaChangeHandler.cancel(cancelAlterTableStmt); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.CANCELLED, jobV2.getJobState()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java index b87ae340f1bef3..7099008be67720 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java @@ -851,8 +851,13 @@ public void testDupAddOrDropNgramBfIndex() throws Exception { String buildNgramBfIndexStmtStr = "BUILD INDEX idx_error_msg on test.sc_dup "; AlterTableStmt buildNgramBfIndexStmt = (AlterTableStmt) parseAndAnalyzeStmt(buildNgramBfIndexStmtStr); - org.junit.jupiter.api.Assertions.assertThrows(org.apache.doris.common.DdlException.class, - () -> Env.getCurrentEnv().getAlterInstance().processAlterTable(buildNgramBfIndexStmt)); + Env.getCurrentEnv().getAlterInstance().processAlterTable(buildNgramBfIndexStmt); + + jobSize++; + alterJobs = Env.getCurrentEnv().getSchemaChangeHandler().getAlterJobsV2(); + LOG.info("alterJobs:{}", alterJobs); + Assertions.assertEquals(jobSize, alterJobs.size()); + waitAlterJobDone(alterJobs); tbl.readLock(); try { diff --git a/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out b/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out new file mode 100644 index 00000000000000..6f916a99c91705 --- /dev/null +++ b/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out @@ -0,0 +1,69 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + diff --git a/regression-test/pipeline/cloud_p1/conf/fe_custom.conf b/regression-test/pipeline/cloud_p1/conf/fe_custom.conf index 0e43d458f38b65..5bba5437f73589 100644 --- a/regression-test/pipeline/cloud_p1/conf/fe_custom.conf +++ b/regression-test/pipeline/cloud_p1/conf/fe_custom.conf @@ -34,6 +34,5 @@ cloud_http_port=18030 meta_service_endpoint=127.0.0.1:5000 arrow_flight_sql_port = 8081 cloud_unique_id=cloud_unique_id_sql_server00 -enable_light_index_change=false enable_advance_next_id = true enable_job_schedule_second_for_test = true \ No newline at end of file diff --git a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy new file mode 100644 index 00000000000000..6fea7a68f92b8e --- /dev/null +++ b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy @@ -0,0 +1,286 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +import groovy.json.JsonSlurper + +suite("test_ngram_bloomfilter_index_change") { + def tableName = 'test_ngram_bloomfilter_index_change' + def timeout = 60000 + def delta_time = 1000 + def alter_res = "null" + def useTime = 0 + + def wait_for_latest_op_on_table_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${table_name}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(3000) // wait change table state to normal + logger.info(table_name + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } + + // Function to insert test data batch + def insertTestData = { -> + // insert 10 records + sql "INSERT INTO ${tableName} VALUES (1001, '2023-10-06 15:00:00', 'Laptop', 'John Smith', 199.99, 'North');" + sql "INSERT INTO ${tableName} VALUES (1002, '2023-10-09 17:05:00', 'Smartphone', 'Emily Johnson', 299.99, 'South');" + sql "INSERT INTO ${tableName} VALUES (1003, '2023-10-12 19:10:00', 'Headphones', 'Michael Brown', 399.99, 'East');" + sql "INSERT INTO ${tableName} VALUES (1004, '2023-10-15 21:15:00', 'Monitor', 'Jessica Davis', 499.99, 'West');" + sql "INSERT INTO ${tableName} VALUES (1005, '2023-10-18 23:20:00', 'Keyboard', 'David Wilson', 89.99, 'North');" + sql "INSERT INTO ${tableName} VALUES (1006, '2023-10-21 07:25:00', 'Mouse', 'Sarah Taylor', 699.99, 'South');" + sql "INSERT INTO ${tableName} VALUES (1007, '2023-10-24 09:30:00', 'Printer', 'Thomas Anderson', 799.99, 'East');" + sql "INSERT INTO ${tableName} VALUES (1008, '2023-10-27 11:35:00', 'Speaker', 'Jennifer Martin', 899.99, 'West');" + sql "INSERT INTO ${tableName} VALUES (1009, '2023-10-02 13:40:00', 'External SSD', 'Robert Clark', 999.99, 'North');" + sql "INSERT INTO ${tableName} VALUES (1010, '2023-10-05 15:45:00', 'Webcam', 'Amanda Lewis', 89.99, 'South');" + sql "sync" + } + + // Test setup + // 1. Create table + // 2. Insert test data + // 3. Add NGRAM Bloom Filter index + // 4. Build index + // 5. Insert more data + // 6. Drop index + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + `sale_id` int NULL, + `sale_date` datetime NULL, + `product_name` varchar(100) NULL, + `customer_name` varchar(100) NULL, + `amount` decimal(10,2) NULL, + `region` char(50) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`sale_id`) + PARTITION BY RANGE(`sale_date`) ( + PARTITION p202310 VALUES [('2023-10-01 00:00:00'), ('2023-11-01 00:00:00')), + PARTITION p202311 VALUES [('2023-11-01 00:00:00'), ('2023-12-01 00:00:00')), + PARTITION p202312 VALUES [('2023-12-01 00:00:00'), ('2024-01-01 00:00:00')) + ) + DISTRIBUTED BY HASH(`sale_id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "false" + ); + """ + + // Insert first batch of data + insertTestData() + + // Test settings + sql "set enable_function_pushdown=true" + sql "set enable_profile=true" + sql "set profile_level=2" + + // Verify data loaded correctly + qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Define test query + def query = "SELECT /*+SET_VAR(enable_function_pushdown = true, enable_profile = true, profile_level = 2)*/ * FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id" + + // Test 1: without NGRAM Bloom Filter index + profile("sql_select_like_without_ngram_index") { + run { + sql "/* sql_select_like_without_ngram_index */ ${query}" + sleep(1000) // sleep 1s wait for the profile collection to be completed + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } + + // Test 2: After adding NGRAM Bloom Filter index + sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" + wait_for_latest_op_on_table_finish(tableName, timeout) + profile("sql_select_like_with_ngram_index_added") { + run { + sql "/* sql_select_like_with_ngram_index_added */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } + + // Test 3: After building the index + sql "BUILD INDEX idx_ngram_customer_name ON ${tableName};" + wait_for_latest_op_on_table_finish(tableName, timeout) + profile("sql_select_like_with_ngram_index_built") { + run { + sql "/* sql_select_like_with_ngram_index_built */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 10")) + } + } + + // Insert second batch of data + insertTestData() + // Verify data loaded correctly + qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test 4: Verify filtering with more data + profile("sql_select_like_with_ngram_index_more_data") { + run { + sql "/* sql_select_like_with_ngram_index_more_data */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 20")) + } + } + + // Test 5: After dropping the index + sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" + wait_for_latest_op_on_table_finish(tableName, timeout) + profile("sql_select_like_with_ngram_index_dropped") { + run { + sql "/* sql_select_like_with_ngram_index_dropped */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } + + // recreate table + // 1. Create table + // 2. Add NGRAM Bloom Filter index + // 3. Insert data + // 4. Insert more data + // 5. Build index + // 6. Drop index + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + `sale_id` int NULL, + `sale_date` datetime NULL, + `product_name` varchar(100) NULL, + `customer_name` varchar(100) NULL, + `amount` decimal(10,2) NULL, + `region` char(50) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`sale_id`) + PARTITION BY RANGE(`sale_date`) ( + PARTITION p202310 VALUES [('2023-10-01 00:00:00'), ('2023-11-01 00:00:00')), + PARTITION p202311 VALUES [('2023-11-01 00:00:00'), ('2023-12-01 00:00:00')), + PARTITION p202312 VALUES [('2023-12-01 00:00:00'), ('2024-01-01 00:00:00')) + ) + DISTRIBUTED BY HASH(`sale_id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "false" + ); + """ + + // add ngram bf index + sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" + wait_for_latest_op_on_table_finish(tableName, timeout) + + // insert data + insertTestData() + + // Verify data loaded correctly + qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test 6: Verify filtering with index added + profile("sql_select_like_with_ngram_index_recreated") { + run { + sql "/* sql_select_like_with_ngram_index_recreated */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 10")) + } + } + + // insert more data + insertTestData() + + // Verify data loaded correctly + qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test 7: Verify filtering with more data + profile("sql_select_like_with_ngram_index_recreated_more_data") { + run { + sql "/* sql_select_like_with_ngram_index_recreated_more_data */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 20")) + } + } + + // build index + sql "BUILD INDEX idx_ngram_customer_name ON ${tableName};" + wait_for_latest_op_on_table_finish(tableName, timeout) + + // Test 8: Verify filtering with index built + profile("sql_select_like_with_ngram_index_recreated_built") { + run { + sql "/* sql_select_like_with_ngram_index_recreated_built */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 20")) + } + } + + // drop index + sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" + wait_for_latest_op_on_table_finish(tableName, timeout) + + // Test 9: Verify filtering with index dropped + profile("sql_select_like_with_ngram_index_recreated_dropped") { + run { + sql "/* sql_select_like_with_ngram_index_recreated_dropped */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } +} \ No newline at end of file From 5c598573024da0d39160247c748d8a3b2c6d2c3c Mon Sep 17 00:00:00 2001 From: airborne12 Date: Mon, 30 Jun 2025 11:04:45 +0800 Subject: [PATCH 2/7] [feature](index change)Support light index change for inverted index without parser (#52251) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem Summary: This PR adds support for lightweight index add for inverted indexes without parser in cloud mode and extends the existing “light” mode for NGRAM Bloom Filter indexes. And also introduces a new session variable (enable_add_index_for_new_data) to control the above logic. --- .../doris/alter/SchemaChangeHandler.java | 36 +- .../doris/analysis/BuildIndexClause.java | 6 +- .../java/org/apache/doris/catalog/Index.java | 27 +- .../datasource/CloudInternalCatalog.java | 8 + .../org/apache/doris/qe/SessionVariable.java | 16 + .../apache/doris/alter/CloudIndexTest.java | 664 ++++++++++++++++++ .../doris/alter/IndexChangeJobTest.java | 69 +- .../doris/alter/SchemaChangeHandlerTest.java | 10 +- .../apache/doris/catalog/CatalogTestUtil.java | 42 +- .../org/apache/doris/catalog/FakeEditLog.java | 10 + .../test_ngram_bloomfilter_index_change.out | 42 +- ...test_ngram_bloomfilter_index_change.groovy | 225 +++--- 12 files changed, 989 insertions(+), 166 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index a5e678eb779897..c147f3542401ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1278,12 +1278,7 @@ private void checkAssignedTargetIndexName(String baseIndexName, String targetInd } private void createJob(String rawSql, long dbId, OlapTable olapTable, Map> indexSchemaMap, - Map propertyMap, List indexes, - boolean isBuildIndex) throws UserException { - if (isBuildIndex) { - // remove the index which is not the base index, only base index can be built index - indexSchemaMap.entrySet().removeIf(entry -> !entry.getKey().equals(olapTable.getBaseIndexId())); - } + Map propertyMap, List indexes) throws UserException { checkReplicaCount(olapTable); // process properties first @@ -1319,7 +1314,7 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map newSet = new HashSet<>(indexes); Set oriSet = new HashSet<>(olapTable.getIndexes()); - if (!newSet.equals(oriSet) || isBuildIndex) { + if (!newSet.equals(oriSet)) { hasIndexChange = true; } @@ -2150,9 +2145,21 @@ public int getAsInt() { } lightSchemaChange = false; + // Check if the index supports light index change and session variable is enabled + boolean enableAddIndexForNewData = true; + try { + ConnectContext context = ConnectContext.get(); + if (context != null && context.getSessionVariable() != null) { + enableAddIndexForNewData = context.getSessionVariable().isEnableAddIndexForNewData(); + } + } catch (Exception e) { + LOG.warn("Failed to get session variable enable_add_index_for_new_data, " + + "using default value: false", e); + } + // ngram_bf index can do light_schema_change in both local and cloud mode // inverted index can only do light_schema_change in local mode - if (index.isLightIndexChangeSupported()) { + if (index.isLightAddIndexSupported(enableAddIndexForNewData)) { alterIndexes.add(index); isDropIndex = false; lightIndexChange = true; @@ -2161,7 +2168,7 @@ public int getAsInt() { BuildIndexClause buildIndexClause = (BuildIndexClause) alterClause; IndexDef indexDef = buildIndexClause.getIndexDef(); Index index = buildIndexClause.getIndex(); - if (Config.isCloudMode() && index.getIndexType() == IndexDef.IndexType.INVERTED) { + if (Config.isCloudMode()) { throw new DdlException("BUILD INDEX operation failed: No need to do it in cloud mode."); } @@ -2223,17 +2230,12 @@ public int getAsInt() { if (alterIndexes.isEmpty()) { throw new DdlException("Altered index is empty. please check your alter stmt."); } - IndexDef.IndexType indexType = alterIndexes.get(0).getIndexType(); if (Config.enable_light_index_change) { - if (indexType == IndexDef.IndexType.INVERTED) { - buildOrDeleteTableInvertedIndices(db, olapTable, indexSchemaMap, - alterIndexes, indexOnPartitions, false); - } else { - createJob(rawSql, db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes, true); - } + buildOrDeleteTableInvertedIndices(db, olapTable, indexSchemaMap, + alterIndexes, indexOnPartitions, false); } } else { - createJob(rawSql, db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes, false); + createJob(rawSql, db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes); } } finally { olapTable.writeUnlock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java index 355b043e3d57c1..7cd1d915150911 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java @@ -107,10 +107,10 @@ public void analyze(Analyzer analyzer) throws AnalysisException { } IndexDef.IndexType indexType = existedIdx.getIndexType(); - if (!existedIdx.isLightIndexChangeSupported()) { - throw new AnalysisException(indexType.toString() + " index is not needed to build."); + if (indexType == IndexDef.IndexType.NGRAM_BF + || indexType == IndexDef.IndexType.BLOOMFILTER) { + throw new AnalysisException("ngram bloomfilter or bloomfilter index is not needed to build."); } - indexDef = new IndexDef(indexName, partitionNames, indexType, true); if (!table.isPartitionedTable()) { List specifiedPartitions = indexDef.getPartitionNames(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index 1b6197524cf431..54407443a24d01 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -153,6 +153,10 @@ public String getInvertedIndexParser() { return InvertedIndexUtil.getInvertedIndexParser(properties); } + public boolean isInvertedIndexParserNone() { + return InvertedIndexUtil.INVERTED_INDEX_PARSER_NONE.equals(getInvertedIndexParser()); + } + public String getInvertedIndexParserMode() { return InvertedIndexUtil.getInvertedIndexParserMode(properties); } @@ -170,16 +174,25 @@ public String getInvertedIndexParserStopwords() { } // Whether the index can be changed in light mode - // cloud mode only supports light change for ngram_bf index - // local mode supports light change for both inverted index and ngram_bf index - // the rest of the index types do not support light change public boolean isLightIndexChangeSupported() { + return indexType == IndexDef.IndexType.INVERTED; + } + + // Whether the index can be added in light mode + // cloud mode supports light add for ngram_bf index and non-tokenized inverted index (parser="none") + // local mode supports light add for both inverted index and ngram_bf index + // the rest of the index types do not support light add + public boolean isLightAddIndexSupported(boolean enableAddIndexForNewData) { if (Config.isCloudMode()) { - return indexType == IndexDef.IndexType.NGRAM_BF; - } else { - return indexType == IndexDef.IndexType.INVERTED - || indexType == IndexDef.IndexType.NGRAM_BF; + if (indexType == IndexDef.IndexType.INVERTED) { + return isInvertedIndexParserNone() && enableAddIndexForNewData; + } else if (indexType == IndexDef.IndexType.NGRAM_BF) { + return enableAddIndexForNewData; + } + return false; } + return (indexType == IndexDef.IndexType.NGRAM_BF && enableAddIndexForNewData) + || (indexType == IndexDef.IndexType.INVERTED); } public String getInvertedIndexCustomAnalyzer() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index bcb17d3c03992f..f4f3893c87d4a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -339,6 +339,14 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2); } else if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V3) { schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V3); + } else if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.DEFAULT) { + if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) { + schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V1); + } else if (Config.inverted_index_storage_format.equalsIgnoreCase("V3")) { + schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V3); + } else { + schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2); + } } else { throw new DdlException("invalid inverted index storage format"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index c268fddee53906..9307a138732a16 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -732,6 +732,7 @@ public class SessionVariable implements Serializable, Writable { public static final String SQL_CONVERTOR_CONFIG = "sql_convertor_config"; public static final String PREFER_UDF_OVER_BUILTIN = "prefer_udf_over_builtin"; + public static final String ENABLE_ADD_INDEX_FOR_NEW_DATA = "enable_add_index_for_new_data"; /** * If set false, user couldn't submit analyze SQL and FE won't allocate any related resources. @@ -2557,6 +2558,13 @@ public void setEnableJoinSpill(boolean enableJoinSpill) { public boolean isEnableSortSpill() { return enableSortSpill; } + @VariableMgr.VarAttr(name = ENABLE_ADD_INDEX_FOR_NEW_DATA, fuzzy = true, description = { + "是否启用仅对新数据生效的索引添加模式，开启时新建索引只对后续写入的数据生效，关闭时对全部数据重建索引", + "Whether to enable add index mode that only affects new data, " + + "when enabled new indexes only affect subsequently written data, " + + "when disabled rebuild indexes for all data" + }) + public boolean enableAddIndexForNewData = false; // If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables, // not the default value set in the code. @@ -4873,5 +4881,13 @@ public boolean getEnableExternalTableBatchMode() { public int getGlobalVariantMaxSubcolumnsCount() { return globalVariantMaxSubcolumnsCount; } + + public boolean isEnableAddIndexForNewData() { + return enableAddIndexForNewData; + } + + public void setEnableAddIndexForNewData(boolean enableAddIndexForNewData) { + this.enableAddIndexForNewData = enableAddIndexForNewData; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java new file mode 100644 index 00000000000000..1836edc7bf6580 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java @@ -0,0 +1,664 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.alter; + +import org.apache.doris.analysis.AlterClause; +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.BuildIndexClause; +import org.apache.doris.analysis.CreateIndexClause; +import org.apache.doris.analysis.DataSortInfo; +import org.apache.doris.analysis.DropIndexClause; +import org.apache.doris.analysis.IndexDef; +import org.apache.doris.analysis.IndexDef.IndexType; +import org.apache.doris.analysis.ResourceTypeEnum; +import org.apache.doris.analysis.TableName; +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.CatalogTestUtil; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.EnvFactory; +import org.apache.doris.catalog.FakeEditLog; +import org.apache.doris.catalog.FakeEnv; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.OlapTable.OlapTableState; +import org.apache.doris.cloud.catalog.CloudEnv; +import org.apache.doris.cloud.catalog.CloudEnvFactory; +import org.apache.doris.cloud.datasource.CloudInternalCatalog; +import org.apache.doris.cloud.proto.Cloud; +import org.apache.doris.cloud.proto.Cloud.MetaServiceCode; +import org.apache.doris.cloud.rpc.MetaServiceProxy; +import org.apache.doris.cloud.system.CloudSystemInfoService; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.UserException; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.Auth; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.commands.CancelBuildIndexCommand; +import org.apache.doris.persist.EditLog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.resource.computegroup.ComputeGroup; +import org.apache.doris.resource.computegroup.ComputeGroupMgr; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.task.AgentTask; +import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; +import org.apache.doris.thrift.TSortType; +import org.apache.doris.thrift.TTaskType; +import org.apache.doris.utframe.MockedMetaServerFactory; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import mockit.Mock; +import mockit.MockUp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class CloudIndexTest { + private static final Logger LOG = LogManager.getLogger(CloudIndexTest.class); + + private static String fileName = "./CloudIndexTest"; + + private static FakeEditLog fakeEditLog; + private static FakeEnv fakeEnv; + private static Env masterEnv; + private static EditLog testEditLog; + private ConnectContext ctx; + + private static Analyzer analyzer; + private static Database db; + private static OlapTable olapTable; + private static CreateIndexClause createIndexClause; + private static BuildIndexClause buildIndexClause; + private static DropIndexClause dropIndexClause; + private static CancelBuildIndexCommand cancelBuildIndexCommand; + private static SchemaChangeHandler schemaChangeHandler; + + @Before + public void setUp() throws InstantiationException, IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException, SecurityException, UserException { + FeConstants.runningUnitTest = true; + // Setup for MetaServiceProxy mock + new MockUp(MetaServiceProxy.class) { + + @Mock + public Cloud.BeginTxnResponse beginTxn(Cloud.BeginTxnRequest request) { + Cloud.BeginTxnResponse.Builder beginTxnResponseBuilder = Cloud.BeginTxnResponse.newBuilder(); + beginTxnResponseBuilder.setTxnId(1000) + .setStatus( + Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK).setMsg("OK")); + return beginTxnResponseBuilder.build(); + } + + @Mock + public Cloud.CommitTxnResponse commitTxn(Cloud.CommitTxnRequest request) { + Cloud.TxnInfoPB.Builder txnInfoBuilder = Cloud.TxnInfoPB.newBuilder(); + txnInfoBuilder.setDbId(CatalogTestUtil.testDbId1); + txnInfoBuilder.addAllTableIds(Lists.newArrayList(olapTable.getId())); + txnInfoBuilder.setLabel("test_label"); + txnInfoBuilder.setListenerId(-1); + Cloud.CommitTxnResponse.Builder commitTxnResponseBuilder = Cloud.CommitTxnResponse.newBuilder(); + commitTxnResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .setTxnInfo(txnInfoBuilder.build()); + return commitTxnResponseBuilder.build(); + } + + @Mock + public Cloud.CheckTxnConflictResponse checkTxnConflict(Cloud.CheckTxnConflictRequest request) { + Cloud.CheckTxnConflictResponse.Builder checkTxnConflictResponseBuilder = + Cloud.CheckTxnConflictResponse.newBuilder(); + checkTxnConflictResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .setFinished(true); + return checkTxnConflictResponseBuilder.build(); + } + + @Mock + public Cloud.GetClusterResponse getCluster(Cloud.GetClusterRequest request) { + Cloud.GetClusterResponse.Builder getClusterResponseBuilder = Cloud.GetClusterResponse.newBuilder(); + Cloud.ClusterPB.Builder clusterBuilder = Cloud.ClusterPB.newBuilder(); + clusterBuilder.setClusterId("test_id").setClusterName("test_group"); + + Cloud.NodeInfoPB.Builder node1 = Cloud.NodeInfoPB.newBuilder(); + node1.setCloudUniqueId("test_cloud") + .setName("host1") + .setIp("host1") + .setHost("host1") + .setHeartbeatPort(123) + .setEditLogPort(125) + .setStatus(Cloud.NodeStatusPB.NODE_STATUS_RUNNING); + clusterBuilder.addNodes(node1.build()); + getClusterResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .addCluster(clusterBuilder.build()); + return getClusterResponseBuilder.build(); + } + + @Mock + public Cloud.CreateTabletsResponse createTablets(Cloud.CreateTabletsRequest request) { + Cloud.CreateTabletsResponse.Builder responseBuilder = Cloud.CreateTabletsResponse.newBuilder(); + responseBuilder.setStatus( + Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK).setMsg("OK")); + return responseBuilder.build(); + } + + @Mock + public Cloud.FinishTabletJobResponse finishTabletJob(Cloud.FinishTabletJobRequest request) { + Cloud.FinishTabletJobResponse.Builder responseBuilder = Cloud.FinishTabletJobResponse.newBuilder(); + responseBuilder.setStatus( + Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK).setMsg("OK")); + return responseBuilder.build(); + } + + @Mock + public Cloud.IndexResponse prepareIndex(Cloud.IndexRequest request) { + Cloud.IndexResponse.Builder builder = Cloud.IndexResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.IndexResponse commitIndex(Cloud.IndexRequest request) { + Cloud.IndexResponse.Builder builder = Cloud.IndexResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.IndexResponse dropIndex(Cloud.IndexRequest request) { + Cloud.IndexResponse.Builder builder = Cloud.IndexResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.CheckKVResponse checkKv(Cloud.CheckKVRequest request) { + Cloud.CheckKVResponse.Builder builder = Cloud.CheckKVResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.GetCurrentMaxTxnResponse getCurrentMaxTxnId(Cloud.GetCurrentMaxTxnRequest request) { + Cloud.GetCurrentMaxTxnResponse.Builder builder = Cloud.GetCurrentMaxTxnResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .setCurrentMaxTxnId(1000); + return builder.build(); + } + }; + + Config.cloud_unique_id = "test_cloud"; + Config.meta_service_endpoint = MockedMetaServerFactory.METASERVER_DEFAULT_IP + ":" + 20121; + + EnvFactory envFactory = EnvFactory.getInstance(); + masterEnv = envFactory.createEnv(false); + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + FakeEnv.setSystemInfo(cloudSystemInfo); + + fakeEditLog = new FakeEditLog(); + testEditLog = null; // Will be set by MockUp + FakeEnv.setEnv(masterEnv); + + ctx = new ConnectContext(); + ctx.setEnv(masterEnv); + ctx.setQualifiedUser("root"); + UserIdentity rootUser = new UserIdentity("root", "%"); + rootUser.setIsAnalyzed(); + ctx.setCurrentUserIdentity(rootUser); + ctx.setThreadLocalInfo(); + ctx.setCloudCluster("test_group"); + Assert.assertTrue(envFactory instanceof CloudEnvFactory); + Assert.assertTrue(masterEnv instanceof CloudEnv); + new MockUp() { + @Mock + public Env getCurrentEnv() { + return masterEnv; + } + + @Mock + public EditLog getEditLog() { + if (testEditLog == null) { + // Create a mock EditLog using a no-op approach + testEditLog = new EditLog("test") { + // Override to avoid initialization issues + }; + } + return testEditLog; + } + + @Mock + public ComputeGroupMgr getComputeGroupMgr() { + return new ComputeGroupMgr(Env.getCurrentSystemInfo()); + } + + @Mock + public SchemaChangeHandler getSchemaChangeHandler() { + // Create a new independent SchemaChangeHandler for each call + return schemaChangeHandler; + } + + @Mock + public AccessControllerManager getAccessManager() { + return new AccessControllerManager(masterEnv.getAuth()) { + @Override + public boolean checkTblPriv(ConnectContext ctx, String ctl, String db, String tbl, PrivPredicate wanted) { + return true; // Allow all access for test + } + + @Override + public boolean checkCloudPriv(UserIdentity user, String cluster, PrivPredicate wanted, ResourceTypeEnum resourceType) { + return true; // Allow all cloud privileges for test + } + }; + } + }; + + new MockUp() { + @Mock + public String getDefaultCloudCluster(String user) { + return "test_group"; // Return default cluster for test + } + + @Mock + public ComputeGroup getComputeGroup(String user) { + try { + return masterEnv.getComputeGroupMgr().getComputeGroupByName("test_group"); + } catch (Exception e) { + return masterEnv.getComputeGroupMgr().getAllBackendComputeGroup(); + } + } + }; + + // Mock cloud environment permissions + new MockUp() { + @Mock + public void checkCloudClusterPriv(String cluster) throws Exception { + // Always allow for tests + } + }; + + // Mock ConnectContext to avoid compute group permission check + new MockUp() { + @Mock + public String getCloudCluster() { + return "test_group"; + } + + @Mock + public UserIdentity getCurrentUserIdentity() { + UserIdentity rootUser = new UserIdentity("root", "%"); + rootUser.setIsAnalyzed(); + return rootUser; + } + }; + + analyzer = new Analyzer(masterEnv, ctx); + + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + // Mock addCloudCluster to avoid EditLog issues + new MockUp() { + @Mock + public void addCloudCluster(String clusterName, String clusterId) { + // Create backend manually for test + Backend backend = new Backend(10001L, "host1", 123); + backend.setAlive(true); + backend.setBePort(456); + backend.setHttpPort(789); + backend.setBrpcPort(321); + backend.setTagMap(Maps.newHashMap()); + backend.getTagMap().put("cloud_cluster_id", "test_id"); + backend.getTagMap().put("cloud_unique_id", "test_cloud"); + backend.getTagMap().put("cloud_cluster_name", "test_group"); + backend.getTagMap().put("cloud_cluster_status", "NORMAL"); + backend.getTagMap().put("location", "default"); + backend.getTagMap().put("cloud_cluster_private_endpoint", ""); + backend.getTagMap().put("cloud_cluster_public_endpoint", ""); + CloudSystemInfoService systemInfo = (CloudSystemInfoService) Env.getCurrentSystemInfo(); + systemInfo.addBackend(backend); + } + }; + ((CloudSystemInfoService) Env.getCurrentSystemInfo()).addCloudCluster("test_group", ""); + List backends = + ((CloudSystemInfoService) Env.getCurrentSystemInfo()).getBackendsByClusterName("test_group"); + Assert.assertEquals(1, backends.size()); + Assert.assertEquals("host1", backends.get(0).getHost()); + backends.get(0).setAlive(true); + ctx.setComputeGroup(masterEnv.getComputeGroupMgr().getAllBackendComputeGroup()); + + db = new Database(CatalogTestUtil.testDbId1, CatalogTestUtil.testDb1); + masterEnv.unprotectCreateDb(db); + + AgentTaskQueue.clearAllTasks(); + schemaChangeHandler = masterEnv.getSchemaChangeHandler(); + } + + @Test + public void testCreateNgramBfIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + table.setDataSortInfo(dataSortInfo); + String indexName = "ngram_bf_index"; + + // Add required properties for NGRAM_BF index + Map properties = Maps.newHashMap(); + properties.put("gram_size", "2"); + properties.put("bf_size", "256"); + + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + properties, "ngram bf index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + ctx.getSessionVariable().setEnableAddIndexForNewData(true); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); + Assert.assertEquals(OlapTableState.NORMAL, table.getState()); + + long createJobId = indexChangeJobMap.values().stream().findAny().get().jobId; + + // Finish the create index job first + SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2) indexChangeJobMap.get(createJobId); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, createJobV2.getJobState()); + } + + @Test + public void testNormalCreateNgramBfIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + table.setDataSortInfo(dataSortInfo); + String indexName = "ngram_bf_index"; + + // Add required properties for NGRAM_BF index + Map properties = Maps.newHashMap(); + properties.put("gram_size", "2"); + properties.put("bf_size", "256"); + + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + properties, "ngram bf index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + // Set session variable to false (default) + ctx.getSessionVariable().setEnableAddIndexForNewData(false); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + + long createJobId = indexChangeJobMap.values().stream().findAny().get().jobId; + + // Finish the create index job first + SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2) indexChangeJobMap.get(createJobId); + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, createJobV2.getJobState()); + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, createJobV2.getJobState()); + Assert.assertEquals(1, createJobV2.schemaChangeBatchTask.getTaskNum()); + + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, createJobV2.getJobState()); + Assert.assertEquals(OlapTableState.NORMAL, table.getState()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); + } + + @Test + public void testCreateInvertedIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + table.setDataSortInfo(dataSortInfo); + String indexName = "raw_inverted_index"; + // Explicitly set parser="none" for raw inverted index + Map properties = Maps.newHashMap(); + properties.put("parser", "none"); + + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + IndexType.INVERTED, + properties, "raw inverted index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + ctx.getSessionVariable().setEnableAddIndexForNewData(false); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + + long createJobId = indexChangeJobMap.values().stream().findAny().get().jobId; + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + + // Finish the create index job first + SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2) indexChangeJobMap.get(createJobId); + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, createJobV2.getJobState()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, createJobV2.getJobState()); + Assert.assertEquals(1, createJobV2.schemaChangeBatchTask.getTaskNum()); + + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, createJobV2.getJobState()); + Assert.assertEquals(OlapTableState.NORMAL, table.getState()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("raw_inverted_index", table.getIndexes().get(0).getIndexName()); + } + + @Test + public void testCreateInvertedIndexWithLightweightMode() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + table.setDataSortInfo(dataSortInfo); + String indexName = "lightweight_raw_inverted_index"; + // Explicitly set parser="none" for raw inverted index + Map properties = Maps.newHashMap(); + properties.put("parser", "none"); + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + IndexType.INVERTED, + properties, "lightweight raw inverted index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + // Test with enable_add_index_for_new_data = true, should use lightweight mode + ctx.getSessionVariable().setEnableAddIndexForNewData(true); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + // Lightweight mode should not create any schema change jobs + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("lightweight_raw_inverted_index", table.getIndexes().get(0).getIndexName()); + Assert.assertEquals(OlapTableState.NORMAL, table.getState()); + // Verify the index properties + Assert.assertEquals("none", table.getIndexes().get(0).getProperties().get("parser")); + } + + @Test + public void testCreateTokenizedInvertedIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + table.setDataSortInfo(dataSortInfo); + + // Set inverted index file storage format to V2 for cloud mode + table.setInvertedIndexFileStorageFormat(TInvertedIndexFileStorageFormat.V2); + + String indexName = "tokenized_inverted_index"; + Map properties = Maps.newHashMap(); + properties.put("parser", "english"); + properties.put("support_phrase", "true"); + properties.put("lower_case", "true"); + + // Use VARCHAR column v1 (index 2) for string type support + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(2).getName()), + IndexType.INVERTED, + properties, "tokenized inverted index with english parser"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + + SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() + .findFirst() + .orElse(null); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + // This should be a heavyweight schema change for tokenized index + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, jobV2.getJobState()); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); + + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("tokenized_inverted_index", table.getIndexes().get(0).getIndexName()); + + // Verify that the index has the correct properties + Assert.assertEquals("english", table.getIndexes().get(0).getProperties().get("parser")); + Assert.assertEquals("true", table.getIndexes().get(0).getProperties().get("support_phrase")); + Assert.assertEquals("true", table.getIndexes().get(0).getProperties().get("lower_case")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java index 20bd26fa92db50..97eaa209679b42 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java @@ -46,6 +46,9 @@ import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.UserException; import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.nereids.trees.plans.commands.CancelAlterTableCommand; +import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.thrift.TStatusCode; @@ -89,6 +92,7 @@ public class IndexChangeJobTest { private static CancelAlterTableStmt cancelAlterTableStmt; private static TableName tableName; private static String indexName; + private static ConnectContext ctx; @Rule public ExpectedException expectedEx = ExpectedException.none(); @@ -119,6 +123,15 @@ public Env getCurrentEnv() { } }; + // Initialize ConnectContext + ctx = new ConnectContext(); + new MockUp() { + @Mock + public ConnectContext get() { + return ctx; + } + }; + // set mow table property Map properties = Maps.newHashMap(); properties.put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE, "false"); @@ -600,25 +613,45 @@ public void testNgramBfBuildIndex() throws UserException { SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); ArrayList alterClauses = new ArrayList<>(); alterClauses.add(createIndexClause); + + // Test with enable_add_index_for_new_data = true + ConnectContext context = ConnectContext.get(); + context.getSessionVariable().setEnableAddIndexForNewData(true); schemaChangeHandler.process(alterClauses, db, table); Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); Assert.assertEquals(1, indexChangeJobMap.size()); Assert.assertEquals(1, table.getIndexes().size()); Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); - long jobId = indexChangeJobMap.values().stream().findAny().get().jobId; + SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() + .findFirst() + .orElse(null); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); - buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); - buildIndexClause.analyze(analyzer); - alterClauses.clear(); - alterClauses.add(buildIndexClause); + // Clean up for next test + table.setIndexes(Lists.newArrayList()); + indexChangeJobMap.clear(); + AgentTaskQueue.clearAllTasks(); - schemaChangeHandler.process(alterClauses, db, table); - Assert.assertEquals(2, indexChangeJobMap.size()); + // Test with enable_add_index_for_new_data = false + context.getSessionVariable().setEnableAddIndexForNewData(false); + String indexName2 = "ngram_bf_index2"; + IndexDef indexDef2 = new IndexDef(indexName2, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + Maps.newHashMap(), "ngram bf index2"); + + createIndexClause = new CreateIndexClause(tableName, indexDef2, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses2 = new ArrayList<>(); + alterClauses2.add(createIndexClause); + schemaChangeHandler.process(alterClauses2, db, table); + indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); - SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() - .filter(job -> job.jobId != jobId) + jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() .findFirst() .orElse(null); Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); @@ -643,6 +676,8 @@ public void testNgramBfBuildIndex() throws UserException { schemaChangeHandler.runAfterCatalogReady(); Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("ngram_bf_index2", table.getIndexes().get(0).getIndexName()); } @Test @@ -664,25 +699,15 @@ public void testCancelNgramBfBuildIndex() throws UserException { SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); ArrayList alterClauses = new ArrayList<>(); alterClauses.add(createIndexClause); + + //cancel test can only with enable_add_index_for_new_data = false + ctx.getSessionVariable().setEnableAddIndexForNewData(false); schemaChangeHandler.process(alterClauses, db, table); Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); Assert.assertEquals(1, indexChangeJobMap.size()); - Assert.assertEquals(1, table.getIndexes().size()); - Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); - - long jobId = indexChangeJobMap.values().stream().findAny().get().jobId; - - buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); - buildIndexClause.analyze(analyzer); - alterClauses.clear(); - alterClauses.add(buildIndexClause); - - schemaChangeHandler.process(alterClauses, db, table); - Assert.assertEquals(2, indexChangeJobMap.size()); Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() - .filter(job -> job.jobId != jobId) .findFirst() .orElse(null); Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java index 7099008be67720..77693efa6e4607 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java @@ -850,14 +850,8 @@ public void testDupAddOrDropNgramBfIndex() throws Exception { waitAlterJobDone(alterJobs); String buildNgramBfIndexStmtStr = "BUILD INDEX idx_error_msg on test.sc_dup "; - AlterTableStmt buildNgramBfIndexStmt = (AlterTableStmt) parseAndAnalyzeStmt(buildNgramBfIndexStmtStr); - Env.getCurrentEnv().getAlterInstance().processAlterTable(buildNgramBfIndexStmt); - - jobSize++; - alterJobs = Env.getCurrentEnv().getSchemaChangeHandler().getAlterJobsV2(); - LOG.info("alterJobs:{}", alterJobs); - Assertions.assertEquals(jobSize, alterJobs.size()); - waitAlterJobDone(alterJobs); + Assertions.assertThrows(org.apache.doris.common.AnalysisException.class, + () -> parseAndAnalyzeStmt(buildNgramBfIndexStmtStr)); tbl.readLock(); try { diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java index 590b6563e1106c..b16d3d15cf7e52 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java @@ -22,7 +22,8 @@ import org.apache.doris.analysis.SinglePartitionDesc; import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MaterializedIndex.IndexState; -import org.apache.doris.catalog.Replica.ReplicaState; +import org.apache.doris.cloud.catalog.CloudReplica; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.persist.EditLog; @@ -175,13 +176,26 @@ public static Database createSimpleDb(long dbId, long tableId, long partitionId, long version) { Env.getCurrentInvertedIndex().clear(); - // replica - Replica replica1 = new Replica(testReplicaId1, testBackendId1, version, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); - Replica replica2 = new Replica(testReplicaId2, testBackendId2, version, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); - Replica replica3 = new Replica(testReplicaId3, testBackendId3, version, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); + Replica replica1; + Replica replica2; + Replica replica3; + if (Config.isCloudMode()) { + // In cloud mode we must create CloudReplica instances to avoid ClassCastException + replica1 = new CloudReplica(testReplicaId1, testBackendId1, Replica.ReplicaState.NORMAL, version, + /*schemaHash*/ 0, dbId, tableId, partitionId, indexId, /*idx*/ 0); + replica2 = new CloudReplica(testReplicaId2, testBackendId2, Replica.ReplicaState.NORMAL, version, + 0, dbId, tableId, partitionId, indexId, 1); + replica3 = new CloudReplica(testReplicaId3, testBackendId3, Replica.ReplicaState.NORMAL, version, + 0, dbId, tableId, partitionId, indexId, 2); + } else { + replica1 = new Replica(testReplicaId1, testBackendId1, version, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + replica2 = new Replica(testReplicaId2, testBackendId2, version, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + replica3 = new Replica(testReplicaId3, testBackendId3, version, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + } + // tablet Tablet tablet = new Tablet(tabletId); @@ -244,10 +258,14 @@ public static Database createSimpleDb(long dbId, long tableId, long partitionId, } public static void createDupTable(Database db) { - - // replica - Replica replica = new Replica(testReplicaId4, testBackendId1, testStartVersion, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); + Replica replica; + if (Config.isCloudMode()) { + replica = new CloudReplica(testReplicaId4, testBackendId1, Replica.ReplicaState.NORMAL, testStartVersion, + 0, db.getId(), testTableId2, testPartitionId2, testIndexId2, 0); + } else { + replica = new Replica(testReplicaId4, testBackendId1, testStartVersion, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + } // tablet Tablet tablet = new Tablet(testTabletId2); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEditLog.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEditLog.java index 08800510a1a6ef..e2b113958366c8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEditLog.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEditLog.java @@ -100,6 +100,16 @@ public void logModifyDistributionType(TableInfo tableInfo) { } + @Mock + public void logAddBackend(Backend be) { + // do nothing for test + } + + @Mock + public int getNumEditStreams() { + return 1; // fake that we have streams + } + public TransactionState getTransaction(long transactionId) { return allTransactionState.get(transactionId); } diff --git a/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out b/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out index 6f916a99c91705..49570b96d291f2 100644 --- a/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out +++ b/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out @@ -1,5 +1,5 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !select -- +-- !select_light_mode_init -- 1001 2023-10-06T15:00 Laptop John Smith 199.99 North 1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South 1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East @@ -11,7 +11,7 @@ 1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North 1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South --- !select -- +-- !select_light_mode_more_data -- 1001 2023-10-06T15:00 Laptop John Smith 199.99 North 1001 2023-10-06T15:00 Laptop John Smith 199.99 North 1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South @@ -33,7 +33,7 @@ 1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South 1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South --- !select -- +-- !select_schema_change_mode_init -- 1001 2023-10-06T15:00 Laptop John Smith 199.99 North 1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South 1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East @@ -45,7 +45,41 @@ 1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North 1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South --- !select -- +-- !select_schema_change_mode_more_data -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select_lifecycle_after_data -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select_lifecycle_final -- 1001 2023-10-06T15:00 Laptop John Smith 199.99 North 1001 2023-10-06T15:00 Laptop John Smith 199.99 North 1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South diff --git a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy index 6fea7a68f92b8e..402f72ddc36065 100644 --- a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy +++ b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy @@ -54,13 +54,16 @@ suite("test_ngram_bloomfilter_index_change") { sql "sync" } - // Test setup - // 1. Create table - // 2. Insert test data - // 3. Add NGRAM Bloom Filter index - // 4. Build index - // 5. Insert more data - // 6. Drop index + // Test settings + sql "set enable_function_pushdown=true" + sql "set enable_profile=true" + sql "set profile_level=2" + + // Define test query + def query = "SELECT /*+SET_VAR(enable_function_pushdown = true, enable_profile = true, profile_level = 2)*/ * FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id" + // Test Case 1: Test with enable_add_index_for_new_data = true + logger.info("=== Test Case 1: enable_add_index_for_new_data = true ===") + // Create table sql "DROP TABLE IF EXISTS ${tableName}" sql """ CREATE TABLE ${tableName} ( @@ -82,29 +85,20 @@ suite("test_ngram_bloomfilter_index_change") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false" + "disable_auto_compaction" = "true" ); """ - // Insert first batch of data + // Insert test data insertTestData() - - // Test settings - sql "set enable_function_pushdown=true" - sql "set enable_profile=true" - sql "set profile_level=2" - // Verify data loaded correctly - qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" + qt_select_light_mode_init "SELECT * FROM ${tableName} ORDER BY sale_id" - // Define test query - def query = "SELECT /*+SET_VAR(enable_function_pushdown = true, enable_profile = true, profile_level = 2)*/ * FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id" - - // Test 1: without NGRAM Bloom Filter index - profile("sql_select_like_without_ngram_index") { + // Test without NGRAM Bloom Filter index + profile("sql_select_like_without_ngram_index_light_mode") { run { - sql "/* sql_select_like_without_ngram_index */ ${query}" - sleep(1000) // sleep 1s wait for the profile collection to be completed + sql "/* sql_select_like_without_ngram_index_light_mode */ ${query}" + sleep(1000) } check { profileString, exception -> @@ -112,13 +106,41 @@ suite("test_ngram_bloomfilter_index_change") { assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) } } + sql "set enable_add_index_for_new_data = true" - // Test 2: After adding NGRAM Bloom Filter index + // Add NGRAM Bloom Filter index (should be immediate in light mode) sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" + + // In light mode, the index should be effective immediately, no need to wait for alter job + // But let's give it a moment to ensure metadata is updated + sleep(2000) + + // Insert more data after index added + insertTestData() + // Verify more data loaded correctly + qt_select_light_mode_more_data "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test with more data (should still filter correctly) + profile("sql_select_like_with_ngram_index_light_mode_more_data") { + run { + sql "/* sql_select_like_with_ngram_index_light_mode_more_data */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 10")) + } + } + + // Drop index + sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" wait_for_latest_op_on_table_finish(tableName, timeout) - profile("sql_select_like_with_ngram_index_added") { + + // Test after dropping index + profile("sql_select_like_with_ngram_index_light_mode_dropped") { run { - sql "/* sql_select_like_with_ngram_index_added */ ${query}" + sql "/* sql_select_like_with_ngram_index_light_mode_dropped */ ${query}" sleep(1000) } @@ -128,12 +150,61 @@ suite("test_ngram_bloomfilter_index_change") { } } - // Test 3: After building the index - sql "BUILD INDEX idx_ngram_customer_name ON ${tableName};" + // Test Case 2: Test with enable_add_index_for_new_data = false (schema change mode) + logger.info("=== Test Case 2: enable_add_index_for_new_data = false ===") + // Set enable_add_index_for_new_data = false + sql "set enable_add_index_for_new_data = false" + // Create new table + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + `sale_id` int NULL, + `sale_date` datetime NULL, + `product_name` varchar(100) NULL, + `customer_name` varchar(100) NULL, + `amount` decimal(10,2) NULL, + `region` char(50) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`sale_id`) + PARTITION BY RANGE(`sale_date`) ( + PARTITION p202310 VALUES [('2023-10-01 00:00:00'), ('2023-11-01 00:00:00')), + PARTITION p202311 VALUES [('2023-11-01 00:00:00'), ('2023-12-01 00:00:00')), + PARTITION p202312 VALUES [('2023-12-01 00:00:00'), ('2024-01-01 00:00:00')) + ) + DISTRIBUTED BY HASH(`sale_id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "true" + ); + """ + // Insert test data + insertTestData() + // Verify data loaded correctly + qt_select_schema_change_mode_init "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test without NGRAM Bloom Filter index + profile("sql_select_like_without_ngram_index_schema_change_mode") { + run { + sql "/* sql_select_like_without_ngram_index_schema_change_mode */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } + + // Add NGRAM Bloom Filter index (will trigger schema change in this mode) + sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" wait_for_latest_op_on_table_finish(tableName, timeout) - profile("sql_select_like_with_ngram_index_built") { + + // Test after adding NGRAM Bloom Filter index (should filter existing data) + profile("sql_select_like_with_ngram_index_schema_change_mode_added") { run { - sql "/* sql_select_like_with_ngram_index_built */ ${query}" + sql "/* sql_select_like_with_ngram_index_schema_change_mode_added */ ${query}" sleep(1000) } @@ -143,15 +214,15 @@ suite("test_ngram_bloomfilter_index_change") { } } - // Insert second batch of data + // Insert more data after index is built insertTestData() - // Verify data loaded correctly - qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" + // Verify more data loaded correctly + qt_select_schema_change_mode_more_data "SELECT * FROM ${tableName} ORDER BY sale_id" - // Test 4: Verify filtering with more data - profile("sql_select_like_with_ngram_index_more_data") { + // Test with more data (should filter all data) + profile("sql_select_like_with_ngram_index_schema_change_mode_more_data") { run { - sql "/* sql_select_like_with_ngram_index_more_data */ ${query}" + sql "/* sql_select_like_with_ngram_index_schema_change_mode_more_data */ ${query}" sleep(1000) } @@ -161,12 +232,14 @@ suite("test_ngram_bloomfilter_index_change") { } } - // Test 5: After dropping the index + // Drop index sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" wait_for_latest_op_on_table_finish(tableName, timeout) - profile("sql_select_like_with_ngram_index_dropped") { + + // Test after dropping index + profile("sql_select_like_with_ngram_index_schema_change_mode_dropped") { run { - sql "/* sql_select_like_with_ngram_index_dropped */ ${query}" + sql "/* sql_select_like_with_ngram_index_schema_change_mode_dropped */ ${query}" sleep(1000) } @@ -176,13 +249,11 @@ suite("test_ngram_bloomfilter_index_change") { } } - // recreate table - // 1. Create table - // 2. Add NGRAM Bloom Filter index - // 3. Insert data - // 4. Insert more data - // 5. Build index - // 6. Drop index + // Test Case 3: Test different scenarios for index lifecycle + logger.info("=== Test Case 3: Index lifecycle with enable_add_index_for_new_data = true ===") + // Set enable_add_index_for_new_data = true + sql "set enable_add_index_for_new_data = true" + // Create table and add index before inserting data sql "DROP TABLE IF EXISTS ${tableName}" sql """ CREATE TABLE ${tableName} ( @@ -204,24 +275,23 @@ suite("test_ngram_bloomfilter_index_change") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false" + "disable_auto_compaction" = "true" ); """ - // add ngram bf index + // Add ngram bf index before data insertion sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" wait_for_latest_op_on_table_finish(tableName, timeout) - // insert data + // Insert data after index creation insertTestData() - // Verify data loaded correctly - qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" + qt_select_lifecycle_after_data "SELECT * FROM ${tableName} ORDER BY sale_id" - // Test 6: Verify filtering with index added - profile("sql_select_like_with_ngram_index_recreated") { + // Test filtering with index added before data insertion + profile("sql_select_like_with_ngram_index_lifecycle_test") { run { - sql "/* sql_select_like_with_ngram_index_recreated */ ${query}" + sql "/* sql_select_like_with_ngram_index_lifecycle_test */ ${query}" sleep(1000) } @@ -231,33 +301,15 @@ suite("test_ngram_bloomfilter_index_change") { } } - // insert more data + // Insert more data insertTestData() - - // Verify data loaded correctly - qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" - - // Test 7: Verify filtering with more data - profile("sql_select_like_with_ngram_index_recreated_more_data") { - run { - sql "/* sql_select_like_with_ngram_index_recreated_more_data */ ${query}" - sleep(1000) - } - - check { profileString, exception -> - log.info(profileString) - assertTrue(profileString.contains("RowsBloomFilterFiltered: 20")) - } - } - - // build index - sql "BUILD INDEX idx_ngram_customer_name ON ${tableName};" - wait_for_latest_op_on_table_finish(tableName, timeout) + // Verify more data loaded correctly + qt_select_lifecycle_final "SELECT * FROM ${tableName} ORDER BY sale_id" - // Test 8: Verify filtering with index built - profile("sql_select_like_with_ngram_index_recreated_built") { + // Test filtering with more data + profile("sql_select_like_with_ngram_index_lifecycle_final") { run { - sql "/* sql_select_like_with_ngram_index_recreated_built */ ${query}" + sql "/* sql_select_like_with_ngram_index_lifecycle_final */ ${query}" sleep(1000) } @@ -267,20 +319,7 @@ suite("test_ngram_bloomfilter_index_change") { } } - // drop index + // Final cleanup sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" - wait_for_latest_op_on_table_finish(tableName, timeout) - - // Test 9: Verify filtering with index dropped - profile("sql_select_like_with_ngram_index_recreated_dropped") { - run { - sql "/* sql_select_like_with_ngram_index_recreated_dropped */ ${query}" - sleep(1000) - } - - check { profileString, exception -> - log.info(profileString) - assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) - } - } + sleep(2000) } \ No newline at end of file From 38c19fb52fbac9a0bf7f785b0ac50f26c1d53166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Tue, 8 Jul 2025 10:48:35 +0800 Subject: [PATCH 3/7] fix style --- .../src/main/java/org/apache/doris/qe/SessionVariable.java | 1 + .../test/java/org/apache/doris/alter/IndexChangeJobTest.java | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 9307a138732a16..7891b735d03a1d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -2558,6 +2558,7 @@ public void setEnableJoinSpill(boolean enableJoinSpill) { public boolean isEnableSortSpill() { return enableSortSpill; } + @VariableMgr.VarAttr(name = ENABLE_ADD_INDEX_FOR_NEW_DATA, fuzzy = true, description = { "是否启用仅对新数据生效的索引添加模式，开启时新建索引只对后续写入的数据生效，关闭时对全部数据重建索引", "Whether to enable add index mode that only affects new data, " diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java index 97eaa209679b42..d4bcf9ab02066e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java @@ -46,8 +46,6 @@ import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.UserException; import org.apache.doris.common.util.PropertyAnalyzer; -import org.apache.doris.nereids.trees.plans.commands.CancelAlterTableCommand; -import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; import org.apache.doris.qe.ConnectContext; import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskQueue; From 27c87e93f9f4233e0286e23da0739ba987018700 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Tue, 8 Jul 2025 14:39:05 +0800 Subject: [PATCH 4/7] fix ut --- .../apache/doris/alter/CloudIndexTest.java | 137 +++++++++--------- 1 file changed, 71 insertions(+), 66 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java index 1836edc7bf6580..e96d7b8117b4f3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java @@ -19,10 +19,8 @@ import org.apache.doris.analysis.AlterClause; import org.apache.doris.analysis.Analyzer; -import org.apache.doris.analysis.BuildIndexClause; import org.apache.doris.analysis.CreateIndexClause; import org.apache.doris.analysis.DataSortInfo; -import org.apache.doris.analysis.DropIndexClause; import org.apache.doris.analysis.IndexDef; import org.apache.doris.analysis.IndexDef.IndexType; import org.apache.doris.analysis.ResourceTypeEnum; @@ -38,6 +36,7 @@ import org.apache.doris.catalog.OlapTable.OlapTableState; import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.cloud.catalog.CloudEnvFactory; +import org.apache.doris.cloud.catalog.ComputeGroup; import org.apache.doris.cloud.datasource.CloudInternalCatalog; import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.cloud.proto.Cloud.MetaServiceCode; @@ -49,11 +48,8 @@ import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.Auth; import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.nereids.trees.plans.commands.CancelBuildIndexCommand; import org.apache.doris.persist.EditLog; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.resource.computegroup.ComputeGroup; -import org.apache.doris.resource.computegroup.ComputeGroupMgr; import org.apache.doris.system.Backend; import org.apache.doris.system.SystemInfoService; import org.apache.doris.task.AgentTask; @@ -81,6 +77,28 @@ public class CloudIndexTest { private static final Logger LOG = LogManager.getLogger(CloudIndexTest.class); + // Simple ComputeGroupMgr wrapper for CloudSystemInfoService + public static class ComputeGroupMgr { + private final SystemInfoService systemInfoService; + + public ComputeGroupMgr(SystemInfoService systemInfoService) { + this.systemInfoService = systemInfoService; + } + + public ComputeGroup getComputeGroupByName(String name) { + if (systemInfoService instanceof CloudSystemInfoService) { + return ((CloudSystemInfoService) systemInfoService).getComputeGroupByName(name); + } + return null; + } + + public ComputeGroup getAllBackendComputeGroup() { + // Return a default compute group for all backends + return new ComputeGroup("default_compute_group", "default_compute_group", + ComputeGroup.ComputeTypeEnum.SQL); + } + } + private static String fileName = "./CloudIndexTest"; private static FakeEditLog fakeEditLog; @@ -88,14 +106,11 @@ public class CloudIndexTest { private static Env masterEnv; private static EditLog testEditLog; private ConnectContext ctx; + private static OlapTable olapTable; private static Analyzer analyzer; private static Database db; - private static OlapTable olapTable; private static CreateIndexClause createIndexClause; - private static BuildIndexClause buildIndexClause; - private static DropIndexClause dropIndexClause; - private static CancelBuildIndexCommand cancelBuildIndexCommand; private static SchemaChangeHandler schemaChangeHandler; @Before @@ -289,15 +304,6 @@ public boolean checkCloudPriv(UserIdentity user, String cluster, PrivPredicate w public String getDefaultCloudCluster(String user) { return "test_group"; // Return default cluster for test } - - @Mock - public ComputeGroup getComputeGroup(String user) { - try { - return masterEnv.getComputeGroupMgr().getComputeGroupByName("test_group"); - } catch (Exception e) { - return masterEnv.getComputeGroupMgr().getAllBackendComputeGroup(); - } - } }; // Mock cloud environment permissions @@ -354,7 +360,6 @@ public void addCloudCluster(String clusterName, String clusterId) { Assert.assertEquals(1, backends.size()); Assert.assertEquals("host1", backends.get(0).getHost()); backends.get(0).setAlive(true); - ctx.setComputeGroup(masterEnv.getComputeGroupMgr().getAllBackendComputeGroup()); db = new Database(CatalogTestUtil.testDbId1, CatalogTestUtil.testDb1); masterEnv.unprotectCreateDb(db); @@ -377,10 +382,10 @@ public void testCreateNgramBfIndex() throws Exception { Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); CatalogTestUtil.createDupTable(db); - OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); DataSortInfo dataSortInfo = new DataSortInfo(); dataSortInfo.setSortType(TSortType.LEXICAL); - table.setDataSortInfo(dataSortInfo); + olapTable.setDataSortInfo(dataSortInfo); String indexName = "ngram_bf_index"; // Add required properties for NGRAM_BF index @@ -389,22 +394,22 @@ public void testCreateNgramBfIndex() throws Exception { properties.put("bf_size", "256"); IndexDef indexDef = new IndexDef(indexName, false, - Lists.newArrayList(table.getBaseSchema().get(3).getName()), + Lists.newArrayList(olapTable.getBaseSchema().get(3).getName()), org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, properties, "ngram bf index"); TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), - table.getName()); + olapTable.getName()); createIndexClause = new CreateIndexClause(tableName, indexDef, false); createIndexClause.analyze(analyzer); ArrayList alterClauses = new ArrayList<>(); alterClauses.add(createIndexClause); ctx.getSessionVariable().setEnableAddIndexForNewData(true); - schemaChangeHandler.process(alterClauses, db, table); + schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); Assert.assertEquals(1, indexChangeJobMap.size()); - Assert.assertEquals(1, table.getIndexes().size()); - Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); - Assert.assertEquals(OlapTableState.NORMAL, table.getState()); + Assert.assertEquals(1, olapTable.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", olapTable.getIndexes().get(0).getIndexName()); + Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState()); long createJobId = indexChangeJobMap.values().stream().findAny().get().jobId; @@ -427,10 +432,10 @@ public void testNormalCreateNgramBfIndex() throws Exception { Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); CatalogTestUtil.createDupTable(db); - OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); DataSortInfo dataSortInfo = new DataSortInfo(); dataSortInfo.setSortType(TSortType.LEXICAL); - table.setDataSortInfo(dataSortInfo); + olapTable.setDataSortInfo(dataSortInfo); String indexName = "ngram_bf_index"; // Add required properties for NGRAM_BF index @@ -439,21 +444,21 @@ public void testNormalCreateNgramBfIndex() throws Exception { properties.put("bf_size", "256"); IndexDef indexDef = new IndexDef(indexName, false, - Lists.newArrayList(table.getBaseSchema().get(3).getName()), + Lists.newArrayList(olapTable.getBaseSchema().get(3).getName()), org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, properties, "ngram bf index"); TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), - table.getName()); + olapTable.getName()); createIndexClause = new CreateIndexClause(tableName, indexDef, false); createIndexClause.analyze(analyzer); ArrayList alterClauses = new ArrayList<>(); alterClauses.add(createIndexClause); // Set session variable to false (default) ctx.getSessionVariable().setEnableAddIndexForNewData(false); - schemaChangeHandler.process(alterClauses, db, table); + schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); Assert.assertEquals(1, indexChangeJobMap.size()); - Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, olapTable.getState()); long createJobId = indexChangeJobMap.values().stream().findAny().get().jobId; @@ -473,9 +478,9 @@ public void testNormalCreateNgramBfIndex() throws Exception { schemaChangeHandler.runAfterCatalogReady(); Assert.assertEquals(AlterJobV2.JobState.FINISHED, createJobV2.getJobState()); - Assert.assertEquals(OlapTableState.NORMAL, table.getState()); - Assert.assertEquals(1, table.getIndexes().size()); - Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); + Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState()); + Assert.assertEquals(1, olapTable.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", olapTable.getIndexes().get(0).getIndexName()); } @Test @@ -492,32 +497,32 @@ public void testCreateInvertedIndex() throws Exception { Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); CatalogTestUtil.createDupTable(db); - OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); DataSortInfo dataSortInfo = new DataSortInfo(); dataSortInfo.setSortType(TSortType.LEXICAL); - table.setDataSortInfo(dataSortInfo); + olapTable.setDataSortInfo(dataSortInfo); String indexName = "raw_inverted_index"; // Explicitly set parser="none" for raw inverted index Map properties = Maps.newHashMap(); properties.put("parser", "none"); IndexDef indexDef = new IndexDef(indexName, false, - Lists.newArrayList(table.getBaseSchema().get(3).getName()), + Lists.newArrayList(olapTable.getBaseSchema().get(3).getName()), IndexType.INVERTED, properties, "raw inverted index"); TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), - table.getName()); + olapTable.getName()); createIndexClause = new CreateIndexClause(tableName, indexDef, false); createIndexClause.analyze(analyzer); ArrayList alterClauses = new ArrayList<>(); alterClauses.add(createIndexClause); ctx.getSessionVariable().setEnableAddIndexForNewData(false); - schemaChangeHandler.process(alterClauses, db, table); + schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); Assert.assertEquals(1, indexChangeJobMap.size()); long createJobId = indexChangeJobMap.values().stream().findAny().get().jobId; - Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, olapTable.getState()); // Finish the create index job first SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2) indexChangeJobMap.get(createJobId); @@ -536,9 +541,9 @@ public void testCreateInvertedIndex() throws Exception { schemaChangeHandler.runAfterCatalogReady(); Assert.assertEquals(AlterJobV2.JobState.FINISHED, createJobV2.getJobState()); - Assert.assertEquals(OlapTableState.NORMAL, table.getState()); - Assert.assertEquals(1, table.getIndexes().size()); - Assert.assertEquals("raw_inverted_index", table.getIndexes().get(0).getIndexName()); + Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState()); + Assert.assertEquals(1, olapTable.getIndexes().size()); + Assert.assertEquals("raw_inverted_index", olapTable.getIndexes().get(0).getIndexName()); } @Test @@ -555,35 +560,35 @@ public void testCreateInvertedIndexWithLightweightMode() throws Exception { Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); CatalogTestUtil.createDupTable(db); - OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); DataSortInfo dataSortInfo = new DataSortInfo(); dataSortInfo.setSortType(TSortType.LEXICAL); - table.setDataSortInfo(dataSortInfo); + olapTable.setDataSortInfo(dataSortInfo); String indexName = "lightweight_raw_inverted_index"; // Explicitly set parser="none" for raw inverted index Map properties = Maps.newHashMap(); properties.put("parser", "none"); IndexDef indexDef = new IndexDef(indexName, false, - Lists.newArrayList(table.getBaseSchema().get(3).getName()), + Lists.newArrayList(olapTable.getBaseSchema().get(3).getName()), IndexType.INVERTED, properties, "lightweight raw inverted index"); TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), - table.getName()); + olapTable.getName()); createIndexClause = new CreateIndexClause(tableName, indexDef, false); createIndexClause.analyze(analyzer); ArrayList alterClauses = new ArrayList<>(); alterClauses.add(createIndexClause); // Test with enable_add_index_for_new_data = true, should use lightweight mode ctx.getSessionVariable().setEnableAddIndexForNewData(true); - schemaChangeHandler.process(alterClauses, db, table); + schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); // Lightweight mode should not create any schema change jobs Assert.assertEquals(1, indexChangeJobMap.size()); - Assert.assertEquals(1, table.getIndexes().size()); - Assert.assertEquals("lightweight_raw_inverted_index", table.getIndexes().get(0).getIndexName()); - Assert.assertEquals(OlapTableState.NORMAL, table.getState()); + Assert.assertEquals(1, olapTable.getIndexes().size()); + Assert.assertEquals("lightweight_raw_inverted_index", olapTable.getIndexes().get(0).getIndexName()); + Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState()); // Verify the index properties - Assert.assertEquals("none", table.getIndexes().get(0).getProperties().get("parser")); + Assert.assertEquals("none", olapTable.getIndexes().get(0).getProperties().get("parser")); } @Test @@ -600,13 +605,13 @@ public void testCreateTokenizedInvertedIndex() throws Exception { Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); CatalogTestUtil.createDupTable(db); - OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); DataSortInfo dataSortInfo = new DataSortInfo(); dataSortInfo.setSortType(TSortType.LEXICAL); - table.setDataSortInfo(dataSortInfo); + olapTable.setDataSortInfo(dataSortInfo); // Set inverted index file storage format to V2 for cloud mode - table.setInvertedIndexFileStorageFormat(TInvertedIndexFileStorageFormat.V2); + olapTable.setInvertedIndexFileStorageFormat(TInvertedIndexFileStorageFormat.V2); String indexName = "tokenized_inverted_index"; Map properties = Maps.newHashMap(); @@ -616,19 +621,19 @@ public void testCreateTokenizedInvertedIndex() throws Exception { // Use VARCHAR column v1 (index 2) for string type support IndexDef indexDef = new IndexDef(indexName, false, - Lists.newArrayList(table.getBaseSchema().get(2).getName()), + Lists.newArrayList(olapTable.getBaseSchema().get(2).getName()), IndexType.INVERTED, properties, "tokenized inverted index with english parser"); TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), - table.getName()); + olapTable.getName()); createIndexClause = new CreateIndexClause(tableName, indexDef, false); createIndexClause.analyze(analyzer); ArrayList alterClauses = new ArrayList<>(); alterClauses.add(createIndexClause); - schemaChangeHandler.process(alterClauses, db, table); + schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); Assert.assertEquals(1, indexChangeJobMap.size()); - Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, olapTable.getState()); SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() .findFirst() @@ -653,12 +658,12 @@ public void testCreateTokenizedInvertedIndex() throws Exception { schemaChangeHandler.runAfterCatalogReady(); Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); - Assert.assertEquals(1, table.getIndexes().size()); - Assert.assertEquals("tokenized_inverted_index", table.getIndexes().get(0).getIndexName()); + Assert.assertEquals(1, olapTable.getIndexes().size()); + Assert.assertEquals("tokenized_inverted_index", olapTable.getIndexes().get(0).getIndexName()); // Verify that the index has the correct properties - Assert.assertEquals("english", table.getIndexes().get(0).getProperties().get("parser")); - Assert.assertEquals("true", table.getIndexes().get(0).getProperties().get("support_phrase")); - Assert.assertEquals("true", table.getIndexes().get(0).getProperties().get("lower_case")); + Assert.assertEquals("english", olapTable.getIndexes().get(0).getProperties().get("parser")); + Assert.assertEquals("true", olapTable.getIndexes().get(0).getProperties().get("support_phrase")); + Assert.assertEquals("true", olapTable.getIndexes().get(0).getProperties().get("lower_case")); } } From f3bd602008ecec81f35129f207fcf928051f2cc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Tue, 8 Jul 2025 16:03:51 +0800 Subject: [PATCH 5/7] fix ut --- .../test/java/org/apache/doris/alter/IndexChangeJobTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java index d4bcf9ab02066e..29aed7e050d72f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java @@ -722,7 +722,7 @@ public void testCancelNgramBfBuildIndex() throws UserException { Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); - cancelAlterTableStmt = new CancelAlterTableStmt(ShowAlterStmt.AlterType.INDEX, tableName); + cancelAlterTableStmt = new CancelAlterTableStmt(ShowAlterStmt.AlterType.COLUMN, tableName); cancelAlterTableStmt.analyze(analyzer); schemaChangeHandler.cancel(cancelAlterTableStmt); From 20a0a9f38d1085d5344539e3f312330b48518c05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Tue, 8 Jul 2025 20:20:43 +0800 Subject: [PATCH 6/7] fix ut --- .../apache/doris/alter/CloudIndexTest.java | 47 +++++++------------ .../org/apache/doris/catalog/FakeEditLog.java | 10 ---- 2 files changed, 18 insertions(+), 39 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java index e96d7b8117b4f3..10adecb167675f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java @@ -332,35 +332,24 @@ public UserIdentity getCurrentUserIdentity() { analyzer = new Analyzer(masterEnv, ctx); Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); - // Mock addCloudCluster to avoid EditLog issues - new MockUp() { - @Mock - public void addCloudCluster(String clusterName, String clusterId) { - // Create backend manually for test - Backend backend = new Backend(10001L, "host1", 123); - backend.setAlive(true); - backend.setBePort(456); - backend.setHttpPort(789); - backend.setBrpcPort(321); - backend.setTagMap(Maps.newHashMap()); - backend.getTagMap().put("cloud_cluster_id", "test_id"); - backend.getTagMap().put("cloud_unique_id", "test_cloud"); - backend.getTagMap().put("cloud_cluster_name", "test_group"); - backend.getTagMap().put("cloud_cluster_status", "NORMAL"); - backend.getTagMap().put("location", "default"); - backend.getTagMap().put("cloud_cluster_private_endpoint", ""); - backend.getTagMap().put("cloud_cluster_public_endpoint", ""); - CloudSystemInfoService systemInfo = (CloudSystemInfoService) Env.getCurrentSystemInfo(); - systemInfo.addBackend(backend); - } - }; - ((CloudSystemInfoService) Env.getCurrentSystemInfo()).addCloudCluster("test_group", ""); - List backends = - ((CloudSystemInfoService) Env.getCurrentSystemInfo()).getBackendsByClusterName("test_group"); - Assert.assertEquals(1, backends.size()); - Assert.assertEquals("host1", backends.get(0).getHost()); - backends.get(0).setAlive(true); - + CloudSystemInfoService systemInfo = (CloudSystemInfoService) Env.getCurrentSystemInfo(); + ComputeGroup pcg1 = new ComputeGroup("test_id", "test_group", ComputeGroup.ComputeTypeEnum.COMPUTE); + systemInfo.addComputeGroup("test_id", pcg1); + Backend backend = new Backend(10001L, "host1", 123); + backend.setAlive(true); + backend.setBePort(456); + backend.setHttpPort(789); + backend.setBrpcPort(321); + Map newTagMap = org.apache.doris.resource.Tag.DEFAULT_BACKEND_TAG.toMap(); + newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_STATUS, "NORMAL"); + newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_NAME, "test_cluster"); + newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_ID, "test_id"); + newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_PUBLIC_ENDPOINT, ""); + newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_PRIVATE_ENDPOINT, ""); + newTagMap.put(org.apache.doris.resource.Tag.CLOUD_UNIQUE_ID, "test_cloud"); + backend.setTagMap(newTagMap); + List backends = Lists.newArrayList(backend); + systemInfo.updateCloudClusterMapNoLock(backends, new ArrayList<>()); db = new Database(CatalogTestUtil.testDbId1, CatalogTestUtil.testDb1); masterEnv.unprotectCreateDb(db); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEditLog.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEditLog.java index e2b113958366c8..08800510a1a6ef 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEditLog.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEditLog.java @@ -100,16 +100,6 @@ public void logModifyDistributionType(TableInfo tableInfo) { } - @Mock - public void logAddBackend(Backend be) { - // do nothing for test - } - - @Mock - public int getNumEditStreams() { - return 1; // fake that we have streams - } - public TransactionState getTransaction(long transactionId) { return allTransactionState.get(transactionId); } From f8795dc633598c2ad3443fb4860b9d704641950e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Wed, 9 Jul 2025 09:47:36 +0800 Subject: [PATCH 7/7] fix ut --- .../org/apache/doris/alter/SchemaChangeHandler.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index c147f3542401ea..37e415afe027b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -2220,12 +2220,12 @@ public int getAsInt() { long jobId = Env.getCurrentEnv().getNextId(); //for schema change add/drop value column optimize, direct modify table meta. modifyTableLightSchemaChange(rawSql, db, olapTable, indexSchemaMap, newIndexes, - null, isDropIndex, jobId, false, propertyMap); + null, isDropIndex, jobId, false); } else if (Config.enable_light_index_change && lightIndexChange) { long jobId = Env.getCurrentEnv().getNextId(); //for schema change add/drop inverted index and ngram_bf optimize, direct modify table meta firstly. modifyTableLightSchemaChange(rawSql, db, olapTable, indexSchemaMap, newIndexes, - alterIndexes, isDropIndex, jobId, false, propertyMap); + alterIndexes, isDropIndex, jobId, false); } else if (buildIndexChange) { if (alterIndexes.isEmpty()) { throw new DdlException("Altered index is empty. please check your alter stmt."); @@ -2943,7 +2943,7 @@ public void replayAlterJobV2(AlterJobV2 alterJob) throws AnalysisException { public void modifyTableLightSchemaChange(String rawSql, Database db, OlapTable olapTable, Map> indexSchemaMap, List indexes, List alterIndexes, boolean isDropIndex, - long jobId, boolean isReplay, Map propertyMap) + long jobId, boolean isReplay) throws DdlException, AnalysisException { if (LOG.isDebugEnabled()) { @@ -3083,8 +3083,7 @@ public void replayModifyTableLightSchemaChange(TableAddOrDropColumnsInfo info) OlapTable olapTable = (OlapTable) db.getTableOrMetaException(tableId, TableType.OLAP); olapTable.writeLock(); try { - modifyTableLightSchemaChange("", db, olapTable, indexSchemaMap, indexes, null, false, jobId, - true, new HashMap<>()); + modifyTableLightSchemaChange("", db, olapTable, indexSchemaMap, indexes, null, false, jobId, true); } catch (DdlException e) { // should not happen LOG.warn("failed to replay modify table add or drop or modify columns", e); @@ -3235,7 +3234,7 @@ public void replayModifyTableAddOrDropInvertedIndices(TableAddOrDropInvertedIndi olapTable.writeLock(); try { modifyTableLightSchemaChange("", db, olapTable, indexSchemaMap, newIndexes, - alterIndexes, isDropIndex, jobId, true, new HashMap<>()); + alterIndexes, isDropIndex, jobId, true); } catch (UserException e) { // should not happen LOG.warn("failed to replay modify table add or drop indexes", e);