diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 95d77384a81231..5340fd10a8412e 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -900,7 +900,6 @@ nonterminal Map key_value_map, opt_key_value_map, opt_key_value_ opt_ext_properties, opt_enable_feature_properties, properties; nonterminal ColumnDef column_definition; nonterminal IndexDef index_definition; -nonterminal IndexDef build_index_definition; nonterminal ArrayList column_definition_list; nonterminal ArrayList index_definition_list; nonterminal AggregateType opt_agg_type; @@ -2194,7 +2193,7 @@ create_stmt ::= :} | KW_BUILD KW_INDEX ident:indexName KW_ON table_name:tableName opt_partition_names:partitionNames {: - RESULT = new AlterTableStmt(tableName, Lists.newArrayList(new BuildIndexClause(tableName, new IndexDef(indexName, partitionNames, true), false))); + RESULT = new AlterTableStmt(tableName, Lists.newArrayList(new BuildIndexClause(tableName, indexName, partitionNames, false))); :} /* stage */ | KW_CREATE KW_STAGE opt_if_not_exists:ifNotExists ident:stageName KW_PROPERTIES opt_key_value_map:properties @@ -4121,13 +4120,6 @@ index_definition ::= :} ; -build_index_definition ::= - KW_INDEX ident:indexName opt_partition_names:partitionNames - {: - RESULT = new IndexDef(indexName, partitionNames, true); - :} - ; - opt_nullable_type ::= {: RESULT = ColumnNullableType.DEFAULT; diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 7751cbac9f1002..37e415afe027b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1330,7 +1330,7 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map oriBfColumns = olapTable.getCopiedBfColumns(); double oriBfFpp = olapTable.getBfFpp(); @@ -1991,7 +1991,7 @@ public int getAsInt() { List newIndexes = olapTable.getCopiedIndexes(); List alterIndexes = new ArrayList<>(); - Map> invertedIndexOnPartitions = new HashMap<>(); + Map> indexOnPartitions = new HashMap<>(); boolean isDropIndex = false; Map propertyMap = new HashMap<>(); for (AlterClause alterClause : alterClauses) { @@ -2145,10 +2145,23 @@ public int getAsInt() { } lightSchemaChange = false; - if (index.isLightIndexChangeSupported() && !Config.isCloudMode()) { + // Check if the index supports light index change and session variable is enabled + boolean enableAddIndexForNewData = true; + try { + ConnectContext context = ConnectContext.get(); + if (context != null && context.getSessionVariable() != null) { + enableAddIndexForNewData = context.getSessionVariable().isEnableAddIndexForNewData(); + } + } catch (Exception e) { + LOG.warn("Failed to get session variable enable_add_index_for_new_data, " + + "using default value: false", e); + } + + // ngram_bf index can do light_schema_change in both local and cloud mode + // inverted index can only do light_schema_change in local mode + if (index.isLightAddIndexSupported(enableAddIndexForNewData)) { alterIndexes.add(index); isDropIndex = false; - // now only support light index change for inverted index lightIndexChange = true; } } else if (alterClause instanceof BuildIndexClause) { @@ -2159,54 +2172,14 @@ public int getAsInt() { throw new DdlException("BUILD INDEX operation failed: No need to do it in cloud mode."); } - if (!olapTable.isPartitionedTable()) { - List specifiedPartitions = indexDef.getPartitionNames(); - if (!specifiedPartitions.isEmpty()) { - throw new DdlException("table " + olapTable.getName() - + " is not partitioned, cannot build index with partitions."); - } - } - List existedIndexes = olapTable.getIndexes(); - boolean found = false; - for (Index existedIdx : existedIndexes) { - if (existedIdx.getIndexName().equalsIgnoreCase(indexDef.getIndexName())) { - found = true; - if (!existedIdx.isLightIndexChangeSupported()) { - throw new DdlException("BUILD INDEX operation failed: The index " - + existedIdx.getIndexName() + " of type " + existedIdx.getIndexType() - + " does not support lightweight index changes."); - } - for (Column column : olapTable.getBaseSchema()) { - if (!column.getType().isVariantType()) { - continue; - } - // variant type column can not support for building index - for (String indexColumn : existedIdx.getColumns()) { - if (column.getName().equalsIgnoreCase(indexColumn)) { - throw new DdlException("BUILD INDEX operation failed: The " - + indexDef.getIndexName() + " index can not be built on the " - + indexColumn + " column, because it is a variant type column."); - } - } - } - index = existedIdx.clone(); - if (indexDef.getPartitionNames().isEmpty()) { - invertedIndexOnPartitions.put(index.getIndexId(), olapTable.getPartitionNames()); - } else { - invertedIndexOnPartitions.put( - index.getIndexId(), new HashSet<>(indexDef.getPartitionNames())); - } - break; - } - } - if (!found) { - throw new DdlException("index " + indexDef.getIndexName() - + " not exist, cannot build it with defferred."); + if (indexDef.getPartitionNames().isEmpty()) { + indexOnPartitions.put(index.getIndexId(), olapTable.getPartitionNames()); + } else { + indexOnPartitions.put( + index.getIndexId(), new HashSet<>(indexDef.getPartitionNames())); } - if (indexDef.isInvertedIndex()) { - alterIndexes.add(index); - } + alterIndexes.add(index); buildIndexChange = true; lightSchemaChange = false; } else if (alterClause instanceof DropIndexClause) { @@ -2224,7 +2197,9 @@ public int getAsInt() { break; } } - if (found.isLightIndexChangeSupported() && !Config.isCloudMode()) { + // only inverted index with local mode can do light drop index change + if (found != null && found.getIndexType() == IndexDef.IndexType.INVERTED + && Config.isNotCloudMode()) { alterIndexes.add(found); isDropIndex = true; lightIndexChange = true; @@ -2248,13 +2223,16 @@ public int getAsInt() { null, isDropIndex, jobId, false); } else if (Config.enable_light_index_change && lightIndexChange) { long jobId = Env.getCurrentEnv().getNextId(); - //for schema change add/drop inverted index optimize, direct modify table meta firstly. + //for schema change add/drop inverted index and ngram_bf optimize, direct modify table meta firstly. modifyTableLightSchemaChange(rawSql, db, olapTable, indexSchemaMap, newIndexes, alterIndexes, isDropIndex, jobId, false); } else if (buildIndexChange) { + if (alterIndexes.isEmpty()) { + throw new DdlException("Altered index is empty. please check your alter stmt."); + } if (Config.enable_light_index_change) { buildOrDeleteTableInvertedIndices(db, olapTable, indexSchemaMap, - alterIndexes, invertedIndexOnPartitions, false); + alterIndexes, indexOnPartitions, false); } } else { createJob(rawSql, db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes); @@ -2747,6 +2725,8 @@ private void cancelIndexJob(CancelAlterTableStmt cancelAlterTableStmt) throws Dd olapTable.writeUnlock(); } + // if this table has ngram_bf index, we must run cancel for schema change job + boolean hasNGramBFIndex = ((OlapTable) olapTable).hasIndexOfType(IndexDef.IndexType.NGRAM_BF); // alter job v2's cancel must be called outside the table lock if (jobList.size() > 0) { for (IndexChangeJob job : jobList) { @@ -2761,6 +2741,8 @@ private void cancelIndexJob(CancelAlterTableStmt cancelAlterTableStmt) throws Dd LOG.info("cancel build index job {} on table {} success", jobId, tableName); } } + } else if (hasNGramBFIndex) { + cancelColumnJob(cancelAlterTableStmt); } else { throw new DdlException("No job to cancel for Table[" + tableName + "]"); } @@ -2800,7 +2782,7 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab Column column = olapTable.getColumn(col); if (column != null) { indexDef.checkColumn(column, olapTable.getKeysType(), - olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite(), + olapTable.getEnableUniqueKeyMergeOnWrite(), olapTable.getInvertedIndexFileStorageFormat()); if (!InvertedIndexUtil.getInvertedIndexFieldPattern(indexDef.getProperties()).isEmpty()) { throw new DdlException("Can not create index with field pattern"); @@ -3040,7 +3022,7 @@ public void modifyTableLightSchemaChange(String rawSql, Database db, OlapTable o } try { buildOrDeleteTableInvertedIndices(db, olapTable, indexSchemaMap, - alterIndexes, invertedIndexOnPartitions, true); + alterIndexes, invertedIndexOnPartitions, true); } catch (Exception e) { throw new DdlException(e.getMessage()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 2e4ca2c6a70ee4..ad4a6ef60d1229 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -151,7 +151,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 { private boolean hasEnableUniqueKeySkipBitmapChanged = false; // save all schema change tasks - private AgentBatchTask schemaChangeBatchTask = new AgentBatchTask(); + AgentBatchTask schemaChangeBatchTask = new AgentBatchTask(); protected SchemaChangeJobV2() { super(JobType.SCHEMA_CHANGE); @@ -678,14 +678,13 @@ protected void runRunningJob() throws AlterCancelException { healthyReplicaNum++; } } - if (!FeConstants.runningUnitTest) { - if (healthyReplicaNum < expectReplicationNum / 2 + 1) { - LOG.warn("shadow tablet {} has few healthy replicas: {}, schema change job: {}" - + " healthyReplicaNum {} expectReplicationNum {}", - shadowTablet.getId(), replicas, jobId, healthyReplicaNum, expectReplicationNum); - throw new AlterCancelException( + + if ((healthyReplicaNum < expectReplicationNum / 2 + 1) && !FeConstants.runningUnitTest) { + LOG.warn("shadow tablet {} has few healthy replicas: {}, schema change job: {}" + + " healthyReplicaNum {} expectReplicationNum {}", + shadowTablet.getId(), replicas, jobId, healthyReplicaNum, expectReplicationNum); + throw new AlterCancelException( "shadow tablet " + shadowTablet.getId() + " has few healthy replicas"); - } } } // end for tablets } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java index d04d24e86e403a..7cd1d915150911 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java @@ -18,12 +18,17 @@ package org.apache.doris.analysis; import org.apache.doris.alter.AlterOpType; +import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Index; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import com.google.common.collect.Maps; +import java.util.List; import java.util.Map; public class BuildIndexClause extends AlterTableClause { @@ -36,11 +41,14 @@ public class BuildIndexClause extends AlterTableClause { private boolean alter; // index internal class private Index index; + private String indexName; + private PartitionNames partitionNames; - public BuildIndexClause(TableName tableName, IndexDef indexDef, boolean alter) { + public BuildIndexClause(TableName tableName, String indexName, PartitionNames partitionNames, boolean alter) { super(AlterOpType.SCHEMA_CHANGE); this.tableName = tableName; - this.indexDef = indexDef; + this.indexName = indexName; + this.partitionNames = partitionNames; this.alter = alter; } @@ -67,17 +75,52 @@ public TableName getTableName() { @Override public void analyze(Analyzer analyzer) throws AnalysisException { - if (indexDef == null) { - throw new AnalysisException("index definition expected."); + tableName.analyze(analyzer); + DatabaseIf db = Env.getCurrentEnv().getCatalogMgr().getInternalCatalog() + .getDb(tableName.getDb()).orElse(null); + if (db == null) { + throw new AnalysisException("Database[" + tableName.getDb() + "] is not exist"); } - if (indexDef.getIndexType() == IndexDef.IndexType.NGRAM_BF - || indexDef.getIndexType() == IndexDef.IndexType.BLOOMFILTER) { + + TableIf table = db.getTable(tableName.getTbl()).orElse(null); + if (table == null) { + throw new AnalysisException("Table[" + tableName.getTbl() + "] is not exist"); + } + if (!(table instanceof OlapTable)) { + throw new AnalysisException("Only olap table support build index"); + } + + Index existedIdx = null; + for (Index index : table.getTableIndexes().getIndexes()) { + if (index.getIndexName().equalsIgnoreCase(indexName)) { + existedIdx = index; + if (!existedIdx.isLightIndexChangeSupported()) { + throw new AnalysisException("BUILD INDEX operation failed: The index " + + existedIdx.getIndexName() + " of type " + existedIdx.getIndexType() + + " does not support lightweight index changes."); + } + break; + } + } + if (existedIdx == null) { + throw new AnalysisException("Index[" + indexName + "] is not exist in table[" + tableName.getTbl() + "]"); + } + + IndexDef.IndexType indexType = existedIdx.getIndexType(); + if (indexType == IndexDef.IndexType.NGRAM_BF + || indexType == IndexDef.IndexType.BLOOMFILTER) { throw new AnalysisException("ngram bloomfilter or bloomfilter index is not needed to build."); } + indexDef = new IndexDef(indexName, partitionNames, indexType, true); + if (!table.isPartitionedTable()) { + List specifiedPartitions = indexDef.getPartitionNames(); + if (!specifiedPartitions.isEmpty()) { + throw new AnalysisException("table " + table.getName() + + " is not partitioned, cannot build index with partitions."); + } + } indexDef.analyze(); - this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(), - indexDef.getColumns(), indexDef.getIndexType(), - indexDef.getProperties(), indexDef.getComment()); + this.index = existedIdx.clone(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java index 8d1f6a5bfd6a74..a1df981898e665 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java @@ -83,9 +83,9 @@ public IndexDef(String indexName, boolean ifNotExists, List columns, Ind } } - public IndexDef(String indexName, PartitionNames partitionNames, boolean isBuildDeferred) { + public IndexDef(String indexName, PartitionNames partitionNames, IndexType indexType, boolean isBuildDeferred) { this.indexName = indexName; - this.indexType = IndexType.INVERTED; + this.indexType = indexType; this.partitionNames = partitionNames; this.isBuildDeferred = isBuildDeferred; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index 312e3194ed3eab..54407443a24d01 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.IndexDef; import org.apache.doris.analysis.InvertedIndexUtil; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.PrintableMap; @@ -152,6 +153,10 @@ public String getInvertedIndexParser() { return InvertedIndexUtil.getInvertedIndexParser(properties); } + public boolean isInvertedIndexParserNone() { + return InvertedIndexUtil.INVERTED_INDEX_PARSER_NONE.equals(getInvertedIndexParser()); + } + public String getInvertedIndexParserMode() { return InvertedIndexUtil.getInvertedIndexParserMode(properties); } @@ -168,10 +173,28 @@ public String getInvertedIndexParserStopwords() { return InvertedIndexUtil.getInvertedIndexParserStopwords(properties); } + // Whether the index can be changed in light mode public boolean isLightIndexChangeSupported() { return indexType == IndexDef.IndexType.INVERTED; } + // Whether the index can be added in light mode + // cloud mode supports light add for ngram_bf index and non-tokenized inverted index (parser="none") + // local mode supports light add for both inverted index and ngram_bf index + // the rest of the index types do not support light add + public boolean isLightAddIndexSupported(boolean enableAddIndexForNewData) { + if (Config.isCloudMode()) { + if (indexType == IndexDef.IndexType.INVERTED) { + return isInvertedIndexParserNone() && enableAddIndexForNewData; + } else if (indexType == IndexDef.IndexType.NGRAM_BF) { + return enableAddIndexForNewData; + } + return false; + } + return (indexType == IndexDef.IndexType.NGRAM_BF && enableAddIndexForNewData) + || (indexType == IndexDef.IndexType.INVERTED); + } + public String getInvertedIndexCustomAnalyzer() { return InvertedIndexUtil.getInvertedIndexCustomAnalyzer(properties); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index d4dca760bfef3d..e39cc1ec0d06a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -24,6 +24,7 @@ import org.apache.doris.analysis.CreateMaterializedViewStmt; import org.apache.doris.analysis.DataSortInfo; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.IndexDef; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotRef; import org.apache.doris.backup.Status; @@ -356,6 +357,19 @@ public List getIndexIds() { return indexes.getIndexIds(); } + /** + * Checks if the table contains at least one index of the specified type. + * @param indexType The index type to check for + * @return true if the table has at least one index of the specified type, false otherwise + */ + public boolean hasIndexOfType(IndexDef.IndexType indexType) { + if (indexes == null) { + return false; + } + return indexes.getIndexes().stream() + .anyMatch(index -> index.getIndexType() == indexType); + } + @Override public TableIndexes getTableIndexes() { return indexes; diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index bcb17d3c03992f..f4f3893c87d4a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -339,6 +339,14 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2); } else if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V3) { schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V3); + } else if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.DEFAULT) { + if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) { + schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V1); + } else if (Config.inverted_index_storage_format.equalsIgnoreCase("V3")) { + schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V3); + } else { + schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2); + } } else { throw new DdlException("invalid inverted index storage format"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index c268fddee53906..7891b735d03a1d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -732,6 +732,7 @@ public class SessionVariable implements Serializable, Writable { public static final String SQL_CONVERTOR_CONFIG = "sql_convertor_config"; public static final String PREFER_UDF_OVER_BUILTIN = "prefer_udf_over_builtin"; + public static final String ENABLE_ADD_INDEX_FOR_NEW_DATA = "enable_add_index_for_new_data"; /** * If set false, user couldn't submit analyze SQL and FE won't allocate any related resources. @@ -2558,6 +2559,14 @@ public boolean isEnableSortSpill() { return enableSortSpill; } + @VariableMgr.VarAttr(name = ENABLE_ADD_INDEX_FOR_NEW_DATA, fuzzy = true, description = { + "是否启用仅对新数据生效的索引添加模式，开启时新建索引只对后续写入的数据生效，关闭时对全部数据重建索引", + "Whether to enable add index mode that only affects new data, " + + "when enabled new indexes only affect subsequently written data, " + + "when disabled rebuild indexes for all data" + }) + public boolean enableAddIndexForNewData = false; + // If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables, // not the default value set in the code. @SuppressWarnings("checkstyle:Indentation") @@ -4873,5 +4882,13 @@ public boolean getEnableExternalTableBatchMode() { public int getGlobalVariantMaxSubcolumnsCount() { return globalVariantMaxSubcolumnsCount; } + + public boolean isEnableAddIndexForNewData() { + return enableAddIndexForNewData; + } + + public void setEnableAddIndexForNewData(boolean enableAddIndexForNewData) { + this.enableAddIndexForNewData = enableAddIndexForNewData; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java new file mode 100644 index 00000000000000..10adecb167675f --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java @@ -0,0 +1,658 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.alter; + +import org.apache.doris.analysis.AlterClause; +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.CreateIndexClause; +import org.apache.doris.analysis.DataSortInfo; +import org.apache.doris.analysis.IndexDef; +import org.apache.doris.analysis.IndexDef.IndexType; +import org.apache.doris.analysis.ResourceTypeEnum; +import org.apache.doris.analysis.TableName; +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.CatalogTestUtil; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.EnvFactory; +import org.apache.doris.catalog.FakeEditLog; +import org.apache.doris.catalog.FakeEnv; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.OlapTable.OlapTableState; +import org.apache.doris.cloud.catalog.CloudEnv; +import org.apache.doris.cloud.catalog.CloudEnvFactory; +import org.apache.doris.cloud.catalog.ComputeGroup; +import org.apache.doris.cloud.datasource.CloudInternalCatalog; +import org.apache.doris.cloud.proto.Cloud; +import org.apache.doris.cloud.proto.Cloud.MetaServiceCode; +import org.apache.doris.cloud.rpc.MetaServiceProxy; +import org.apache.doris.cloud.system.CloudSystemInfoService; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.UserException; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.Auth; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.persist.EditLog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.task.AgentTask; +import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; +import org.apache.doris.thrift.TSortType; +import org.apache.doris.thrift.TTaskType; +import org.apache.doris.utframe.MockedMetaServerFactory; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import mockit.Mock; +import mockit.MockUp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class CloudIndexTest { + private static final Logger LOG = LogManager.getLogger(CloudIndexTest.class); + + // Simple ComputeGroupMgr wrapper for CloudSystemInfoService + public static class ComputeGroupMgr { + private final SystemInfoService systemInfoService; + + public ComputeGroupMgr(SystemInfoService systemInfoService) { + this.systemInfoService = systemInfoService; + } + + public ComputeGroup getComputeGroupByName(String name) { + if (systemInfoService instanceof CloudSystemInfoService) { + return ((CloudSystemInfoService) systemInfoService).getComputeGroupByName(name); + } + return null; + } + + public ComputeGroup getAllBackendComputeGroup() { + // Return a default compute group for all backends + return new ComputeGroup("default_compute_group", "default_compute_group", + ComputeGroup.ComputeTypeEnum.SQL); + } + } + + private static String fileName = "./CloudIndexTest"; + + private static FakeEditLog fakeEditLog; + private static FakeEnv fakeEnv; + private static Env masterEnv; + private static EditLog testEditLog; + private ConnectContext ctx; + private static OlapTable olapTable; + + private static Analyzer analyzer; + private static Database db; + private static CreateIndexClause createIndexClause; + private static SchemaChangeHandler schemaChangeHandler; + + @Before + public void setUp() throws InstantiationException, IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException, SecurityException, UserException { + FeConstants.runningUnitTest = true; + // Setup for MetaServiceProxy mock + new MockUp(MetaServiceProxy.class) { + + @Mock + public Cloud.BeginTxnResponse beginTxn(Cloud.BeginTxnRequest request) { + Cloud.BeginTxnResponse.Builder beginTxnResponseBuilder = Cloud.BeginTxnResponse.newBuilder(); + beginTxnResponseBuilder.setTxnId(1000) + .setStatus( + Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK).setMsg("OK")); + return beginTxnResponseBuilder.build(); + } + + @Mock + public Cloud.CommitTxnResponse commitTxn(Cloud.CommitTxnRequest request) { + Cloud.TxnInfoPB.Builder txnInfoBuilder = Cloud.TxnInfoPB.newBuilder(); + txnInfoBuilder.setDbId(CatalogTestUtil.testDbId1); + txnInfoBuilder.addAllTableIds(Lists.newArrayList(olapTable.getId())); + txnInfoBuilder.setLabel("test_label"); + txnInfoBuilder.setListenerId(-1); + Cloud.CommitTxnResponse.Builder commitTxnResponseBuilder = Cloud.CommitTxnResponse.newBuilder(); + commitTxnResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .setTxnInfo(txnInfoBuilder.build()); + return commitTxnResponseBuilder.build(); + } + + @Mock + public Cloud.CheckTxnConflictResponse checkTxnConflict(Cloud.CheckTxnConflictRequest request) { + Cloud.CheckTxnConflictResponse.Builder checkTxnConflictResponseBuilder = + Cloud.CheckTxnConflictResponse.newBuilder(); + checkTxnConflictResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .setFinished(true); + return checkTxnConflictResponseBuilder.build(); + } + + @Mock + public Cloud.GetClusterResponse getCluster(Cloud.GetClusterRequest request) { + Cloud.GetClusterResponse.Builder getClusterResponseBuilder = Cloud.GetClusterResponse.newBuilder(); + Cloud.ClusterPB.Builder clusterBuilder = Cloud.ClusterPB.newBuilder(); + clusterBuilder.setClusterId("test_id").setClusterName("test_group"); + + Cloud.NodeInfoPB.Builder node1 = Cloud.NodeInfoPB.newBuilder(); + node1.setCloudUniqueId("test_cloud") + .setName("host1") + .setIp("host1") + .setHost("host1") + .setHeartbeatPort(123) + .setEditLogPort(125) + .setStatus(Cloud.NodeStatusPB.NODE_STATUS_RUNNING); + clusterBuilder.addNodes(node1.build()); + getClusterResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .addCluster(clusterBuilder.build()); + return getClusterResponseBuilder.build(); + } + + @Mock + public Cloud.CreateTabletsResponse createTablets(Cloud.CreateTabletsRequest request) { + Cloud.CreateTabletsResponse.Builder responseBuilder = Cloud.CreateTabletsResponse.newBuilder(); + responseBuilder.setStatus( + Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK).setMsg("OK")); + return responseBuilder.build(); + } + + @Mock + public Cloud.FinishTabletJobResponse finishTabletJob(Cloud.FinishTabletJobRequest request) { + Cloud.FinishTabletJobResponse.Builder responseBuilder = Cloud.FinishTabletJobResponse.newBuilder(); + responseBuilder.setStatus( + Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK).setMsg("OK")); + return responseBuilder.build(); + } + + @Mock + public Cloud.IndexResponse prepareIndex(Cloud.IndexRequest request) { + Cloud.IndexResponse.Builder builder = Cloud.IndexResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.IndexResponse commitIndex(Cloud.IndexRequest request) { + Cloud.IndexResponse.Builder builder = Cloud.IndexResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.IndexResponse dropIndex(Cloud.IndexRequest request) { + Cloud.IndexResponse.Builder builder = Cloud.IndexResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.CheckKVResponse checkKv(Cloud.CheckKVRequest request) { + Cloud.CheckKVResponse.Builder builder = Cloud.CheckKVResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.GetCurrentMaxTxnResponse getCurrentMaxTxnId(Cloud.GetCurrentMaxTxnRequest request) { + Cloud.GetCurrentMaxTxnResponse.Builder builder = Cloud.GetCurrentMaxTxnResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .setCurrentMaxTxnId(1000); + return builder.build(); + } + }; + + Config.cloud_unique_id = "test_cloud"; + Config.meta_service_endpoint = MockedMetaServerFactory.METASERVER_DEFAULT_IP + ":" + 20121; + + EnvFactory envFactory = EnvFactory.getInstance(); + masterEnv = envFactory.createEnv(false); + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + FakeEnv.setSystemInfo(cloudSystemInfo); + + fakeEditLog = new FakeEditLog(); + testEditLog = null; // Will be set by MockUp + FakeEnv.setEnv(masterEnv); + + ctx = new ConnectContext(); + ctx.setEnv(masterEnv); + ctx.setQualifiedUser("root"); + UserIdentity rootUser = new UserIdentity("root", "%"); + rootUser.setIsAnalyzed(); + ctx.setCurrentUserIdentity(rootUser); + ctx.setThreadLocalInfo(); + ctx.setCloudCluster("test_group"); + Assert.assertTrue(envFactory instanceof CloudEnvFactory); + Assert.assertTrue(masterEnv instanceof CloudEnv); + new MockUp() { + @Mock + public Env getCurrentEnv() { + return masterEnv; + } + + @Mock + public EditLog getEditLog() { + if (testEditLog == null) { + // Create a mock EditLog using a no-op approach + testEditLog = new EditLog("test") { + // Override to avoid initialization issues + }; + } + return testEditLog; + } + + @Mock + public ComputeGroupMgr getComputeGroupMgr() { + return new ComputeGroupMgr(Env.getCurrentSystemInfo()); + } + + @Mock + public SchemaChangeHandler getSchemaChangeHandler() { + // Create a new independent SchemaChangeHandler for each call + return schemaChangeHandler; + } + + @Mock + public AccessControllerManager getAccessManager() { + return new AccessControllerManager(masterEnv.getAuth()) { + @Override + public boolean checkTblPriv(ConnectContext ctx, String ctl, String db, String tbl, PrivPredicate wanted) { + return true; // Allow all access for test + } + + @Override + public boolean checkCloudPriv(UserIdentity user, String cluster, PrivPredicate wanted, ResourceTypeEnum resourceType) { + return true; // Allow all cloud privileges for test + } + }; + } + }; + + new MockUp() { + @Mock + public String getDefaultCloudCluster(String user) { + return "test_group"; // Return default cluster for test + } + }; + + // Mock cloud environment permissions + new MockUp() { + @Mock + public void checkCloudClusterPriv(String cluster) throws Exception { + // Always allow for tests + } + }; + + // Mock ConnectContext to avoid compute group permission check + new MockUp() { + @Mock + public String getCloudCluster() { + return "test_group"; + } + + @Mock + public UserIdentity getCurrentUserIdentity() { + UserIdentity rootUser = new UserIdentity("root", "%"); + rootUser.setIsAnalyzed(); + return rootUser; + } + }; + + analyzer = new Analyzer(masterEnv, ctx); + + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CloudSystemInfoService systemInfo = (CloudSystemInfoService) Env.getCurrentSystemInfo(); + ComputeGroup pcg1 = new ComputeGroup("test_id", "test_group", ComputeGroup.ComputeTypeEnum.COMPUTE); + systemInfo.addComputeGroup("test_id", pcg1); + Backend backend = new Backend(10001L, "host1", 123); + backend.setAlive(true); + backend.setBePort(456); + backend.setHttpPort(789); + backend.setBrpcPort(321); + Map newTagMap = org.apache.doris.resource.Tag.DEFAULT_BACKEND_TAG.toMap(); + newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_STATUS, "NORMAL"); + newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_NAME, "test_cluster"); + newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_ID, "test_id"); + newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_PUBLIC_ENDPOINT, ""); + newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_PRIVATE_ENDPOINT, ""); + newTagMap.put(org.apache.doris.resource.Tag.CLOUD_UNIQUE_ID, "test_cloud"); + backend.setTagMap(newTagMap); + List backends = Lists.newArrayList(backend); + systemInfo.updateCloudClusterMapNoLock(backends, new ArrayList<>()); + db = new Database(CatalogTestUtil.testDbId1, CatalogTestUtil.testDb1); + masterEnv.unprotectCreateDb(db); + + AgentTaskQueue.clearAllTasks(); + schemaChangeHandler = masterEnv.getSchemaChangeHandler(); + } + + @Test + public void testCreateNgramBfIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + olapTable.setDataSortInfo(dataSortInfo); + String indexName = "ngram_bf_index"; + + // Add required properties for NGRAM_BF index + Map properties = Maps.newHashMap(); + properties.put("gram_size", "2"); + properties.put("bf_size", "256"); + + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(olapTable.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + properties, "ngram bf index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + olapTable.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + ctx.getSessionVariable().setEnableAddIndexForNewData(true); + schemaChangeHandler.process(alterClauses, db, olapTable); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(1, olapTable.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", olapTable.getIndexes().get(0).getIndexName()); + Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState()); + + long createJobId = indexChangeJobMap.values().stream().findAny().get().jobId; + + // Finish the create index job first + SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2) indexChangeJobMap.get(createJobId); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, createJobV2.getJobState()); + } + + @Test + public void testNormalCreateNgramBfIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + olapTable.setDataSortInfo(dataSortInfo); + String indexName = "ngram_bf_index"; + + // Add required properties for NGRAM_BF index + Map properties = Maps.newHashMap(); + properties.put("gram_size", "2"); + properties.put("bf_size", "256"); + + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(olapTable.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + properties, "ngram bf index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + olapTable.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + // Set session variable to false (default) + ctx.getSessionVariable().setEnableAddIndexForNewData(false); + schemaChangeHandler.process(alterClauses, db, olapTable); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, olapTable.getState()); + + long createJobId = indexChangeJobMap.values().stream().findAny().get().jobId; + + // Finish the create index job first + SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2) indexChangeJobMap.get(createJobId); + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, createJobV2.getJobState()); + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, createJobV2.getJobState()); + Assert.assertEquals(1, createJobV2.schemaChangeBatchTask.getTaskNum()); + + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, createJobV2.getJobState()); + Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState()); + Assert.assertEquals(1, olapTable.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", olapTable.getIndexes().get(0).getIndexName()); + } + + @Test + public void testCreateInvertedIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + olapTable.setDataSortInfo(dataSortInfo); + String indexName = "raw_inverted_index"; + // Explicitly set parser="none" for raw inverted index + Map properties = Maps.newHashMap(); + properties.put("parser", "none"); + + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(olapTable.getBaseSchema().get(3).getName()), + IndexType.INVERTED, + properties, "raw inverted index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + olapTable.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + ctx.getSessionVariable().setEnableAddIndexForNewData(false); + schemaChangeHandler.process(alterClauses, db, olapTable); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + + long createJobId = indexChangeJobMap.values().stream().findAny().get().jobId; + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, olapTable.getState()); + + // Finish the create index job first + SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2) indexChangeJobMap.get(createJobId); + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, createJobV2.getJobState()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, createJobV2.getJobState()); + Assert.assertEquals(1, createJobV2.schemaChangeBatchTask.getTaskNum()); + + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, createJobV2.getJobState()); + Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState()); + Assert.assertEquals(1, olapTable.getIndexes().size()); + Assert.assertEquals("raw_inverted_index", olapTable.getIndexes().get(0).getIndexName()); + } + + @Test + public void testCreateInvertedIndexWithLightweightMode() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + olapTable.setDataSortInfo(dataSortInfo); + String indexName = "lightweight_raw_inverted_index"; + // Explicitly set parser="none" for raw inverted index + Map properties = Maps.newHashMap(); + properties.put("parser", "none"); + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(olapTable.getBaseSchema().get(3).getName()), + IndexType.INVERTED, + properties, "lightweight raw inverted index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + olapTable.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + // Test with enable_add_index_for_new_data = true, should use lightweight mode + ctx.getSessionVariable().setEnableAddIndexForNewData(true); + schemaChangeHandler.process(alterClauses, db, olapTable); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + // Lightweight mode should not create any schema change jobs + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(1, olapTable.getIndexes().size()); + Assert.assertEquals("lightweight_raw_inverted_index", olapTable.getIndexes().get(0).getIndexName()); + Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState()); + // Verify the index properties + Assert.assertEquals("none", olapTable.getIndexes().get(0).getProperties().get("parser")); + } + + @Test + public void testCreateTokenizedInvertedIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + olapTable.setDataSortInfo(dataSortInfo); + + // Set inverted index file storage format to V2 for cloud mode + olapTable.setInvertedIndexFileStorageFormat(TInvertedIndexFileStorageFormat.V2); + + String indexName = "tokenized_inverted_index"; + Map properties = Maps.newHashMap(); + properties.put("parser", "english"); + properties.put("support_phrase", "true"); + properties.put("lower_case", "true"); + + // Use VARCHAR column v1 (index 2) for string type support + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(olapTable.getBaseSchema().get(2).getName()), + IndexType.INVERTED, + properties, "tokenized inverted index with english parser"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + olapTable.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + schemaChangeHandler.process(alterClauses, db, olapTable); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, olapTable.getState()); + + SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() + .findFirst() + .orElse(null); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + // This should be a heavyweight schema change for tokenized index + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, jobV2.getJobState()); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); + + Assert.assertEquals(1, olapTable.getIndexes().size()); + Assert.assertEquals("tokenized_inverted_index", olapTable.getIndexes().get(0).getIndexName()); + + // Verify that the index has the correct properties + Assert.assertEquals("english", olapTable.getIndexes().get(0).getProperties().get("parser")); + Assert.assertEquals("true", olapTable.getIndexes().get(0).getProperties().get("support_phrase")); + Assert.assertEquals("true", olapTable.getIndexes().get(0).getProperties().get("lower_case")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java index 6b0943c306f851..29aed7e050d72f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java @@ -46,6 +46,7 @@ import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.UserException; import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.thrift.TStatusCode; @@ -55,6 +56,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import mockit.Mock; +import mockit.MockUp; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -85,6 +88,9 @@ public class IndexChangeJobTest { private static BuildIndexClause buildIndexClause; private static DropIndexClause dropIndexClause; private static CancelAlterTableStmt cancelAlterTableStmt; + private static TableName tableName; + private static String indexName; + private static ConnectContext ctx; @Rule public ExpectedException expectedEx = ExpectedException.none(); @@ -108,25 +114,40 @@ public void setUp() db = masterEnv.getInternalCatalog().getDbOrDdlException(CatalogTestUtil.testDbId1); olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId1); + new MockUp() { + @Mock + public Env getCurrentEnv() { + return masterEnv; + } + }; + + // Initialize ConnectContext + ctx = new ConnectContext(); + new MockUp() { + @Mock + public ConnectContext get() { + return ctx; + } + }; + // set mow table property Map properties = Maps.newHashMap(); properties.put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE, "false"); TableProperty tableProperty = new TableProperty(properties); olapTable.setTableProperty(tableProperty); - TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), olapTable.getName()); - IndexDef indexDef = new IndexDef("index1", false, + indexName = "index1"; + IndexDef indexDef = new IndexDef(indexName, false, Lists.newArrayList(olapTable.getBaseSchema().get(1).getName()), IndexDef.IndexType.INVERTED, Maps.newHashMap(), "balabala"); + FakeEnv.setEnv(masterEnv); createIndexClause = new CreateIndexClause(tableName, indexDef, false); createIndexClause.analyze(analyzer); - buildIndexClause = new BuildIndexClause(tableName, indexDef, false); - buildIndexClause.analyze(analyzer); - - dropIndexClause = new DropIndexClause("index1", false, tableName, false); + dropIndexClause = new DropIndexClause(indexName, false, tableName, false); dropIndexClause.analyze(analyzer); cancelAlterTableStmt = new CancelAlterTableStmt(ShowAlterStmt.AlterType.INDEX, tableName); @@ -163,6 +184,8 @@ public void testBuildIndexIndexChange() throws UserException { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -203,6 +226,8 @@ public void testBuildIndexIndexChangeNormal() throws UserException { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -292,6 +317,8 @@ public void testCancelBuildIndexIndexChangeNormal() throws UserException { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -339,6 +366,8 @@ public void testBuildIndexIndexChangeWhileTableNotStable() throws Exception { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -467,6 +496,8 @@ public void testBuildIndexFailedWithMinFailedNum() throws Exception { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -519,6 +550,8 @@ public void testBuildIndexFailedWithMaxFailedNum() throws Exception { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -526,13 +559,13 @@ public void testBuildIndexFailedWithMaxFailedNum() throws Exception { Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState()); IndexChangeJob indexChangejob = indexChangeJobMap.values().stream().findAny().get(); - Assert.assertEquals(indexChangejob.invertedIndexBatchTask.getTaskNum(), 0); + Assert.assertEquals(0, indexChangejob.invertedIndexBatchTask.getTaskNum()); Assert.assertEquals(IndexChangeJob.JobState.WAITING_TXN, indexChangejob.getJobState()); // run waiting txn job schemaChangeHandler.runAfterCatalogReady(); Assert.assertEquals(IndexChangeJob.JobState.RUNNING, indexChangejob.getJobState()); - Assert.assertEquals(indexChangejob.invertedIndexBatchTask.getTaskNum(), 3); + Assert.assertEquals(3, indexChangejob.invertedIndexBatchTask.getTaskNum()); // run running job schemaChangeHandler.runAfterCatalogReady(); Assert.assertEquals(IndexChangeJob.JobState.RUNNING, indexChangejob.getJobState()); @@ -565,17 +598,135 @@ public void testNgramBfBuildIndex() throws UserException { fakeEditLog = new FakeEditLog(); FakeEnv.setEnv(masterEnv); - IndexDef indexDef = new IndexDef("ngram_bf_index", false, - Lists.newArrayList(olapTable.getBaseSchema().get(1).getName()), + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + String indexName = "ngram_bf_index"; + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, Maps.newHashMap(), "ngram bf index"); TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), - olapTable.getName()); + table.getName()); createIndexClause = new CreateIndexClause(tableName, indexDef, false); createIndexClause.analyze(analyzer); + SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + + // Test with enable_add_index_for_new_data = true + ConnectContext context = ConnectContext.get(); + context.getSessionVariable().setEnableAddIndexForNewData(true); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); + + SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() + .findFirst() + .orElse(null); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); + + // Clean up for next test + table.setIndexes(Lists.newArrayList()); + indexChangeJobMap.clear(); + AgentTaskQueue.clearAllTasks(); + + // Test with enable_add_index_for_new_data = false + context.getSessionVariable().setEnableAddIndexForNewData(false); + String indexName2 = "ngram_bf_index2"; + IndexDef indexDef2 = new IndexDef(indexName2, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + Maps.newHashMap(), "ngram bf index2"); + + createIndexClause = new CreateIndexClause(tableName, indexDef2, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses2 = new ArrayList<>(); + alterClauses2.add(createIndexClause); + schemaChangeHandler.process(alterClauses2, db, table); + indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + + jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() + .findFirst() + .orElse(null); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, jobV2.getJobState()); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); - buildIndexClause = new BuildIndexClause(tableName, indexDef, false); - org.junit.jupiter.api.Assertions.assertThrows(org.apache.doris.common.AnalysisException.class, - () -> buildIndexClause.analyze(analyzer)); + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("ngram_bf_index2", table.getIndexes().get(0).getIndexName()); + } + + @Test + public void testCancelNgramBfBuildIndex() throws UserException { + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + String indexName = "ngram_bf_index"; + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + Maps.newHashMap(), "ngram bf index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + + //cancel test can only with enable_add_index_for_new_data = false + ctx.getSessionVariable().setEnableAddIndexForNewData(false); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + + SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() + .findFirst() + .orElse(null); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, jobV2.getJobState()); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + cancelAlterTableStmt = new CancelAlterTableStmt(ShowAlterStmt.AlterType.COLUMN, tableName); + cancelAlterTableStmt.analyze(analyzer); + schemaChangeHandler.cancel(cancelAlterTableStmt); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.CANCELLED, jobV2.getJobState()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java index b87ae340f1bef3..77693efa6e4607 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java @@ -850,9 +850,8 @@ public void testDupAddOrDropNgramBfIndex() throws Exception { waitAlterJobDone(alterJobs); String buildNgramBfIndexStmtStr = "BUILD INDEX idx_error_msg on test.sc_dup "; - AlterTableStmt buildNgramBfIndexStmt = (AlterTableStmt) parseAndAnalyzeStmt(buildNgramBfIndexStmtStr); - org.junit.jupiter.api.Assertions.assertThrows(org.apache.doris.common.DdlException.class, - () -> Env.getCurrentEnv().getAlterInstance().processAlterTable(buildNgramBfIndexStmt)); + Assertions.assertThrows(org.apache.doris.common.AnalysisException.class, + () -> parseAndAnalyzeStmt(buildNgramBfIndexStmtStr)); tbl.readLock(); try { diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java index 590b6563e1106c..b16d3d15cf7e52 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java @@ -22,7 +22,8 @@ import org.apache.doris.analysis.SinglePartitionDesc; import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MaterializedIndex.IndexState; -import org.apache.doris.catalog.Replica.ReplicaState; +import org.apache.doris.cloud.catalog.CloudReplica; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.persist.EditLog; @@ -175,13 +176,26 @@ public static Database createSimpleDb(long dbId, long tableId, long partitionId, long version) { Env.getCurrentInvertedIndex().clear(); - // replica - Replica replica1 = new Replica(testReplicaId1, testBackendId1, version, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); - Replica replica2 = new Replica(testReplicaId2, testBackendId2, version, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); - Replica replica3 = new Replica(testReplicaId3, testBackendId3, version, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); + Replica replica1; + Replica replica2; + Replica replica3; + if (Config.isCloudMode()) { + // In cloud mode we must create CloudReplica instances to avoid ClassCastException + replica1 = new CloudReplica(testReplicaId1, testBackendId1, Replica.ReplicaState.NORMAL, version, + /*schemaHash*/ 0, dbId, tableId, partitionId, indexId, /*idx*/ 0); + replica2 = new CloudReplica(testReplicaId2, testBackendId2, Replica.ReplicaState.NORMAL, version, + 0, dbId, tableId, partitionId, indexId, 1); + replica3 = new CloudReplica(testReplicaId3, testBackendId3, Replica.ReplicaState.NORMAL, version, + 0, dbId, tableId, partitionId, indexId, 2); + } else { + replica1 = new Replica(testReplicaId1, testBackendId1, version, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + replica2 = new Replica(testReplicaId2, testBackendId2, version, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + replica3 = new Replica(testReplicaId3, testBackendId3, version, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + } + // tablet Tablet tablet = new Tablet(tabletId); @@ -244,10 +258,14 @@ public static Database createSimpleDb(long dbId, long tableId, long partitionId, } public static void createDupTable(Database db) { - - // replica - Replica replica = new Replica(testReplicaId4, testBackendId1, testStartVersion, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); + Replica replica; + if (Config.isCloudMode()) { + replica = new CloudReplica(testReplicaId4, testBackendId1, Replica.ReplicaState.NORMAL, testStartVersion, + 0, db.getId(), testTableId2, testPartitionId2, testIndexId2, 0); + } else { + replica = new Replica(testReplicaId4, testBackendId1, testStartVersion, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + } // tablet Tablet tablet = new Tablet(testTabletId2); diff --git a/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out b/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out new file mode 100644 index 00000000000000..49570b96d291f2 --- /dev/null +++ b/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out @@ -0,0 +1,103 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_light_mode_init -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select_light_mode_more_data -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select_schema_change_mode_init -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select_schema_change_mode_more_data -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select_lifecycle_after_data -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select_lifecycle_final -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + diff --git a/regression-test/pipeline/cloud_p1/conf/fe_custom.conf b/regression-test/pipeline/cloud_p1/conf/fe_custom.conf index 0e43d458f38b65..5bba5437f73589 100644 --- a/regression-test/pipeline/cloud_p1/conf/fe_custom.conf +++ b/regression-test/pipeline/cloud_p1/conf/fe_custom.conf @@ -34,6 +34,5 @@ cloud_http_port=18030 meta_service_endpoint=127.0.0.1:5000 arrow_flight_sql_port = 8081 cloud_unique_id=cloud_unique_id_sql_server00 -enable_light_index_change=false enable_advance_next_id = true enable_job_schedule_second_for_test = true \ No newline at end of file diff --git a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy new file mode 100644 index 00000000000000..402f72ddc36065 --- /dev/null +++ b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy @@ -0,0 +1,325 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +import groovy.json.JsonSlurper + +suite("test_ngram_bloomfilter_index_change") { + def tableName = 'test_ngram_bloomfilter_index_change' + def timeout = 60000 + def delta_time = 1000 + def alter_res = "null" + def useTime = 0 + + def wait_for_latest_op_on_table_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${table_name}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(3000) // wait change table state to normal + logger.info(table_name + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } + + // Function to insert test data batch + def insertTestData = { -> + // insert 10 records + sql "INSERT INTO ${tableName} VALUES (1001, '2023-10-06 15:00:00', 'Laptop', 'John Smith', 199.99, 'North');" + sql "INSERT INTO ${tableName} VALUES (1002, '2023-10-09 17:05:00', 'Smartphone', 'Emily Johnson', 299.99, 'South');" + sql "INSERT INTO ${tableName} VALUES (1003, '2023-10-12 19:10:00', 'Headphones', 'Michael Brown', 399.99, 'East');" + sql "INSERT INTO ${tableName} VALUES (1004, '2023-10-15 21:15:00', 'Monitor', 'Jessica Davis', 499.99, 'West');" + sql "INSERT INTO ${tableName} VALUES (1005, '2023-10-18 23:20:00', 'Keyboard', 'David Wilson', 89.99, 'North');" + sql "INSERT INTO ${tableName} VALUES (1006, '2023-10-21 07:25:00', 'Mouse', 'Sarah Taylor', 699.99, 'South');" + sql "INSERT INTO ${tableName} VALUES (1007, '2023-10-24 09:30:00', 'Printer', 'Thomas Anderson', 799.99, 'East');" + sql "INSERT INTO ${tableName} VALUES (1008, '2023-10-27 11:35:00', 'Speaker', 'Jennifer Martin', 899.99, 'West');" + sql "INSERT INTO ${tableName} VALUES (1009, '2023-10-02 13:40:00', 'External SSD', 'Robert Clark', 999.99, 'North');" + sql "INSERT INTO ${tableName} VALUES (1010, '2023-10-05 15:45:00', 'Webcam', 'Amanda Lewis', 89.99, 'South');" + sql "sync" + } + + // Test settings + sql "set enable_function_pushdown=true" + sql "set enable_profile=true" + sql "set profile_level=2" + + // Define test query + def query = "SELECT /*+SET_VAR(enable_function_pushdown = true, enable_profile = true, profile_level = 2)*/ * FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id" + // Test Case 1: Test with enable_add_index_for_new_data = true + logger.info("=== Test Case 1: enable_add_index_for_new_data = true ===") + // Create table + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + `sale_id` int NULL, + `sale_date` datetime NULL, + `product_name` varchar(100) NULL, + `customer_name` varchar(100) NULL, + `amount` decimal(10,2) NULL, + `region` char(50) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`sale_id`) + PARTITION BY RANGE(`sale_date`) ( + PARTITION p202310 VALUES [('2023-10-01 00:00:00'), ('2023-11-01 00:00:00')), + PARTITION p202311 VALUES [('2023-11-01 00:00:00'), ('2023-12-01 00:00:00')), + PARTITION p202312 VALUES [('2023-12-01 00:00:00'), ('2024-01-01 00:00:00')) + ) + DISTRIBUTED BY HASH(`sale_id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "true" + ); + """ + + // Insert test data + insertTestData() + // Verify data loaded correctly + qt_select_light_mode_init "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test without NGRAM Bloom Filter index + profile("sql_select_like_without_ngram_index_light_mode") { + run { + sql "/* sql_select_like_without_ngram_index_light_mode */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } + sql "set enable_add_index_for_new_data = true" + + // Add NGRAM Bloom Filter index (should be immediate in light mode) + sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" + + // In light mode, the index should be effective immediately, no need to wait for alter job + // But let's give it a moment to ensure metadata is updated + sleep(2000) + + // Insert more data after index added + insertTestData() + // Verify more data loaded correctly + qt_select_light_mode_more_data "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test with more data (should still filter correctly) + profile("sql_select_like_with_ngram_index_light_mode_more_data") { + run { + sql "/* sql_select_like_with_ngram_index_light_mode_more_data */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 10")) + } + } + + // Drop index + sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" + wait_for_latest_op_on_table_finish(tableName, timeout) + + // Test after dropping index + profile("sql_select_like_with_ngram_index_light_mode_dropped") { + run { + sql "/* sql_select_like_with_ngram_index_light_mode_dropped */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } + + // Test Case 2: Test with enable_add_index_for_new_data = false (schema change mode) + logger.info("=== Test Case 2: enable_add_index_for_new_data = false ===") + // Set enable_add_index_for_new_data = false + sql "set enable_add_index_for_new_data = false" + // Create new table + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + `sale_id` int NULL, + `sale_date` datetime NULL, + `product_name` varchar(100) NULL, + `customer_name` varchar(100) NULL, + `amount` decimal(10,2) NULL, + `region` char(50) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`sale_id`) + PARTITION BY RANGE(`sale_date`) ( + PARTITION p202310 VALUES [('2023-10-01 00:00:00'), ('2023-11-01 00:00:00')), + PARTITION p202311 VALUES [('2023-11-01 00:00:00'), ('2023-12-01 00:00:00')), + PARTITION p202312 VALUES [('2023-12-01 00:00:00'), ('2024-01-01 00:00:00')) + ) + DISTRIBUTED BY HASH(`sale_id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "true" + ); + """ + // Insert test data + insertTestData() + // Verify data loaded correctly + qt_select_schema_change_mode_init "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test without NGRAM Bloom Filter index + profile("sql_select_like_without_ngram_index_schema_change_mode") { + run { + sql "/* sql_select_like_without_ngram_index_schema_change_mode */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } + + // Add NGRAM Bloom Filter index (will trigger schema change in this mode) + sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" + wait_for_latest_op_on_table_finish(tableName, timeout) + + // Test after adding NGRAM Bloom Filter index (should filter existing data) + profile("sql_select_like_with_ngram_index_schema_change_mode_added") { + run { + sql "/* sql_select_like_with_ngram_index_schema_change_mode_added */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 10")) + } + } + + // Insert more data after index is built + insertTestData() + // Verify more data loaded correctly + qt_select_schema_change_mode_more_data "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test with more data (should filter all data) + profile("sql_select_like_with_ngram_index_schema_change_mode_more_data") { + run { + sql "/* sql_select_like_with_ngram_index_schema_change_mode_more_data */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 20")) + } + } + + // Drop index + sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" + wait_for_latest_op_on_table_finish(tableName, timeout) + + // Test after dropping index + profile("sql_select_like_with_ngram_index_schema_change_mode_dropped") { + run { + sql "/* sql_select_like_with_ngram_index_schema_change_mode_dropped */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } + + // Test Case 3: Test different scenarios for index lifecycle + logger.info("=== Test Case 3: Index lifecycle with enable_add_index_for_new_data = true ===") + // Set enable_add_index_for_new_data = true + sql "set enable_add_index_for_new_data = true" + // Create table and add index before inserting data + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + `sale_id` int NULL, + `sale_date` datetime NULL, + `product_name` varchar(100) NULL, + `customer_name` varchar(100) NULL, + `amount` decimal(10,2) NULL, + `region` char(50) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`sale_id`) + PARTITION BY RANGE(`sale_date`) ( + PARTITION p202310 VALUES [('2023-10-01 00:00:00'), ('2023-11-01 00:00:00')), + PARTITION p202311 VALUES [('2023-11-01 00:00:00'), ('2023-12-01 00:00:00')), + PARTITION p202312 VALUES [('2023-12-01 00:00:00'), ('2024-01-01 00:00:00')) + ) + DISTRIBUTED BY HASH(`sale_id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "true" + ); + """ + + // Add ngram bf index before data insertion + sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" + wait_for_latest_op_on_table_finish(tableName, timeout) + + // Insert data after index creation + insertTestData() + // Verify data loaded correctly + qt_select_lifecycle_after_data "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test filtering with index added before data insertion + profile("sql_select_like_with_ngram_index_lifecycle_test") { + run { + sql "/* sql_select_like_with_ngram_index_lifecycle_test */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 10")) + } + } + + // Insert more data + insertTestData() + // Verify more data loaded correctly + qt_select_lifecycle_final "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test filtering with more data + profile("sql_select_like_with_ngram_index_lifecycle_final") { + run { + sql "/* sql_select_like_with_ngram_index_lifecycle_final */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 20")) + } + } + + // Final cleanup + sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" + sleep(2000) +} \ No newline at end of file