diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index fa597fcb3d13be..8abddccf7d3911 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -879,7 +879,6 @@ nonterminal Map key_value_map, opt_key_value_map, opt_key_value_ opt_ext_properties, opt_enable_feature_properties, properties; nonterminal ColumnDef column_definition; nonterminal IndexDef index_definition; -nonterminal IndexDef build_index_definition; nonterminal ArrayList column_definition_list; nonterminal ArrayList index_definition_list; nonterminal AggregateType opt_agg_type; @@ -2070,7 +2069,7 @@ create_stmt ::= :} | KW_BUILD KW_INDEX ident:indexName KW_ON table_name:tableName opt_partition_names:partitionNames {: - RESULT = new AlterTableStmt(tableName, Lists.newArrayList(new BuildIndexClause(tableName, new IndexDef(indexName, partitionNames, true), false))); + RESULT = new AlterTableStmt(tableName, Lists.newArrayList(new BuildIndexClause(tableName, indexName, partitionNames, false))); :} /* stage */ | KW_CREATE KW_STAGE opt_if_not_exists:ifNotExists ident:stageName KW_PROPERTIES opt_key_value_map:properties @@ -4043,13 +4042,6 @@ index_definition ::= :} ; -build_index_definition ::= - KW_INDEX ident:indexName opt_partition_names:partitionNames - {: - RESULT = new IndexDef(indexName, partitionNames, true); - :} - ; - opt_nullable_type ::= {: RESULT = ColumnNullableType.DEFAULT; diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index a235fd76182977..66f66c571ed180 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1259,7 +1259,12 @@ private void checkAssignedTargetIndexName(String baseIndexName, String targetInd } private void createJob(String rawSql, long dbId, OlapTable olapTable, Map> indexSchemaMap, - Map propertyMap, List indexes) throws UserException { + Map propertyMap, List indexes, + boolean isBuildIndex) throws UserException { + if (isBuildIndex) { + // remove the index which is not the base index, only base index can be built index + indexSchemaMap.entrySet().removeIf(entry -> !entry.getKey().equals(olapTable.getBaseIndexId())); + } checkReplicaCount(olapTable); // process properties first @@ -1295,7 +1300,7 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map newSet = new HashSet<>(indexes); Set oriSet = new HashSet<>(olapTable.getIndexes()); - if (!newSet.equals(oriSet)) { + if (!newSet.equals(oriSet) || isBuildIndex) { hasIndexChange = true; } @@ -1311,7 +1316,7 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map oriBfColumns = olapTable.getCopiedBfColumns(); double oriBfFpp = olapTable.getBfFpp(); @@ -1947,7 +1952,7 @@ public int getAsInt() { List newIndexes = olapTable.getCopiedIndexes(); List alterIndexes = new ArrayList<>(); - Map> invertedIndexOnPartitions = new HashMap<>(); + Map> indexOnPartitions = new HashMap<>(); boolean isDropIndex = false; Map propertyMap = new HashMap<>(); for (AlterClause alterClause : alterClauses) { @@ -2085,68 +2090,29 @@ public int getAsInt() { } lightSchemaChange = false; - if (index.isLightIndexChangeSupported() && !Config.isCloudMode()) { + // ngram_bf index can do light_schema_change in both local and cloud mode + // inverted index can only do light_schema_change in local mode + if (index.isLightIndexChangeSupported()) { alterIndexes.add(index); isDropIndex = false; - // now only support light index change for inverted index lightIndexChange = true; } } else if (alterClause instanceof BuildIndexClause) { BuildIndexClause buildIndexClause = (BuildIndexClause) alterClause; IndexDef indexDef = buildIndexClause.getIndexDef(); Index index = buildIndexClause.getIndex(); - if (Config.isCloudMode()) { + if (Config.isCloudMode() && index.getIndexType() == IndexDef.IndexType.INVERTED) { throw new DdlException("BUILD INDEX operation failed: No need to do it in cloud mode."); } - if (!olapTable.isPartitionedTable()) { - List specifiedPartitions = indexDef.getPartitionNames(); - if (!specifiedPartitions.isEmpty()) { - throw new DdlException("table " + olapTable.getName() - + " is not partitioned, cannot build index with partitions."); - } - } - List existedIndexes = olapTable.getIndexes(); - boolean found = false; - for (Index existedIdx : existedIndexes) { - if (existedIdx.getIndexName().equalsIgnoreCase(indexDef.getIndexName())) { - found = true; - if (!existedIdx.isLightIndexChangeSupported()) { - throw new DdlException("BUILD INDEX operation failed: The index " - + existedIdx.getIndexName() + " of type " + existedIdx.getIndexType() - + " does not support lightweight index changes."); - } - for (Column column : olapTable.getBaseSchema()) { - if (!column.getType().isVariantType()) { - continue; - } - // variant type column can not support for building index - for (String indexColumn : existedIdx.getColumns()) { - if (column.getName().equalsIgnoreCase(indexColumn)) { - throw new DdlException("BUILD INDEX operation failed: The " - + indexDef.getIndexName() + " index can not be built on the " - + indexColumn + " column, because it is a variant type column."); - } - } - } - index = existedIdx.clone(); - if (indexDef.getPartitionNames().isEmpty()) { - invertedIndexOnPartitions.put(index.getIndexId(), olapTable.getPartitionNames()); - } else { - invertedIndexOnPartitions.put( - index.getIndexId(), new HashSet<>(indexDef.getPartitionNames())); - } - break; - } - } - if (!found) { - throw new DdlException("index " + indexDef.getIndexName() - + " not exist, cannot build it with defferred."); + if (indexDef.getPartitionNames().isEmpty()) { + indexOnPartitions.put(index.getIndexId(), olapTable.getPartitionNames()); + } else { + indexOnPartitions.put( + index.getIndexId(), new HashSet<>(indexDef.getPartitionNames())); } - if (indexDef.isInvertedIndex()) { - alterIndexes.add(index); - } + alterIndexes.add(index); buildIndexChange = true; lightSchemaChange = false; } else if (alterClause instanceof DropIndexClause) { @@ -2164,7 +2130,9 @@ public int getAsInt() { break; } } - if (found.isLightIndexChangeSupported() && !Config.isCloudMode()) { + // only inverted index with local mode can do light drop index change + if (found != null && found.getIndexType() == IndexDef.IndexType.INVERTED + && Config.isNotCloudMode()) { alterIndexes.add(found); isDropIndex = true; lightIndexChange = true; @@ -2185,19 +2153,27 @@ public int getAsInt() { long jobId = Env.getCurrentEnv().getNextId(); //for schema change add/drop value column optimize, direct modify table meta. modifyTableLightSchemaChange(rawSql, db, olapTable, indexSchemaMap, newIndexes, - null, isDropIndex, jobId, false); + null, isDropIndex, jobId, false, propertyMap); } else if (Config.enable_light_index_change && lightIndexChange) { long jobId = Env.getCurrentEnv().getNextId(); - //for schema change add/drop inverted index optimize, direct modify table meta firstly. + //for schema change add/drop inverted index and ngram_bf optimize, direct modify table meta firstly. modifyTableLightSchemaChange(rawSql, db, olapTable, indexSchemaMap, newIndexes, - alterIndexes, isDropIndex, jobId, false); + alterIndexes, isDropIndex, jobId, false, propertyMap); } else if (buildIndexChange) { + if (alterIndexes.isEmpty()) { + throw new DdlException("Altered index is empty. please check your alter stmt."); + } + IndexDef.IndexType indexType = alterIndexes.get(0).getIndexType(); if (Config.enable_light_index_change) { - buildOrDeleteTableInvertedIndices(db, olapTable, indexSchemaMap, - alterIndexes, invertedIndexOnPartitions, false); + if (indexType == IndexDef.IndexType.INVERTED) { + buildOrDeleteTableInvertedIndices(db, olapTable, indexSchemaMap, + alterIndexes, indexOnPartitions, false); + } else { + createJob(rawSql, db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes, true); + } } } else { - createJob(rawSql, db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes); + createJob(rawSql, db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes, false); } } finally { olapTable.writeUnlock(); @@ -2693,6 +2669,8 @@ private void cancelIndexJob(CancelAlterTableStmt cancelAlterTableStmt) throws Dd olapTable.writeUnlock(); } + // if this table has ngram_bf index, we must run cancel for schema change job + boolean hasNGramBFIndex = ((OlapTable) olapTable).hasIndexOfType(IndexDef.IndexType.NGRAM_BF); // alter job v2's cancel must be called outside the table lock if (jobList.size() > 0) { for (IndexChangeJob job : jobList) { @@ -2707,6 +2685,8 @@ private void cancelIndexJob(CancelAlterTableStmt cancelAlterTableStmt) throws Dd LOG.info("cancel build index job {} on table {} success", jobId, tableName); } } + } else if (hasNGramBFIndex) { + cancelColumnJob(cancelAlterTableStmt); } else { throw new DdlException("No job to cancel for Table[" + tableName + "]"); } @@ -2746,7 +2726,7 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab Column column = olapTable.getColumn(col); if (column != null) { indexDef.checkColumn(column, olapTable.getKeysType(), - olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite(), + olapTable.getEnableUniqueKeyMergeOnWrite(), olapTable.getInvertedIndexFileStorageFormat()); } else { throw new DdlException("index column does not exist in table. invalid column: " + col); @@ -2881,7 +2861,7 @@ public void replayAlterJobV2(AlterJobV2 alterJob) throws AnalysisException { public void modifyTableLightSchemaChange(String rawSql, Database db, OlapTable olapTable, Map> indexSchemaMap, List indexes, List alterIndexes, boolean isDropIndex, - long jobId, boolean isReplay) + long jobId, boolean isReplay, Map propertyMap) throws DdlException, AnalysisException { if (LOG.isDebugEnabled()) { @@ -2960,7 +2940,7 @@ public void modifyTableLightSchemaChange(String rawSql, Database db, OlapTable o } try { buildOrDeleteTableInvertedIndices(db, olapTable, indexSchemaMap, - alterIndexes, invertedIndexOnPartitions, true); + alterIndexes, invertedIndexOnPartitions, true); } catch (Exception e) { throw new DdlException(e.getMessage()); } @@ -3021,7 +3001,8 @@ public void replayModifyTableLightSchemaChange(TableAddOrDropColumnsInfo info) OlapTable olapTable = (OlapTable) db.getTableOrMetaException(tableId, TableType.OLAP); olapTable.writeLock(); try { - modifyTableLightSchemaChange("", db, olapTable, indexSchemaMap, indexes, null, false, jobId, true); + modifyTableLightSchemaChange("", db, olapTable, indexSchemaMap, indexes, null, false, jobId, + true, new HashMap<>()); } catch (DdlException e) { // should not happen LOG.warn("failed to replay modify table add or drop or modify columns", e); @@ -3161,7 +3142,7 @@ public void replayModifyTableAddOrDropInvertedIndices(TableAddOrDropInvertedIndi olapTable.writeLock(); try { modifyTableLightSchemaChange("", db, olapTable, indexSchemaMap, newIndexes, - alterIndexes, isDropIndex, jobId, true); + alterIndexes, isDropIndex, jobId, true, new HashMap<>()); } catch (UserException e) { // should not happen LOG.warn("failed to replay modify table add or drop indexes", e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 693629bfef4c39..531a5e13cf187c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -145,7 +145,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 { protected boolean hasRowStoreChange = false; // save all schema change tasks - private AgentBatchTask schemaChangeBatchTask = new AgentBatchTask(); + AgentBatchTask schemaChangeBatchTask = new AgentBatchTask(); protected SchemaChangeJobV2() { super(JobType.SCHEMA_CHANGE); @@ -651,14 +651,13 @@ protected void runRunningJob() throws AlterCancelException { healthyReplicaNum++; } } - if (!FeConstants.runningUnitTest) { - if (healthyReplicaNum < expectReplicationNum / 2 + 1) { - LOG.warn("shadow tablet {} has few healthy replicas: {}, schema change job: {}" - + " healthyReplicaNum {} expectReplicationNum {}", - shadowTablet.getId(), replicas, jobId, healthyReplicaNum, expectReplicationNum); - throw new AlterCancelException( + + if ((healthyReplicaNum < expectReplicationNum / 2 + 1) && !FeConstants.runningUnitTest) { + LOG.warn("shadow tablet {} has few healthy replicas: {}, schema change job: {}" + + " healthyReplicaNum {} expectReplicationNum {}", + shadowTablet.getId(), replicas, jobId, healthyReplicaNum, expectReplicationNum); + throw new AlterCancelException( "shadow tablet " + shadowTablet.getId() + " has few healthy replicas"); - } } } // end for tablets } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java index 6da8b86cdad1af..ba72155f7ec5fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java @@ -18,12 +18,17 @@ package org.apache.doris.analysis; import org.apache.doris.alter.AlterOpType; +import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Index; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import com.google.common.collect.Maps; +import java.util.List; import java.util.Map; public class BuildIndexClause extends AlterTableClause { @@ -36,11 +41,14 @@ public class BuildIndexClause extends AlterTableClause { private boolean alter; // index internal class private Index index; + private String indexName; + private PartitionNames partitionNames; - public BuildIndexClause(TableName tableName, IndexDef indexDef, boolean alter) { + public BuildIndexClause(TableName tableName, String indexName, PartitionNames partitionNames, boolean alter) { super(AlterOpType.SCHEMA_CHANGE); this.tableName = tableName; - this.indexDef = indexDef; + this.indexName = indexName; + this.partitionNames = partitionNames; this.alter = alter; } @@ -76,17 +84,52 @@ public TableName getTableName() { @Override public void analyze(Analyzer analyzer) throws AnalysisException { - if (indexDef == null) { - throw new AnalysisException("index definition expected."); + tableName.analyze(analyzer); + DatabaseIf db = Env.getCurrentEnv().getCatalogMgr().getInternalCatalog() + .getDb(tableName.getDb()).orElse(null); + if (db == null) { + throw new AnalysisException("Database[" + tableName.getDb() + "] is not exist"); + } + + TableIf table = db.getTable(tableName.getTbl()).orElse(null); + if (table == null) { + throw new AnalysisException("Table[" + tableName.getTbl() + "] is not exist"); + } + if (!(table instanceof OlapTable)) { + throw new AnalysisException("Only olap table support build index"); + } + + Index existedIdx = null; + for (Index index : table.getTableIndexes().getIndexes()) { + if (index.getIndexName().equalsIgnoreCase(indexName)) { + existedIdx = index; + if (!existedIdx.isLightIndexChangeSupported()) { + throw new AnalysisException("BUILD INDEX operation failed: The index " + + existedIdx.getIndexName() + " of type " + existedIdx.getIndexType() + + " does not support lightweight index changes."); + } + break; + } + } + if (existedIdx == null) { + throw new AnalysisException("Index[" + indexName + "] is not exist in table[" + tableName.getTbl() + "]"); } - if (indexDef.getIndexType() == IndexDef.IndexType.NGRAM_BF - || indexDef.getIndexType() == IndexDef.IndexType.BLOOMFILTER) { - throw new AnalysisException("ngram bloomfilter or bloomfilter index is not needed to build."); + + IndexDef.IndexType indexType = existedIdx.getIndexType(); + if (!existedIdx.isLightIndexChangeSupported()) { + throw new AnalysisException(indexType.toString() + " index is not needed to build."); + } + + indexDef = new IndexDef(indexName, partitionNames, indexType, true); + if (!table.isPartitionedTable()) { + List specifiedPartitions = indexDef.getPartitionNames(); + if (!specifiedPartitions.isEmpty()) { + throw new AnalysisException("table " + table.getName() + + " is not partitioned, cannot build index with partitions."); + } } indexDef.analyze(); - this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(), - indexDef.getColumns(), indexDef.getIndexType(), - indexDef.getProperties(), indexDef.getComment()); + this.index = existedIdx.clone(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java index 143c9f09d2a144..06334873bd9351 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java @@ -83,9 +83,9 @@ public IndexDef(String indexName, boolean ifNotExists, List columns, Ind } } - public IndexDef(String indexName, PartitionNames partitionNames, boolean isBuildDeferred) { + public IndexDef(String indexName, PartitionNames partitionNames, IndexType indexType, boolean isBuildDeferred) { this.indexName = indexName; - this.indexType = IndexType.INVERTED; + this.indexType = indexType; this.partitionNames = partitionNames; this.isBuildDeferred = isBuildDeferred; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index 1718c26bdb4566..af4ff1501d61a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.IndexDef; import org.apache.doris.analysis.InvertedIndexUtil; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.PrintableMap; @@ -167,8 +168,17 @@ public String getInvertedIndexParserStopwords() { return InvertedIndexUtil.getInvertedIndexParserStopwords(properties); } + // Whether the index can be changed in light mode + // cloud mode only supports light change for ngram_bf index + // local mode supports light change for both inverted index and ngram_bf index + // the rest of the index types do not support light change public boolean isLightIndexChangeSupported() { - return indexType == IndexDef.IndexType.INVERTED; + if (Config.isCloudMode()) { + return indexType == IndexDef.IndexType.NGRAM_BF; + } else { + return indexType == IndexDef.IndexType.INVERTED + || indexType == IndexDef.IndexType.NGRAM_BF; + } } public String getComment() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index ca97c09f43db9e..a7c5a066971de8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -24,6 +24,7 @@ import org.apache.doris.analysis.CreateMaterializedViewStmt; import org.apache.doris.analysis.DataSortInfo; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.IndexDef; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotRef; import org.apache.doris.backup.Status; @@ -366,6 +367,19 @@ public List getIndexIds() { return indexes.getIndexIds(); } + /** + * Checks if the table contains at least one index of the specified type. + * @param indexType The index type to check for + * @return true if the table has at least one index of the specified type, false otherwise + */ + public boolean hasIndexOfType(IndexDef.IndexType indexType) { + if (indexes == null) { + return false; + } + return indexes.getIndexes().stream() + .anyMatch(index -> index.getIndexType() == indexType); + } + @Override public TableIndexes getTableIndexes() { return indexes; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 73e0c50e46b8d3..ecfe1f0c862175 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -5015,8 +5015,8 @@ public Command visitBuildIndex(BuildIndexContext ctx) { Pair> partitionSpec = visitPartitionSpec(ctx.partitionSpec()); partitionNamesInfo = new PartitionNamesInfo(partitionSpec.first, partitionSpec.second); } - IndexDefinition indexDefinition = new IndexDefinition(name, partitionNamesInfo); - List alterTableOps = Lists.newArrayList(new BuildIndexOp(tableName, indexDefinition, false)); + List alterTableOps = Lists.newArrayList(new BuildIndexOp(tableName, name, partitionNamesInfo, + false)); return new AlterTableCommand(tableName, alterTableOps); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java index 11b9df12eced9d..36439874760144 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java @@ -20,13 +20,20 @@ import org.apache.doris.alter.AlterOpType; import org.apache.doris.analysis.AlterTableClause; import org.apache.doris.analysis.BuildIndexClause; +import org.apache.doris.analysis.IndexDef; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Index; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.Maps; +import java.util.List; import java.util.Map; /** @@ -36,17 +43,23 @@ public class BuildIndexOp extends AlterTableOp { // in which table the index on, only used when alter = false private final TableNameInfo tableName; // index definition class - private final IndexDefinition indexDef; + private IndexDefinition indexDef; // when alter = true, clause like: alter table add index xxxx // when alter = false, clause like: create index xx on table xxxx private final boolean alter; // index internal class private Index index; + // index name + private final String indexName; + // partition names info + private final PartitionNamesInfo partitionNamesInfo; - public BuildIndexOp(TableNameInfo tableName, IndexDefinition indexDef, boolean alter) { + public BuildIndexOp(TableNameInfo tableName, String indexName, PartitionNamesInfo partitionNamesInfo, + boolean alter) { super(AlterOpType.SCHEMA_CHANGE); this.tableName = tableName; - this.indexDef = indexDef; + this.indexName = indexName; + this.partitionNamesInfo = partitionNamesInfo; this.alter = alter; } @@ -69,16 +82,57 @@ public boolean isAlter() { @Override public void validate(ConnectContext ctx) throws UserException { - if (indexDef == null) { - throw new AnalysisException("index definition expected."); + tableName.analyze(ctx); + DatabaseIf
db = Env.getCurrentEnv().getCatalogMgr().getInternalCatalog() + .getDb(tableName.getDb()).orElse(null); + if (db == null) { + throw new AnalysisException("Database[" + tableName.getDb() + "] is not exist"); + } + + TableIf table = db.getTable(tableName.getTbl()).orElse(null); + if (table == null) { + throw new AnalysisException("Table[" + tableName.getTbl() + "] is not exist"); + } + if (!(table instanceof OlapTable)) { + throw new AnalysisException("Only olap table support build index"); + } + + Index existedIdx = null; + for (Index index : table.getTableIndexes().getIndexes()) { + if (index.getIndexName().equalsIgnoreCase(indexName)) { + existedIdx = index; + if (!existedIdx.isLightIndexChangeSupported()) { + throw new AnalysisException("BUILD INDEX operation failed: The index " + + existedIdx.getIndexName() + " of type " + existedIdx.getIndexType() + + " does not support lightweight index changes."); + } + break; + } + } + if (existedIdx == null) { + throw new AnalysisException("Index[" + indexName + "] is not exist in table[" + tableName.getTbl() + "]"); + } + + IndexDef.IndexType indexType = existedIdx.getIndexType(); + if (!existedIdx.isLightIndexChangeSupported()) { + throw new AnalysisException(indexType.toString() + " index is not needed to build."); + } + + indexDef = new IndexDefinition(indexName, partitionNamesInfo, indexType); + if (!table.isPartitionedTable()) { + List specifiedPartitions = indexDef.getPartitionNames(); + if (!specifiedPartitions.isEmpty()) { + throw new AnalysisException("table " + table.getName() + + " is not partitioned, cannot build index with partitions."); + } } indexDef.validate(); - tableName.analyze(ctx); - index = indexDef.translateToCatalogStyle(); + this.index = existedIdx.clone(); } @Override public AlterTableClause translateToLegacyAlterClause() { + indexDef.getIndexType(); return new BuildIndexClause(tableName.transferToTableName(), indexDef.translateToLegacyIndexDef(), index, alter); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java index d051cc45519fce..0ec4c4cd58ea54 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java @@ -100,9 +100,9 @@ public IndexDefinition(String name, boolean ifNotExists, List cols, Stri /** * constructor for build index */ - public IndexDefinition(String name, PartitionNamesInfo partitionNames) { + public IndexDefinition(String name, PartitionNamesInfo partitionNames, IndexType indexType) { this.name = name; - this.indexType = IndexType.INVERTED; + this.indexType = indexType; this.partitionNames = partitionNames; this.isBuildDeferred = true; this.cols = null; @@ -260,13 +260,17 @@ public Index translateToCatalogStyle() { comment); } + public List getPartitionNames() { + return partitionNames == null ? Lists.newArrayList() : partitionNames.getPartitionNames(); + } + /** * translateToLegacyIndexDef */ public IndexDef translateToLegacyIndexDef() { if (isBuildDeferred) { return new IndexDef(name, partitionNames != null ? partitionNames.translateToLegacyPartitionNames() : null, - true); + indexType, true); } else { return new IndexDef(name, ifNotExists, cols, indexType, properties, comment); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java index 6b0943c306f851..20bd26fa92db50 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java @@ -55,6 +55,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import mockit.Mock; +import mockit.MockUp; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -85,6 +87,8 @@ public class IndexChangeJobTest { private static BuildIndexClause buildIndexClause; private static DropIndexClause dropIndexClause; private static CancelAlterTableStmt cancelAlterTableStmt; + private static TableName tableName; + private static String indexName; @Rule public ExpectedException expectedEx = ExpectedException.none(); @@ -108,25 +112,31 @@ public void setUp() db = masterEnv.getInternalCatalog().getDbOrDdlException(CatalogTestUtil.testDbId1); olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId1); + new MockUp() { + @Mock + public Env getCurrentEnv() { + return masterEnv; + } + }; + // set mow table property Map properties = Maps.newHashMap(); properties.put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE, "false"); TableProperty tableProperty = new TableProperty(properties); olapTable.setTableProperty(tableProperty); - TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), olapTable.getName()); - IndexDef indexDef = new IndexDef("index1", false, + indexName = "index1"; + IndexDef indexDef = new IndexDef(indexName, false, Lists.newArrayList(olapTable.getBaseSchema().get(1).getName()), IndexDef.IndexType.INVERTED, Maps.newHashMap(), "balabala"); + FakeEnv.setEnv(masterEnv); createIndexClause = new CreateIndexClause(tableName, indexDef, false); createIndexClause.analyze(analyzer); - buildIndexClause = new BuildIndexClause(tableName, indexDef, false); - buildIndexClause.analyze(analyzer); - - dropIndexClause = new DropIndexClause("index1", false, tableName, false); + dropIndexClause = new DropIndexClause(indexName, false, tableName, false); dropIndexClause.analyze(analyzer); cancelAlterTableStmt = new CancelAlterTableStmt(ShowAlterStmt.AlterType.INDEX, tableName); @@ -163,6 +173,8 @@ public void testBuildIndexIndexChange() throws UserException { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -203,6 +215,8 @@ public void testBuildIndexIndexChangeNormal() throws UserException { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -292,6 +306,8 @@ public void testCancelBuildIndexIndexChangeNormal() throws UserException { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -339,6 +355,8 @@ public void testBuildIndexIndexChangeWhileTableNotStable() throws Exception { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -467,6 +485,8 @@ public void testBuildIndexFailedWithMinFailedNum() throws Exception { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -519,6 +539,8 @@ public void testBuildIndexFailedWithMaxFailedNum() throws Exception { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -526,13 +548,13 @@ public void testBuildIndexFailedWithMaxFailedNum() throws Exception { Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState()); IndexChangeJob indexChangejob = indexChangeJobMap.values().stream().findAny().get(); - Assert.assertEquals(indexChangejob.invertedIndexBatchTask.getTaskNum(), 0); + Assert.assertEquals(0, indexChangejob.invertedIndexBatchTask.getTaskNum()); Assert.assertEquals(IndexChangeJob.JobState.WAITING_TXN, indexChangejob.getJobState()); // run waiting txn job schemaChangeHandler.runAfterCatalogReady(); Assert.assertEquals(IndexChangeJob.JobState.RUNNING, indexChangejob.getJobState()); - Assert.assertEquals(indexChangejob.invertedIndexBatchTask.getTaskNum(), 3); + Assert.assertEquals(3, indexChangejob.invertedIndexBatchTask.getTaskNum()); // run running job schemaChangeHandler.runAfterCatalogReady(); Assert.assertEquals(IndexChangeJob.JobState.RUNNING, indexChangejob.getJobState()); @@ -565,17 +587,123 @@ public void testNgramBfBuildIndex() throws UserException { fakeEditLog = new FakeEditLog(); FakeEnv.setEnv(masterEnv); - IndexDef indexDef = new IndexDef("ngram_bf_index", false, - Lists.newArrayList(olapTable.getBaseSchema().get(1).getName()), + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + String indexName = "ngram_bf_index"; + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, Maps.newHashMap(), "ngram bf index"); TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), - olapTable.getName()); + table.getName()); createIndexClause = new CreateIndexClause(tableName, indexDef, false); createIndexClause.analyze(analyzer); + SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); + + long jobId = indexChangeJobMap.values().stream().findAny().get().jobId; + + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); + alterClauses.clear(); + alterClauses.add(buildIndexClause); + + schemaChangeHandler.process(alterClauses, db, table); + Assert.assertEquals(2, indexChangeJobMap.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + + SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() + .filter(job -> job.jobId != jobId) + .findFirst() + .orElse(null); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, jobV2.getJobState()); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } - buildIndexClause = new BuildIndexClause(tableName, indexDef, false); - org.junit.jupiter.api.Assertions.assertThrows(org.apache.doris.common.AnalysisException.class, - () -> buildIndexClause.analyze(analyzer)); + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); + } + + @Test + public void testCancelNgramBfBuildIndex() throws UserException { + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + String indexName = "ngram_bf_index"; + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + Maps.newHashMap(), "ngram bf index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); + + long jobId = indexChangeJobMap.values().stream().findAny().get().jobId; + + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); + alterClauses.clear(); + alterClauses.add(buildIndexClause); + + schemaChangeHandler.process(alterClauses, db, table); + Assert.assertEquals(2, indexChangeJobMap.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + + SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() + .filter(job -> job.jobId != jobId) + .findFirst() + .orElse(null); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, jobV2.getJobState()); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + cancelAlterTableStmt = new CancelAlterTableStmt(ShowAlterStmt.AlterType.INDEX, tableName); + cancelAlterTableStmt.analyze(analyzer); + schemaChangeHandler.cancel(cancelAlterTableStmt); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.CANCELLED, jobV2.getJobState()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java index 73af753a69d7f2..4b8f218acea094 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java @@ -850,8 +850,13 @@ public void testDupAddOrDropNgramBfIndex() throws Exception { String buildNgramBfIndexStmtStr = "BUILD INDEX idx_error_msg on test.sc_dup "; AlterTableStmt buildNgramBfIndexStmt = (AlterTableStmt) parseAndAnalyzeStmt(buildNgramBfIndexStmtStr); - org.junit.jupiter.api.Assertions.assertThrows(org.apache.doris.common.DdlException.class, - () -> Env.getCurrentEnv().getAlterInstance().processAlterTable(buildNgramBfIndexStmt)); + Env.getCurrentEnv().getAlterInstance().processAlterTable(buildNgramBfIndexStmt); + + jobSize++; + alterJobs = Env.getCurrentEnv().getSchemaChangeHandler().getAlterJobsV2(); + LOG.info("alterJobs:{}", alterJobs); + Assertions.assertEquals(jobSize, alterJobs.size()); + waitAlterJobDone(alterJobs); tbl.readLock(); try { diff --git a/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out b/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out new file mode 100644 index 00000000000000..6f916a99c91705 --- /dev/null +++ b/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out @@ -0,0 +1,69 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + diff --git a/regression-test/pipeline/cloud_p0/conf/fe_custom.conf b/regression-test/pipeline/cloud_p0/conf/fe_custom.conf index 0b53fa2244df77..6f067f66056878 100644 --- a/regression-test/pipeline/cloud_p0/conf/fe_custom.conf +++ b/regression-test/pipeline/cloud_p0/conf/fe_custom.conf @@ -40,7 +40,6 @@ meta_service_endpoint=127.0.0.1:5000 cloud_unique_id=cloud_unique_id_sql_server00 # for case test_build_mtmv.groovy enable_job_schedule_second_for_test=true -enable_light_index_change=false workload_sched_policy_interval_ms = 1000 workload_group_max_num = 25 diff --git a/regression-test/pipeline/cloud_p1/conf/fe_custom.conf b/regression-test/pipeline/cloud_p1/conf/fe_custom.conf index 31ef26e19d375b..b91a4ed6d38bdf 100644 --- a/regression-test/pipeline/cloud_p1/conf/fe_custom.conf +++ b/regression-test/pipeline/cloud_p1/conf/fe_custom.conf @@ -33,7 +33,6 @@ priority_networks=127.0.0.1/24 cloud_http_port=18030 meta_service_endpoint=127.0.0.1:5000 cloud_unique_id=cloud_unique_id_sql_server00 -enable_light_index_change=false enable_advance_next_id = true arrow_flight_sql_port = 8081 diff --git a/regression-test/pipeline/vault_p0/conf/fe_custom.conf b/regression-test/pipeline/vault_p0/conf/fe_custom.conf index d35632eb7481a7..f62ffa19fc7676 100644 --- a/regression-test/pipeline/vault_p0/conf/fe_custom.conf +++ b/regression-test/pipeline/vault_p0/conf/fe_custom.conf @@ -37,7 +37,6 @@ meta_service_endpoint=127.0.0.1:5000 cloud_unique_id=cloud_unique_id_sql_server00 # for case test_build_mtmv.groovy enable_job_schedule_second_for_test=true -enable_light_index_change=false workload_sched_policy_interval_ms = 1000 enable_advance_next_id = true diff --git a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy new file mode 100644 index 00000000000000..6fea7a68f92b8e --- /dev/null +++ b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy @@ -0,0 +1,286 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +import groovy.json.JsonSlurper + +suite("test_ngram_bloomfilter_index_change") { + def tableName = 'test_ngram_bloomfilter_index_change' + def timeout = 60000 + def delta_time = 1000 + def alter_res = "null" + def useTime = 0 + + def wait_for_latest_op_on_table_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${table_name}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(3000) // wait change table state to normal + logger.info(table_name + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } + + // Function to insert test data batch + def insertTestData = { -> + // insert 10 records + sql "INSERT INTO ${tableName} VALUES (1001, '2023-10-06 15:00:00', 'Laptop', 'John Smith', 199.99, 'North');" + sql "INSERT INTO ${tableName} VALUES (1002, '2023-10-09 17:05:00', 'Smartphone', 'Emily Johnson', 299.99, 'South');" + sql "INSERT INTO ${tableName} VALUES (1003, '2023-10-12 19:10:00', 'Headphones', 'Michael Brown', 399.99, 'East');" + sql "INSERT INTO ${tableName} VALUES (1004, '2023-10-15 21:15:00', 'Monitor', 'Jessica Davis', 499.99, 'West');" + sql "INSERT INTO ${tableName} VALUES (1005, '2023-10-18 23:20:00', 'Keyboard', 'David Wilson', 89.99, 'North');" + sql "INSERT INTO ${tableName} VALUES (1006, '2023-10-21 07:25:00', 'Mouse', 'Sarah Taylor', 699.99, 'South');" + sql "INSERT INTO ${tableName} VALUES (1007, '2023-10-24 09:30:00', 'Printer', 'Thomas Anderson', 799.99, 'East');" + sql "INSERT INTO ${tableName} VALUES (1008, '2023-10-27 11:35:00', 'Speaker', 'Jennifer Martin', 899.99, 'West');" + sql "INSERT INTO ${tableName} VALUES (1009, '2023-10-02 13:40:00', 'External SSD', 'Robert Clark', 999.99, 'North');" + sql "INSERT INTO ${tableName} VALUES (1010, '2023-10-05 15:45:00', 'Webcam', 'Amanda Lewis', 89.99, 'South');" + sql "sync" + } + + // Test setup + // 1. Create table + // 2. Insert test data + // 3. Add NGRAM Bloom Filter index + // 4. Build index + // 5. Insert more data + // 6. Drop index + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + `sale_id` int NULL, + `sale_date` datetime NULL, + `product_name` varchar(100) NULL, + `customer_name` varchar(100) NULL, + `amount` decimal(10,2) NULL, + `region` char(50) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`sale_id`) + PARTITION BY RANGE(`sale_date`) ( + PARTITION p202310 VALUES [('2023-10-01 00:00:00'), ('2023-11-01 00:00:00')), + PARTITION p202311 VALUES [('2023-11-01 00:00:00'), ('2023-12-01 00:00:00')), + PARTITION p202312 VALUES [('2023-12-01 00:00:00'), ('2024-01-01 00:00:00')) + ) + DISTRIBUTED BY HASH(`sale_id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "false" + ); + """ + + // Insert first batch of data + insertTestData() + + // Test settings + sql "set enable_function_pushdown=true" + sql "set enable_profile=true" + sql "set profile_level=2" + + // Verify data loaded correctly + qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Define test query + def query = "SELECT /*+SET_VAR(enable_function_pushdown = true, enable_profile = true, profile_level = 2)*/ * FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id" + + // Test 1: without NGRAM Bloom Filter index + profile("sql_select_like_without_ngram_index") { + run { + sql "/* sql_select_like_without_ngram_index */ ${query}" + sleep(1000) // sleep 1s wait for the profile collection to be completed + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } + + // Test 2: After adding NGRAM Bloom Filter index + sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" + wait_for_latest_op_on_table_finish(tableName, timeout) + profile("sql_select_like_with_ngram_index_added") { + run { + sql "/* sql_select_like_with_ngram_index_added */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } + + // Test 3: After building the index + sql "BUILD INDEX idx_ngram_customer_name ON ${tableName};" + wait_for_latest_op_on_table_finish(tableName, timeout) + profile("sql_select_like_with_ngram_index_built") { + run { + sql "/* sql_select_like_with_ngram_index_built */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 10")) + } + } + + // Insert second batch of data + insertTestData() + // Verify data loaded correctly + qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test 4: Verify filtering with more data + profile("sql_select_like_with_ngram_index_more_data") { + run { + sql "/* sql_select_like_with_ngram_index_more_data */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 20")) + } + } + + // Test 5: After dropping the index + sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" + wait_for_latest_op_on_table_finish(tableName, timeout) + profile("sql_select_like_with_ngram_index_dropped") { + run { + sql "/* sql_select_like_with_ngram_index_dropped */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } + + // recreate table + // 1. Create table + // 2. Add NGRAM Bloom Filter index + // 3. Insert data + // 4. Insert more data + // 5. Build index + // 6. Drop index + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + `sale_id` int NULL, + `sale_date` datetime NULL, + `product_name` varchar(100) NULL, + `customer_name` varchar(100) NULL, + `amount` decimal(10,2) NULL, + `region` char(50) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`sale_id`) + PARTITION BY RANGE(`sale_date`) ( + PARTITION p202310 VALUES [('2023-10-01 00:00:00'), ('2023-11-01 00:00:00')), + PARTITION p202311 VALUES [('2023-11-01 00:00:00'), ('2023-12-01 00:00:00')), + PARTITION p202312 VALUES [('2023-12-01 00:00:00'), ('2024-01-01 00:00:00')) + ) + DISTRIBUTED BY HASH(`sale_id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "false" + ); + """ + + // add ngram bf index + sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" + wait_for_latest_op_on_table_finish(tableName, timeout) + + // insert data + insertTestData() + + // Verify data loaded correctly + qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test 6: Verify filtering with index added + profile("sql_select_like_with_ngram_index_recreated") { + run { + sql "/* sql_select_like_with_ngram_index_recreated */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 10")) + } + } + + // insert more data + insertTestData() + + // Verify data loaded correctly + qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test 7: Verify filtering with more data + profile("sql_select_like_with_ngram_index_recreated_more_data") { + run { + sql "/* sql_select_like_with_ngram_index_recreated_more_data */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 20")) + } + } + + // build index + sql "BUILD INDEX idx_ngram_customer_name ON ${tableName};" + wait_for_latest_op_on_table_finish(tableName, timeout) + + // Test 8: Verify filtering with index built + profile("sql_select_like_with_ngram_index_recreated_built") { + run { + sql "/* sql_select_like_with_ngram_index_recreated_built */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 20")) + } + } + + // drop index + sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" + wait_for_latest_op_on_table_finish(tableName, timeout) + + // Test 9: Verify filtering with index dropped + profile("sql_select_like_with_ngram_index_recreated_dropped") { + run { + sql "/* sql_select_like_with_ngram_index_recreated_dropped */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } +} \ No newline at end of file