Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,10 @@ public class IndexDef {
private boolean isBuildDeferred = false;
private PartitionNames partitionNames;
private List<Integer> columnUniqueIds = Lists.newArrayList();
private static final int MIN_NGRAM_SIZE = 1;
private static final int MAX_NGRAM_SIZE = 255;
private static final int MIN_BF_SIZE = 64;
private static final int MAX_BF_SIZE = 65535;
public static final int MIN_NGRAM_SIZE = 1;
public static final int MAX_NGRAM_SIZE = 255;
public static final int MIN_BF_SIZE = 64;
public static final int MAX_BF_SIZE = 65535;

public static final String NGRAM_SIZE_KEY = "gram_size";
public static final String NGRAM_BF_SIZE_KEY = "bf_size";
Expand Down Expand Up @@ -270,7 +270,7 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe
}
}

private void parseAndValidateProperty(Map<String, String> properties, String key, int minValue, int maxValue)
public static void parseAndValidateProperty(Map<String, String> properties, String key, int minValue, int maxValue)
throws AnalysisException {
String valueStr = properties.get(key);
if (valueStr == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -143,18 +143,12 @@ public void checkColumn(ColumnDefinition column, KeysType keysType,
"ngram_bf index should have gram_size and bf_size properties");
}
try {
int ngramSize = Integer.parseInt(properties.get(IndexDef.NGRAM_SIZE_KEY));
int bfSize = Integer.parseInt(properties.get(IndexDef.NGRAM_BF_SIZE_KEY));
if (ngramSize > 256 || ngramSize < 1) {
throw new AnalysisException(
"gram_size should be integer and less than 256");
}
if (bfSize > 65535 || bfSize < 64) {
throw new AnalysisException(
"bf_size should be integer and between 64 and 65535");
}
} catch (NumberFormatException e) {
throw new AnalysisException("invalid ngram properties:" + e.getMessage(), e);
IndexDef.parseAndValidateProperty(properties, IndexDef.NGRAM_SIZE_KEY, IndexDef.MIN_NGRAM_SIZE,
IndexDef.MAX_NGRAM_SIZE);
IndexDef.parseAndValidateProperty(properties, IndexDef.NGRAM_BF_SIZE_KEY, IndexDef.MIN_BF_SIZE,
IndexDef.MAX_BF_SIZE);
} catch (Exception ex) {
throw new AnalysisException("invalid ngram bf index params:" + ex.getMessage(), ex);
}
}
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,17 @@
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition;
import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.VariantType;

import com.google.common.collect.Lists;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

import java.util.HashMap;
import java.util.Map;

public class IndexDefinitionTest {
@Test
void testVariantIndexFormatV1() throws AnalysisException {
Expand All @@ -39,8 +44,65 @@ void testVariantIndexFormatV1() throws AnalysisException {
null, "comment"), KeysType.UNIQUE_KEYS, true, isIndexFormatV1);
Assertions.fail("No exception throws.");
} catch (AnalysisException e) {
Assertions.assertTrue(e instanceof AnalysisException);
org.junit.jupiter.api.Assertions.assertInstanceOf(
org.apache.doris.nereids.exceptions.AnalysisException.class, e);
Assertions.assertTrue(e.getMessage().contains("not supported in inverted index format V1"));
}
}

@Test
void testNgramBFIndex() throws AnalysisException {
Map<String, String> properties = new HashMap<>();
properties.put("gram_size", "3");
properties.put("bf_size", "10000");

IndexDefinition def = new IndexDefinition("ngram_bf_index", Lists.newArrayList("col1"), "NGRAM_BF",
properties, "comment");
def.checkColumn(
new ColumnDefinition("col1", StringType.INSTANCE, false, AggregateType.NONE, true, null, "comment"),
KeysType.DUP_KEYS, false, false);
}

@Test
void testInvalidNgramBFIndexColumnType() {
Map<String, String> properties = new HashMap<>();
properties.put("gram_size", "3");
properties.put("bf_size", "10000");

IndexDefinition def = new IndexDefinition("ngram_bf_index", Lists.newArrayList("col1"), "NGRAM_BF",
properties, "comment");
Assertions.assertThrows(AnalysisException.class, () ->
def.checkColumn(
new ColumnDefinition("col1", IntegerType.INSTANCE, false, AggregateType.NONE, true, null,
"comment"),
KeysType.DUP_KEYS, false, false));
}

@Test
void testNgramBFIndexInvalidSize() {
Map<String, String> properties = new HashMap<>();
properties.put("gram_size", "256");
properties.put("bf_size", "10000");

IndexDefinition def = new IndexDefinition("ngram_bf_index", Lists.newArrayList("col1"), "NGRAM_BF",
properties, "comment");
Assertions.assertThrows(AnalysisException.class, () ->
def.checkColumn(new ColumnDefinition("col1", StringType.INSTANCE, false, AggregateType.NONE, true, null,
"comment"),
KeysType.DUP_KEYS, false, false));
}

@Test
void testNgramBFIndexInvalidSize2() {
Map<String, String> properties = new HashMap<>();
properties.put("gram_size", "3");
properties.put("bf_size", "65536");

IndexDefinition def = new IndexDefinition("ngram_bf_index", Lists.newArrayList("col1"), "NGRAM_BF",
properties, "comment");
Assertions.assertThrows(AnalysisException.class, () ->
def.checkColumn(new ColumnDefinition("col1", StringType.INSTANCE, false, AggregateType.NONE, true, null,
"comment"),
KeysType.DUP_KEYS, false, false));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ suite("test_ngram_bloomfilter_index") {
DISTRIBUTED BY HASH(`key_id`) BUCKETS 3
PROPERTIES("replication_num" = "1");
"""
exception "bf_size should be integer and between 64 and 65535"
exception "java.sql.SQLException: errCode = 2, detailMessage = invalid ngram bf index params:errCode = 2, detailMessage = 'bf_size' should be an integer between 64 and 65535."
}

def tableName3 = 'test_ngram_bloomfilter_index3'
Expand All @@ -104,10 +104,10 @@ suite("test_ngram_bloomfilter_index") {
"""
test {
sql """ALTER TABLE ${tableName3} ADD INDEX idx_http_url(http_url) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="65536") COMMENT 'http_url ngram_bf index'"""
exception "'bf_size' should be an integer between 64 and 65535"
exception "java.sql.SQLException: errCode = 2, detailMessage = 'bf_size' should be an integer between 64 and 65535."
}
test {
sql """ALTER TABLE ${tableName3} ADD INDEX idx_http_url(http_url) USING NGRAM_BF PROPERTIES("gram_size"="256", "bf_size"="65535") COMMENT 'http_url ngram_bf index'"""
exception "'gram_size' should be an integer between 1 and 255"
exception "java.sql.SQLException: errCode = 2, detailMessage = 'gram_size' should be an integer between 1 and 255."
}
}
Loading