From 2e4a403f872fd72d8823c84d7696a552170465bc Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 21 Nov 2022 17:31:51 -0800 Subject: [PATCH] fix off by one error in nested column range index --- ...NestedFieldLiteralColumnIndexSupplier.java | 5 +- ...edFieldLiteralColumnIndexSupplierTest.java | 118 ++++++++++++++---- 2 files changed, 94 insertions(+), 29 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java b/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java index 373dcc822f2e..36c39f251b79 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java @@ -228,10 +228,10 @@ private IntIntPair getLocalRangeFromDictionary( // valid global index in local dictionary, start here localStartIndex = localFound; } - // global end index is exclusive already, so we don't adjust local end index even for missing values + int localEndFound = localDictionary.indexOf(globalEndIndex); if (localEndFound < 0) { - localEndIndex = -localEndFound; + localEndIndex = -(localEndFound + 1); } else { localEndIndex = localEndFound; } @@ -239,6 +239,7 @@ private IntIntPair getLocalRangeFromDictionary( localStartIndex = Math.min(localStartIndex, localDictionary.size()); localEndIndex = Math.max(localStartIndex, Math.min(localDictionary.size(), localEndIndex)); + return new IntIntImmutablePair(localStartIndex, localEndIndex); } diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java index 15b40b729f07..25fec68bb82b 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java @@ -89,6 +89,9 @@ public void setup() throws IOException stringWriter.write("fo"); stringWriter.write("foo"); stringWriter.write("fooo"); + stringWriter.write("g"); + stringWriter.write("gg"); + stringWriter.write("ggg"); stringWriter.write("z"); writeToBuffer(stringBuffer, stringWriter); @@ -197,13 +200,44 @@ public void testSingleTypeStringColumnRangeIndex() throws IOException Assert.assertNotNull(rangeIndex); // 10 rows + // global: [null, a, b, fo, foo, fooo, g, gg, ggg, z] // local: [b, foo, fooo, z] // column: [foo, b, fooo, b, z, fooo, z, b, b, foo] - BitmapColumnIndex forRange = rangeIndex.forRange("f", true, "g", true); + BitmapColumnIndex forRange = rangeIndex.forRange(null, false, "a", false); Assert.assertNotNull(forRange); - Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap); + + forRange = rangeIndex.forRange(null, false, "b", false); + Assert.assertNotNull(forRange); + Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 1, 3, 7, 8); + + forRange = rangeIndex.forRange("b", false, "fon", false); + Assert.assertNotNull(forRange); + Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 1, 3, 7, 8); + + forRange = rangeIndex.forRange("bb", false, "fon", false); + Assert.assertNotNull(forRange); + Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap); + + forRange = rangeIndex.forRange("b", true, "foo", false); + Assert.assertNotNull(forRange); + Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 9); + + forRange = rangeIndex.forRange("f", true, "g", true); + Assert.assertNotNull(forRange); + Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 5, 9); forRange = rangeIndex.forRange(null, false, "g", true); @@ -257,6 +291,36 @@ public void testSingleTypeStringColumnRangeIndex() throws IOException Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + + forRange = rangeIndex.forRange(null, true, "foa", false); + Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 1, 3, 7, 8); + + forRange = rangeIndex.forRange(null, true, "foooa", false); + Assert.assertEquals(0.8, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 1, 2, 3, 5, 7, 8, 9); + + forRange = rangeIndex.forRange("foooa", true, "ggg", false); + Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap); + + forRange = rangeIndex.forRange("g", true, "gg", false); + Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap); + + forRange = rangeIndex.forRange("z", true, "zz", false); + Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap); + + forRange = rangeIndex.forRange("z", false, "zz", false); + Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 4, 6); } @Test @@ -1123,7 +1187,7 @@ private NestedFieldLiteralColumnIndexSupplier makeSingleTypeStringSupplier() // 10 rows // globals: [ - // [null, a, b, fo, foo, fooo, z], + // [null, a, b, fo, foo, fooo, g, gg, ggg, z], // [1, 2, 3, 5, 100, 300, 9000], // [1.0, 1.1, 1.2, 2.0, 2.5, 3.3, 6.6, 9.9] // ] @@ -1143,7 +1207,7 @@ private NestedFieldLiteralColumnIndexSupplier makeSingleTypeStringSupplier() bitmapWriter.write(fillBitmap(2, 5)); // z - localDictionaryWriter.write(6); + localDictionaryWriter.write(9); bitmapWriter.write(fillBitmap(4, 6)); writeToBuffer(localDictionaryBuffer, localDictionaryWriter); @@ -1193,7 +1257,7 @@ private NestedFieldLiteralColumnIndexSupplier makeSingleTypeStringWithNullsSu bitmapWriter.open(); // 10 rows // globals: [ - // [null, a, b, fo, foo, fooo, z], + // [null, a, b, fo, foo, fooo, g, gg, ggg, z], // [1, 2, 3, 5, 100, 300, 9000], // [1.0, 1.1, 1.2, 2.0, 2.5, 3.3, 6.6, 9.9] // ] @@ -1217,7 +1281,7 @@ private NestedFieldLiteralColumnIndexSupplier makeSingleTypeStringWithNullsSu bitmapWriter.write(fillBitmap(2, 5)); // z - localDictionaryWriter.write(6); + localDictionaryWriter.write(9); bitmapWriter.write(fillBitmap(4, 6)); writeToBuffer(localDictionaryBuffer, localDictionaryWriter); @@ -1276,19 +1340,19 @@ private NestedFieldLiteralColumnIndexSupplier makeSingleTypeLongSupplier() th // column: [100, 1, 300, 1, 3, 3, 100, 300, 300, 1] // 1 - localDictionaryWriter.write(7); + localDictionaryWriter.write(10); bitmapWriter.write(fillBitmap(1, 3, 9)); // 3 - localDictionaryWriter.write(9); + localDictionaryWriter.write(12); bitmapWriter.write(fillBitmap(4, 5)); // 100 - localDictionaryWriter.write(11); + localDictionaryWriter.write(14); bitmapWriter.write(fillBitmap(0, 6)); // 300 - localDictionaryWriter.write(12); + localDictionaryWriter.write(15); bitmapWriter.write(fillBitmap(2, 7, 8)); writeToBuffer(localDictionaryBuffer, localDictionaryWriter); @@ -1351,19 +1415,19 @@ private NestedFieldLiteralColumnIndexSupplier makeSingleTypeLongSupplierWithN bitmapWriter.write(fillBitmap(2, 5, 8)); // 1 - localDictionaryWriter.write(7); + localDictionaryWriter.write(10); bitmapWriter.write(fillBitmap(1, 3, 9)); // 3 - localDictionaryWriter.write(9); + localDictionaryWriter.write(12); bitmapWriter.write(fillBitmap(4)); // 100 - localDictionaryWriter.write(11); + localDictionaryWriter.write(14); bitmapWriter.write(fillBitmap(0, 6)); // 300 - localDictionaryWriter.write(12); + localDictionaryWriter.write(15); bitmapWriter.write(fillBitmap(7)); writeToBuffer(localDictionaryBuffer, localDictionaryWriter); @@ -1422,19 +1486,19 @@ private NestedFieldLiteralColumnIndexSupplier makeSingleTypeDoubleSupplier() // column: [1.1, 1.1, 1.2, 3.3, 1.2, 6.6, 3.3, 1.2, 1.1, 3.3] // 1.1 - localDictionaryWriter.write(15); + localDictionaryWriter.write(18); bitmapWriter.write(fillBitmap(0, 1, 8)); // 1.2 - localDictionaryWriter.write(16); + localDictionaryWriter.write(19); bitmapWriter.write(fillBitmap(2, 4, 7)); // 3.3 - localDictionaryWriter.write(19); + localDictionaryWriter.write(22); bitmapWriter.write(fillBitmap(3, 6, 9)); // 6.6 - localDictionaryWriter.write(20); + localDictionaryWriter.write(23); bitmapWriter.write(fillBitmap(5)); writeToBuffer(localDictionaryBuffer, localDictionaryWriter); @@ -1497,19 +1561,19 @@ private NestedFieldLiteralColumnIndexSupplier makeSingleTypeDoubleSupplierWit bitmapWriter.write(fillBitmap(1, 3, 6)); // 1.1 - localDictionaryWriter.write(15); + localDictionaryWriter.write(18); bitmapWriter.write(fillBitmap(0, 8)); // 1.2 - localDictionaryWriter.write(16); + localDictionaryWriter.write(19); bitmapWriter.write(fillBitmap(2, 4, 7)); // 3.3 - localDictionaryWriter.write(19); + localDictionaryWriter.write(22); bitmapWriter.write(fillBitmap(9)); // 6.6 - localDictionaryWriter.write(20); + localDictionaryWriter.write(23); bitmapWriter.write(fillBitmap(5)); writeToBuffer(localDictionaryBuffer, localDictionaryWriter); @@ -1576,23 +1640,23 @@ private NestedFieldLiteralColumnIndexSupplier makeVariantSupplierWithNull() t bitmapWriter.write(fillBitmap(1, 9)); // z - localDictionaryWriter.write(6); + localDictionaryWriter.write(9); bitmapWriter.write(fillBitmap(6)); // 1 - localDictionaryWriter.write(7); + localDictionaryWriter.write(10); bitmapWriter.write(fillBitmap(0, 5)); // 300 - localDictionaryWriter.write(12); + localDictionaryWriter.write(15); bitmapWriter.write(fillBitmap(4)); // 1.1 - localDictionaryWriter.write(15); + localDictionaryWriter.write(18); bitmapWriter.write(fillBitmap(8)); // 9.9 - localDictionaryWriter.write(21); + localDictionaryWriter.write(24); bitmapWriter.write(fillBitmap(3)); writeToBuffer(localDictionaryBuffer, localDictionaryWriter);