From 2aad66bef1539c7a75ae1d596f37c3cca9d48a7b Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 1 Sep 2023 06:11:19 +0000 Subject: [PATCH 01/16] add config --- codestyle/checkstyle.xml | 4 ++++ pom.xml | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/codestyle/checkstyle.xml b/codestyle/checkstyle.xml index da70ddfad3ae..a39501b6ccbb 100644 --- a/codestyle/checkstyle.xml +++ b/codestyle/checkstyle.xml @@ -52,6 +52,10 @@ + + + + diff --git a/pom.xml b/pom.xml index b7ebfa723013..35d705df2e14 100644 --- a/pom.xml +++ b/pom.xml @@ -1373,6 +1373,11 @@ checkstyle 8.21 + + hu.rxd + checkstyle-plugins + 0.0.3.dirty + From d4fe4863d0c122db3a84e9ad55daf5e07723b309 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 1 Sep 2023 07:41:51 +0000 Subject: [PATCH 02/16] update --- codestyle/checkstyle.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/codestyle/checkstyle.xml b/codestyle/checkstyle.xml index a39501b6ccbb..9d81ddd8c240 100644 --- a/codestyle/checkstyle.xml +++ b/codestyle/checkstyle.xml @@ -53,7 +53,7 @@ - + diff --git a/pom.xml b/pom.xml index 35d705df2e14..9377894529a4 100644 --- a/pom.xml +++ b/pom.xml @@ -1376,7 +1376,7 @@ hu.rxd checkstyle-plugins - 0.0.3.dirty + 0.0.3-2-gca65563.dirty From 29d35140e55c18954d6095942c1edfa0e3bb1ed9 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 1 Sep 2023 07:43:44 +0000 Subject: [PATCH 03/16] fix one --- .../druid/jackson/DefaultObjectMapper.java | 9 ++++-- .../ClockDriftSafeMonitorSchedulerTest.java | 31 ++++++++++--------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/jackson/DefaultObjectMapper.java b/processing/src/main/java/org/apache/druid/jackson/DefaultObjectMapper.java index 7e530836c2f7..b27cd9c07f3f 100644 --- a/processing/src/main/java/org/apache/druid/jackson/DefaultObjectMapper.java +++ b/processing/src/main/java/org/apache/druid/jackson/DefaultObjectMapper.java @@ -118,10 +118,13 @@ public JavaType handleUnknownTypeId(DeserializationContext ctxt, throws IOException { String serviceMsg = (serviceName == null) ? "" : StringUtils.format(" on '%s' service", serviceName); - String msg = StringUtils.format("Please make sure to load all the necessary extensions and jars " + - "with type '%s'%s. " + + String msg = StringUtils.format( + "Please make sure to load all the necessary extensions and jars with type '%s'%s. " + "Could not resolve type id '%s' as a subtype of %s", - subTypeId, serviceMsg, subTypeId, ClassUtil.getTypeDescription(baseType)); + subTypeId, + serviceMsg, + subTypeId, + ClassUtil.getTypeDescription(baseType)); String extraFailureMsg = (failureMsg == null) ? msg : msg + " " + failureMsg; throw InvalidTypeIdException.from(ctxt.getParser(), extraFailureMsg, baseType, subTypeId); } diff --git a/processing/src/test/java/org/apache/druid/java/util/metrics/ClockDriftSafeMonitorSchedulerTest.java b/processing/src/test/java/org/apache/druid/java/util/metrics/ClockDriftSafeMonitorSchedulerTest.java index 712fcbfbb62e..2886eb88650c 100644 --- a/processing/src/test/java/org/apache/druid/java/util/metrics/ClockDriftSafeMonitorSchedulerTest.java +++ b/processing/src/test/java/org/apache/druid/java/util/metrics/ClockDriftSafeMonitorSchedulerTest.java @@ -73,7 +73,7 @@ public void tearDown() throws Exception cronTaskRunner.shutdownNow(); mocks.close(); } - + @Test public void testFindMonitor() { @@ -89,7 +89,7 @@ class Monitor3 extends NoopMonitor final Monitor1 monitor1 = new Monitor1(); final Monitor2 monitor2 = new Monitor2(); - + ExecutorService executor = Mockito.mock(ExecutorService.class); final MonitorScheduler scheduler = new ClockDriftSafeMonitorScheduler( @@ -109,7 +109,7 @@ class Monitor3 extends NoopMonitor Assert.assertFalse(scheduler.findMonitor(Monitor3.class).isPresent()); } - + @Test public void testStart_RepeatScheduling() throws InterruptedException { @@ -174,7 +174,7 @@ public Future answer(InvocationOnMock invocation) throws Exception Mockito.verify(monitor, Mockito.times(2)).monitor(ArgumentMatchers.any()); scheduler.stop(); } - + @Test public void testStart_RepeatAndStopScheduling() throws InterruptedException { @@ -231,15 +231,18 @@ public Future answer(InvocationOnMock invocation) throws Exception latch.await(5, TimeUnit.SECONDS); Mockito.verify(monitor, Mockito.times(1)).start(); - Mockito.verify(cronScheduler, Mockito.times(1)).scheduleAtFixedRate(ArgumentMatchers.anyLong(), - ArgumentMatchers.anyLong(), - ArgumentMatchers.any(), ArgumentMatchers.any(CronTask.class)); + Mockito.verify(cronScheduler, Mockito.times(1)) + .scheduleAtFixedRate( + ArgumentMatchers.anyLong(), + ArgumentMatchers.anyLong(), + ArgumentMatchers.any(), + ArgumentMatchers.any(CronTask.class)); Mockito.verify(executor, Mockito.times(1)).submit(ArgumentMatchers.any(Callable.class)); Mockito.verify(monitor, Mockito.times(2)).monitor(ArgumentMatchers.any()); Mockito.verify(monitor, Mockito.times(1)).stop(); scheduler.stop(); } - + @Test public void testStart_UnexpectedExceptionWhileMonitoring() throws InterruptedException { @@ -284,7 +287,7 @@ public Future answer(InvocationOnMock invocation) throws Exception }).when(cronScheduler).scheduleAtFixedRate(ArgumentMatchers.anyLong(), ArgumentMatchers.anyLong(), ArgumentMatchers.any(), ArgumentMatchers.any(CronTask.class)); - + final MonitorScheduler scheduler = new ClockDriftSafeMonitorScheduler( config, Mockito.mock(ServiceEmitter.class), @@ -304,7 +307,7 @@ public Future answer(InvocationOnMock invocation) throws Exception Assert.assertTrue(monitorResultHolder.get()); scheduler.stop(); } - + @Test public void testStart_UnexpectedExceptionWhileScheduling() throws InterruptedException { @@ -335,7 +338,7 @@ public Future answer(InvocationOnMock invocation) }).when(cronScheduler).scheduleAtFixedRate(ArgumentMatchers.anyLong(), ArgumentMatchers.anyLong(), ArgumentMatchers.any(), ArgumentMatchers.any(CronTask.class)); - + final MonitorScheduler scheduler = new ClockDriftSafeMonitorScheduler( config, Mockito.mock(ServiceEmitter.class), @@ -353,7 +356,7 @@ public Future answer(InvocationOnMock invocation) Mockito.verify(executor, Mockito.times(1)).submit(ArgumentMatchers.any(Callable.class)); scheduler.stop(); } - + private Future createDummyFuture() { Future future = new Future() @@ -393,8 +396,8 @@ public Object get(long timeout, TimeUnit unit) return future; } - - + + private static class NoopMonitor implements Monitor { @Override From 87f83df9f5f5f65acf96d346439cad5ed28da449 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 1 Sep 2023 07:49:29 +0000 Subject: [PATCH 04/16] fix ClockDriftSafeMonitorSchedulerTes --- .../ClockDriftSafeMonitorSchedulerTest.java | 67 +++++++++---------- 1 file changed, 30 insertions(+), 37 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/java/util/metrics/ClockDriftSafeMonitorSchedulerTest.java b/processing/src/test/java/org/apache/druid/java/util/metrics/ClockDriftSafeMonitorSchedulerTest.java index 2886eb88650c..219557738ac2 100644 --- a/processing/src/test/java/org/apache/druid/java/util/metrics/ClockDriftSafeMonitorSchedulerTest.java +++ b/processing/src/test/java/org/apache/druid/java/util/metrics/ClockDriftSafeMonitorSchedulerTest.java @@ -20,6 +20,9 @@ package org.apache.druid.java.util.metrics; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; + import com.google.common.collect.ImmutableList; import io.timeandspace.cronscheduler.CronScheduler; import io.timeandspace.cronscheduler.CronTask; @@ -29,7 +32,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; @@ -136,7 +138,7 @@ public Future answer(InvocationOnMock invocation) throws Exception ((Callable) originalArgument).call(); return CompletableFuture.completedFuture(Boolean.TRUE); } - }).when(executor).submit(ArgumentMatchers.any(Callable.class)); + }).when(executor).submit(any(Callable.class)); cronTaskRunner.submit(() -> { while (scheduleCount < 2) { @@ -148,8 +150,8 @@ public Future answer(InvocationOnMock invocation) throws Exception }); return createDummyFuture(); } - }).when(cronScheduler).scheduleAtFixedRate(ArgumentMatchers.anyLong(), ArgumentMatchers.anyLong(), - ArgumentMatchers.any(), ArgumentMatchers.any(CronTask.class)); + }).when(cronScheduler) + .scheduleAtFixedRate(anyLong(), anyLong(), any(), any(CronTask.class)); Monitor monitor = Mockito.mock(Monitor.class); @@ -167,11 +169,10 @@ public Future answer(InvocationOnMock invocation) throws Exception latch.await(5, TimeUnit.SECONDS); Mockito.verify(monitor, Mockito.times(1)).start(); - Mockito.verify(cronScheduler, Mockito.times(1)).scheduleAtFixedRate(ArgumentMatchers.anyLong(), - ArgumentMatchers.anyLong(), - ArgumentMatchers.any(), ArgumentMatchers.any(CronTask.class)); - Mockito.verify(executor, Mockito.times(2)).submit(ArgumentMatchers.any(Callable.class)); - Mockito.verify(monitor, Mockito.times(2)).monitor(ArgumentMatchers.any()); + Mockito.verify(cronScheduler, Mockito.times(1)) + .scheduleAtFixedRate(anyLong(), anyLong(), any(), any(CronTask.class)); + Mockito.verify(executor, Mockito.times(2)).submit(any(Callable.class)); + Mockito.verify(monitor, Mockito.times(2)).monitor(any()); scheduler.stop(); } @@ -200,7 +201,7 @@ public Future answer(InvocationOnMock invocation) throws Exception ((Callable) originalArgument).call(); return CompletableFuture.completedFuture(Boolean.FALSE); } - }).when(executor).submit(ArgumentMatchers.any(Callable.class)); + }).when(executor).submit(any(Callable.class)); cronTaskRunner.submit(() -> { while (scheduleCount < 2) { @@ -212,8 +213,8 @@ public Future answer(InvocationOnMock invocation) throws Exception }); return createDummyFuture(); } - }).when(cronScheduler).scheduleAtFixedRate(ArgumentMatchers.anyLong(), ArgumentMatchers.anyLong(), - ArgumentMatchers.any(), ArgumentMatchers.any(CronTask.class)); + }).when(cronScheduler) + .scheduleAtFixedRate(anyLong(), anyLong(), any(), any(CronTask.class)); Monitor monitor = Mockito.mock(Monitor.class); @@ -232,13 +233,9 @@ public Future answer(InvocationOnMock invocation) throws Exception Mockito.verify(monitor, Mockito.times(1)).start(); Mockito.verify(cronScheduler, Mockito.times(1)) - .scheduleAtFixedRate( - ArgumentMatchers.anyLong(), - ArgumentMatchers.anyLong(), - ArgumentMatchers.any(), - ArgumentMatchers.any(CronTask.class)); - Mockito.verify(executor, Mockito.times(1)).submit(ArgumentMatchers.any(Callable.class)); - Mockito.verify(monitor, Mockito.times(2)).monitor(ArgumentMatchers.any()); + .scheduleAtFixedRate(anyLong(), anyLong(), any(), any(CronTask.class)); + Mockito.verify(executor, Mockito.times(1)).submit(any(Callable.class)); + Mockito.verify(monitor, Mockito.times(2)).monitor(any()); Mockito.verify(monitor, Mockito.times(1)).stop(); scheduler.stop(); } @@ -248,7 +245,7 @@ public void testStart_UnexpectedExceptionWhileMonitoring() throws InterruptedExc { ExecutorService executor = Mockito.mock(ExecutorService.class); Monitor monitor = Mockito.mock(Monitor.class); - Mockito.when(monitor.monitor(ArgumentMatchers.any(ServiceEmitter.class))) + Mockito.when(monitor.monitor(any(ServiceEmitter.class))) .thenThrow(new RuntimeException("Test throwing exception while monitoring")); DruidMonitorSchedulerConfig config = Mockito.mock(DruidMonitorSchedulerConfig.class); @@ -275,7 +272,7 @@ public Future answer(InvocationOnMock invocation) throws Exception monitorResultHolder.set(continueMonitor); return CompletableFuture.completedFuture(continueMonitor); } - }).when(executor).submit(ArgumentMatchers.any(Callable.class)); + }).when(executor).submit(any(Callable.class)); cronTaskRunner.submit(() -> { task.run(123L); @@ -284,9 +281,8 @@ public Future answer(InvocationOnMock invocation) throws Exception }); return createDummyFuture(); } - }).when(cronScheduler).scheduleAtFixedRate(ArgumentMatchers.anyLong(), ArgumentMatchers.anyLong(), - ArgumentMatchers.any(), ArgumentMatchers.any(CronTask.class)); - + }).when(cronScheduler) + .scheduleAtFixedRate(anyLong(), anyLong(), any(), any(CronTask.class)); final MonitorScheduler scheduler = new ClockDriftSafeMonitorScheduler( config, @@ -299,11 +295,10 @@ public Future answer(InvocationOnMock invocation) throws Exception latch.await(5, TimeUnit.SECONDS); Mockito.verify(monitor, Mockito.times(1)).start(); - Mockito.verify(cronScheduler, Mockito.times(1)).scheduleAtFixedRate(ArgumentMatchers.anyLong(), - ArgumentMatchers.anyLong(), - ArgumentMatchers.any(), ArgumentMatchers.any(CronTask.class)); - Mockito.verify(executor, Mockito.times(1)).submit(ArgumentMatchers.any(Callable.class)); - Mockito.verify(monitor, Mockito.times(1)).monitor(ArgumentMatchers.any()); + Mockito.verify(cronScheduler, Mockito.times(1)) + .scheduleAtFixedRate(anyLong(), anyLong(), any(), any(CronTask.class)); + Mockito.verify(executor, Mockito.times(1)).submit(any(Callable.class)); + Mockito.verify(monitor, Mockito.times(1)).monitor(any()); Assert.assertTrue(monitorResultHolder.get()); scheduler.stop(); } @@ -326,7 +321,7 @@ public Future answer(InvocationOnMock invocation) final Object originalArgument = (invocation.getArguments())[3]; CronTask task = ((CronTask) originalArgument); - Mockito.when(executor.submit(ArgumentMatchers.any(Callable.class))) + Mockito.when(executor.submit(any(Callable.class))) .thenThrow(new RuntimeException("Test throwing exception while scheduling")); cronTaskRunner.submit(() -> { task.run(123L); @@ -335,9 +330,8 @@ public Future answer(InvocationOnMock invocation) }); return createDummyFuture(); } - }).when(cronScheduler).scheduleAtFixedRate(ArgumentMatchers.anyLong(), ArgumentMatchers.anyLong(), - ArgumentMatchers.any(), ArgumentMatchers.any(CronTask.class)); - + }).when(cronScheduler) + .scheduleAtFixedRate(anyLong(), anyLong(), any(), any(CronTask.class)); final MonitorScheduler scheduler = new ClockDriftSafeMonitorScheduler( config, @@ -350,10 +344,9 @@ public Future answer(InvocationOnMock invocation) latch.await(5, TimeUnit.SECONDS); Mockito.verify(monitor, Mockito.times(1)).start(); - Mockito.verify(cronScheduler, Mockito.times(1)).scheduleAtFixedRate(ArgumentMatchers.anyLong(), - ArgumentMatchers.anyLong(), - ArgumentMatchers.any(), ArgumentMatchers.any(CronTask.class)); - Mockito.verify(executor, Mockito.times(1)).submit(ArgumentMatchers.any(Callable.class)); + Mockito.verify(cronScheduler, Mockito.times(1)) + .scheduleAtFixedRate(anyLong(), anyLong(), any(), any(CronTask.class)); + Mockito.verify(executor, Mockito.times(1)).submit(any(Callable.class)); scheduler.stop(); } From 7354f78a4a025da76d9c9faa452b8a4581d21fcc Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 1 Sep 2023 07:54:16 +0000 Subject: [PATCH 05/16] fix some --- .../druid/java/util/common/io/NativeIO.java | 6 ++---- .../hll/HyperLogLogCollectorBenchmark.java | 18 ++++++++++-------- .../query/groupby/GroupByQueryRunnerTest.java | 19 ++++++++++++------- .../join/JoinConditionAnalysisTest.java | 9 +++++++-- 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/java/util/common/io/NativeIO.java b/processing/src/main/java/org/apache/druid/java/util/common/io/NativeIO.java index 367fe28004dc..a7dd577f3d46 100644 --- a/processing/src/main/java/org/apache/druid/java/util/common/io/NativeIO.java +++ b/processing/src/main/java/org/apache/druid/java/util/common/io/NativeIO.java @@ -146,15 +146,13 @@ public static void trySkipCache(int fd, long offset, long len) catch (UnsatisfiedLinkError ule) { // if JNA is unavailable just skipping Direct I/O // instance of this class will act like normal RandomAccessFile - log.warn(ule, "Unsatisfied Link error: posix_fadvise failed on file descriptor [%d], offset [%d]", - fd, offset); + log.warn(ule, "Unsatisfied Link error: posix_fadvise failed on file descriptor [%d], offset [%d]", fd, offset); fadvisePossible = false; } catch (Exception e) { // This is best effort anyway so lets just log that there was an // exception and forget - log.warn(e, "Unknown exception: posix_fadvise failed on file descriptor [%d], offset [%d]", - fd, offset); + log.warn(e, "Unknown exception: posix_fadvise failed on file descriptor [%d], offset [%d]", fd, offset); } } diff --git a/processing/src/test/java/org/apache/druid/hll/HyperLogLogCollectorBenchmark.java b/processing/src/test/java/org/apache/druid/hll/HyperLogLogCollectorBenchmark.java index 62446a6eaac2..c432175afac3 100644 --- a/processing/src/test/java/org/apache/druid/hll/HyperLogLogCollectorBenchmark.java +++ b/processing/src/test/java/org/apache/druid/hll/HyperLogLogCollectorBenchmark.java @@ -159,12 +159,12 @@ public double timeFold(int reps) final int size = sizes[i]; HyperLogLogCollector.makeCollector( - (ByteBuffer) buf.duplicate().position(0).limit( + buf.duplicate().position(0).limit( HyperLogLogCollector.getLatestNumBytesForDenseStorage() ) ).fold( HyperLogLogCollector.makeCollector( - (ByteBuffer) chunk.duplicate().limit(pos + size).position(pos) + chunk.duplicate().limit(pos + size).position(pos) ) ); } @@ -196,17 +196,19 @@ class ByteBuffers private static long lookupAddressOffset(MethodHandles.Lookup lookup) throws Throwable { - MethodHandle objectFieldOffset = lookup.findVirtual(UnsafeUtils.theUnsafeClass(), "objectFieldOffset", - MethodType.methodType(long.class, Field.class) - ); + MethodHandle objectFieldOffset = lookup.findVirtual( + UnsafeUtils.theUnsafeClass(), + "objectFieldOffset", + MethodType.methodType(long.class, Field.class)); return (long) objectFieldOffset.bindTo(UnsafeUtils.theUnsafe()).invoke(Buffer.class.getDeclaredField("address")); } private static MethodHandle lookupGetLong(MethodHandles.Lookup lookup) throws Throwable { - MethodHandle getLong = lookup.findVirtual(UnsafeUtils.theUnsafeClass(), "getLong", - MethodType.methodType(long.class, Object.class, long.class) - ); + MethodHandle getLong = lookup.findVirtual( + UnsafeUtils.theUnsafeClass(), + "getLong", + MethodType.methodType(long.class, Object.class, long.class)); return getLong.bindTo(UnsafeUtils.theUnsafe()); } diff --git a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java index 269067ff1f9b..599c0ae281ee 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java @@ -5119,11 +5119,14 @@ public void testDimFilterHavingSpec() .setDataSource(QueryRunnerTestHelper.DATA_SOURCE) .setInterval("2011-04-02/2011-04-04") .setDimensions(new DefaultDimensionSpec("quality", "alias")) - .setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT, new LongSumAggregatorFactory("idx", "index"), - QueryRunnerTestHelper.INDEX_LONG_MIN, QueryRunnerTestHelper.INDEX_LONG_MAX, - QueryRunnerTestHelper.INDEX_DOUBLE_MIN, QueryRunnerTestHelper.INDEX_DOUBLE_MAX, - QueryRunnerTestHelper.INDEX_FLOAT_MIN, QueryRunnerTestHelper.INDEX_FLOAT_MAX - ) + .setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT, + new LongSumAggregatorFactory("idx", "index"), + QueryRunnerTestHelper.INDEX_LONG_MIN, + QueryRunnerTestHelper.INDEX_LONG_MAX, + QueryRunnerTestHelper.INDEX_DOUBLE_MIN, + QueryRunnerTestHelper.INDEX_DOUBLE_MAX, + QueryRunnerTestHelper.INDEX_FLOAT_MIN, + QueryRunnerTestHelper.INDEX_FLOAT_MAX) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) .setHavingSpec(havingSpec); @@ -5727,7 +5730,8 @@ public void testDifferentGroupingSubquery() TestHelper.assertExpectedObjects( expectedResults, - GroupByQueryRunnerTestHelper.runQuery(factory, runner, query), "subquery" + GroupByQueryRunnerTestHelper.runQuery(factory, runner, query), + "subquery" ); subquery = makeQueryBuilder(subquery) @@ -5751,7 +5755,8 @@ public void testDifferentGroupingSubquery() TestHelper.assertExpectedObjects( expectedResults, - GroupByQueryRunnerTestHelper.runQuery(factory, runner, query), "subquery" + GroupByQueryRunnerTestHelper.runQuery(factory, runner, query), + "subquery" ); } diff --git a/processing/src/test/java/org/apache/druid/segment/join/JoinConditionAnalysisTest.java b/processing/src/test/java/org/apache/druid/segment/join/JoinConditionAnalysisTest.java index 1ab6c0922994..6027baf16168 100644 --- a/processing/src/test/java/org/apache/druid/segment/join/JoinConditionAnalysisTest.java +++ b/processing/src/test/java/org/apache/druid/segment/join/JoinConditionAnalysisTest.java @@ -290,9 +290,14 @@ public void test_equals() .usingGetClass() .withIgnoredFields( // These fields are tightly coupled with originalExpression - "equiConditions", "nonEquiConditions", "requiredColumns", + "equiConditions", + "nonEquiConditions", + "requiredColumns", // These fields are calculated from other other fields in the class - "isAlwaysTrue", "isAlwaysFalse", "canHashJoin", "rightKeyColumns") + "isAlwaysTrue", + "isAlwaysFalse", + "canHashJoin", + "rightKeyColumns") .verify(); } From abdf199493732a012636c7bc008f1a25aba1aacc Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 1 Sep 2023 07:58:06 +0000 Subject: [PATCH 06/16] align --- .../druid/java/util/common/ByteBufferUtils.java | 17 ++++++++++------- .../metadata/metadata/SegmentMetadataQuery.java | 8 +++++--- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/java/util/common/ByteBufferUtils.java b/processing/src/main/java/org/apache/druid/java/util/common/ByteBufferUtils.java index fb67c3238347..c1702867fd8e 100644 --- a/processing/src/main/java/org/apache/druid/java/util/common/ByteBufferUtils.java +++ b/processing/src/main/java/org/apache/druid/java/util/common/ByteBufferUtils.java @@ -121,13 +121,16 @@ private static MethodHandle unmapJava7Or8(MethodHandles.Lookup lookup) throws Re MethodHandle directBufferCleanerMethod = lookup.unreflect(m); Class cleanerClass = directBufferCleanerMethod.type().returnType(); MethodHandle cleanMethod = lookup.findVirtual(cleanerClass, "clean", MethodType.methodType(void.class)); - MethodHandle nonNullTest = lookup.findStatic(Objects.class, "nonNull", - MethodType.methodType(boolean.class, Object.class) - ).asType(MethodType.methodType(boolean.class, cleanerClass)); - MethodHandle noop = MethodHandles.dropArguments(MethodHandles.constant( - Void.class, - null - ).asType(MethodType.methodType(void.class)), 0, cleanerClass); + MethodHandle nonNullTest = lookup.findStatic( + Objects.class, + "nonNull", + MethodType.methodType(boolean.class, Object.class)) + .asType(MethodType.methodType(boolean.class, cleanerClass)); + MethodHandle noop = MethodHandles.dropArguments( + MethodHandles.constant(Void.class, null) + .asType(MethodType.methodType(void.class)), + 0, + cleanerClass); MethodHandle unmapper = MethodHandles.filterReturnValue( directBufferCleanerMethod, MethodHandles.guardWithTest(nonNullTest, cleanMethod, noop) diff --git a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java index 85c84fe38583..cbba148c851a 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java @@ -116,9 +116,11 @@ public SegmentMetadataQuery( // of truth for consumers of this class variable. The defaults are to preserve backwards compatibility. // In a future release, 28.0+, we can remove the deprecated property lenientAggregatorMerge. if (lenientAggregatorMerge != null && aggregatorMergeStrategy != null) { - throw InvalidInput.exception("Both lenientAggregatorMerge [%s] and aggregatorMergeStrategy [%s] parameters cannot be set." - + " Consider using aggregatorMergeStrategy since lenientAggregatorMerge is deprecated.", - lenientAggregatorMerge, aggregatorMergeStrategy); + throw InvalidInput.exception( + "Both lenientAggregatorMerge [%s] and aggregatorMergeStrategy [%s] parameters cannot be set." + + " Consider using aggregatorMergeStrategy since lenientAggregatorMerge is deprecated.", + lenientAggregatorMerge, + aggregatorMergeStrategy); } if (lenientAggregatorMerge != null) { this.aggregatorMergeStrategy = lenientAggregatorMerge From 30baa4f3763075249d9f5b9a8ecd49fa55b158b5 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 1 Sep 2023 08:06:38 +0000 Subject: [PATCH 07/16] fix --- .../java/org/apache/druid/java/util/common/io/NativeIO.java | 6 +++++- .../org/apache/druid/segment/data/GenericIndexedWriter.java | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/java/util/common/io/NativeIO.java b/processing/src/main/java/org/apache/druid/java/util/common/io/NativeIO.java index a7dd577f3d46..f1b9285c914b 100644 --- a/processing/src/main/java/org/apache/druid/java/util/common/io/NativeIO.java +++ b/processing/src/main/java/org/apache/druid/java/util/common/io/NativeIO.java @@ -173,7 +173,11 @@ private static void trySyncFileRange(int fd, long offset, long nbytes, int flags int ret_code = sync_file_range(fd, offset, nbytes, flags); if (ret_code != 0) { log.warn("failed on syncing fd [%d], offset [%d], bytes [%d], ret_code [%d], errno [%d]", - fd, offset, nbytes, ret_code, Native.getLastError()); + fd, + offset, + nbytes, + ret_code, + Native.getLastError()); return; } } diff --git a/processing/src/main/java/org/apache/druid/segment/data/GenericIndexedWriter.java b/processing/src/main/java/org/apache/druid/segment/data/GenericIndexedWriter.java index 394fccba06b8..4080ad28bcfe 100644 --- a/processing/src/main/java/org/apache/druid/segment/data/GenericIndexedWriter.java +++ b/processing/src/main/java/org/apache/druid/segment/data/GenericIndexedWriter.java @@ -207,7 +207,7 @@ private static void writeBytesIntoSmooshedChannel( while (numBytesToPutInFile > 0) { int bytesRead = is.read(buffer, 0, Math.min(buffer.length, Ints.saturatedCast(numBytesToPutInFile))); if (bytesRead != -1) { - smooshChannel.write((ByteBuffer) holderBuffer.clear().limit(bytesRead)); + smooshChannel.write(holderBuffer.clear().limit(bytesRead)); numBytesToPutInFile -= bytesRead; } else { throw new ISE("Could not write [%d] bytes into smooshChannel.", numBytesToPutInFile); @@ -351,7 +351,8 @@ private void writeToSingleFile(WritableByteChannel channel) throws IOException headerOut.size() ); Preconditions.checkState( - numBytesWritten < fileSizeLimit, "Wrote[%s] bytes, which is too many.", + numBytesWritten < fileSizeLimit, + "Wrote[%s] bytes, which is too many.", numBytesWritten ); From dc56e5b40c561d2873940c6c206fb72e9d7f2c3b Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 1 Sep 2023 08:18:00 +0000 Subject: [PATCH 08/16] few more --- codestyle/checkstyle.xml | 2 +- .../org/apache/druid/java/util/common/io/NativeIO.java | 8 +++++--- .../apache/druid/frame/file/FrameFileWriterTest.java | 10 ++++++---- .../druid/java/util/common/CompressionUtilsTest.java | 8 +++++--- .../metrics/ClockDriftSafeMonitorSchedulerTest.java | 7 +++---- ...rializablePairLongStringComplexMetricSerdeTest.java | 10 ++++++---- .../hyperloglog/HyperUniquesAggregatorFactoryTest.java | 10 +++++++--- .../druid/query/context/ResponseContextTest.java | 3 +-- 8 files changed, 34 insertions(+), 24 deletions(-) diff --git a/codestyle/checkstyle.xml b/codestyle/checkstyle.xml index 9d81ddd8c240..6363b951c226 100644 --- a/codestyle/checkstyle.xml +++ b/codestyle/checkstyle.xml @@ -53,7 +53,7 @@ - + diff --git a/processing/src/main/java/org/apache/druid/java/util/common/io/NativeIO.java b/processing/src/main/java/org/apache/druid/java/util/common/io/NativeIO.java index f1b9285c914b..5565e04db682 100644 --- a/processing/src/main/java/org/apache/druid/java/util/common/io/NativeIO.java +++ b/processing/src/main/java/org/apache/druid/java/util/common/io/NativeIO.java @@ -190,8 +190,7 @@ private static void trySyncFileRange(int fd, long offset, long nbytes, int flags syncFileRangePossible = false; } catch (Exception e) { - log.warn(e, "Unknown exception: sync_file_range failed on fd [%d], offset [%d], bytes [%d]", - fd, offset, nbytes); + log.warn(e, "Unknown exception: sync_file_range failed on fd [%d], offset [%d], bytes [%d]", fd, offset, nbytes); syncFileRangePossible = false; } } @@ -226,7 +225,10 @@ public static void chunkedCopy(InputStream src, File dest) throws IOException trySyncFileRange(fd, offset, numBytes, SYNC_FILE_RANGE_WRITE); if (offset > 0) { // This does a blocking write-and-wait on any old ranges - trySyncFileRange(fd, lastOffset, lastBytes, + trySyncFileRange( + fd, + lastOffset, + lastBytes, SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_AFTER); // Remove the old range from the cache trySkipCache(fd, lastOffset, lastBytes); diff --git a/processing/src/test/java/org/apache/druid/frame/file/FrameFileWriterTest.java b/processing/src/test/java/org/apache/druid/frame/file/FrameFileWriterTest.java index 5f0e019712be..64def18a2d31 100644 --- a/processing/src/test/java/org/apache/druid/frame/file/FrameFileWriterTest.java +++ b/processing/src/test/java/org/apache/druid/frame/file/FrameFileWriterTest.java @@ -55,10 +55,12 @@ public void test_abort_afterAllFrames() throws IOException .frames(); final File file = temporaryFolder.newFile(); - final FrameFileWriter fileWriter = FrameFileWriter.open(Files.newByteChannel( - file.toPath(), - StandardOpenOption.WRITE - ), null, ByteTracker.unboundedTracker()); + final FrameFileWriter fileWriter = FrameFileWriter.open( + Files.newByteChannel( + file.toPath(), + StandardOpenOption.WRITE), + null, + ByteTracker.unboundedTracker()); frames.forEach(frame -> { try { diff --git a/processing/src/test/java/org/apache/druid/java/util/common/CompressionUtilsTest.java b/processing/src/test/java/org/apache/druid/java/util/common/CompressionUtilsTest.java index f02e441573a3..df419fc71b63 100644 --- a/processing/src/test/java/org/apache/druid/java/util/common/CompressionUtilsTest.java +++ b/processing/src/test/java/org/apache/druid/java/util/common/CompressionUtilsTest.java @@ -602,7 +602,7 @@ public InputStream openStream() } }; - Assert.assertEquals((long) (EXPECTED.length * 3), CompressionUtils.gunzip(inputStreamFactory, testFile).size()); + Assert.assertEquals(EXPECTED.length * 3, CompressionUtils.gunzip(inputStreamFactory, testFile).size()); try (final InputStream inputStream = new FileInputStream(testFile)) { try (final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(EXPECTED.length * 3)) { @@ -712,7 +712,8 @@ public void testGoodGzipWithException() throws Exception final File gzFile = new File(tmpDir, testFile.getName() + ".gz"); Assert.assertFalse(gzFile.exists()); CompressionUtils.gzip( - Files.asByteSource(testFile), new ByteSink() + Files.asByteSource(testFile), + new ByteSink() { @Override public OutputStream openStream() throws IOException @@ -730,7 +731,8 @@ public void flush() throws IOException } }; } - }, Predicates.alwaysTrue() + }, + Predicates.alwaysTrue() ); Assert.assertTrue(gzFile.exists()); try (final InputStream inputStream = CompressionUtils.decompress(new FileInputStream(gzFile), "file.gz")) { diff --git a/processing/src/test/java/org/apache/druid/java/util/metrics/ClockDriftSafeMonitorSchedulerTest.java b/processing/src/test/java/org/apache/druid/java/util/metrics/ClockDriftSafeMonitorSchedulerTest.java index 219557738ac2..9bae6ea00d3b 100644 --- a/processing/src/test/java/org/apache/druid/java/util/metrics/ClockDriftSafeMonitorSchedulerTest.java +++ b/processing/src/test/java/org/apache/druid/java/util/metrics/ClockDriftSafeMonitorSchedulerTest.java @@ -19,10 +19,6 @@ package org.apache.druid.java.util.metrics; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; - import com.google.common.collect.ImmutableList; import io.timeandspace.cronscheduler.CronScheduler; import io.timeandspace.cronscheduler.CronTask; @@ -48,6 +44,9 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; + public class ClockDriftSafeMonitorSchedulerTest { // A real executor service to execute CronTask asynchronously. diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexMetricSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexMetricSerdeTest.java index f503874a4d6b..48bffa2568b6 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexMetricSerdeTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexMetricSerdeTest.java @@ -70,10 +70,12 @@ public void testSingle() throws Exception public void testLargeString() throws Exception { // single entry spans more than one block in underlying storage - assertExpected(ImmutableList.of(new SerializablePairLongString( - 100L, - randomStringUtils.randomAlphanumeric(2 * 1024 * 1024) - )), 2097182, 2103140); + assertExpected( + ImmutableList.of(new SerializablePairLongString( + 100L, + randomStringUtils.randomAlphanumeric(2 * 1024 * 1024))), + 2097182, + 2103140); } @Test diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactoryTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactoryTest.java index 421a457999d9..31aa5af98625 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactoryTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactoryTest.java @@ -164,9 +164,13 @@ public void testCompareToShouldBehaveConsistentlyWithEstimatedCardinalitiesEvenI Assert.assertEquals( StringUtils.format("orderedByComparator=%d, orderedByCardinality=%d,\n" + "Left={cardinality=%f, hll=%s},\n" + - "Right={cardinality=%f, hll=%s},\n", orderedByComparator, orderedByCardinality, - leftCollector.estimateCardinality(), leftCollector, - rightCollector.estimateCardinality(), rightCollector + "Right={cardinality=%f, hll=%s},\n", + orderedByComparator, + orderedByCardinality, + leftCollector.estimateCardinality(), + leftCollector, + rightCollector.estimateCardinality(), + rightCollector ), orderedByCardinality, orderedByComparator diff --git a/processing/src/test/java/org/apache/druid/query/context/ResponseContextTest.java b/processing/src/test/java/org/apache/druid/query/context/ResponseContextTest.java index 03ed376f9385..94b6ab478970 100644 --- a/processing/src/test/java/org/apache/druid/query/context/ResponseContextTest.java +++ b/processing/src/test/java/org/apache/druid/query/context/ResponseContextTest.java @@ -310,8 +310,7 @@ public void serializeWithTruncateArrayTest() throws IOException final ResponseContext ctx = ResponseContext.createEmpty(); ctx.put( Keys.UNCOVERED_INTERVALS, - Arrays.asList(interval(1), interval(2), interval(3), interval(4), - interval(5), interval(6)) + Arrays.asList(interval(1), interval(2), interval(3), interval(4), interval(5), interval(6)) ); // This value should be longer than the above so it is fully removed // before we truncate the above. From bbe444e54c591a88c9dd246867768657d5df6f09 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 1 Sep 2023 08:30:52 +0000 Subject: [PATCH 09/16] use 0.0.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9377894529a4..c49921307de7 100644 --- a/pom.xml +++ b/pom.xml @@ -1376,7 +1376,7 @@ hu.rxd checkstyle-plugins - 0.0.3-2-gca65563.dirty + 0.0.4 From 6b9e8ba2dbd3ac5fc5b607a5dd914a679d3cc097 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 1 Sep 2023 11:56:05 +0000 Subject: [PATCH 10/16] putback cast --- .../org/apache/druid/segment/data/GenericIndexedWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/segment/data/GenericIndexedWriter.java b/processing/src/main/java/org/apache/druid/segment/data/GenericIndexedWriter.java index 4080ad28bcfe..f18ca47d50b6 100644 --- a/processing/src/main/java/org/apache/druid/segment/data/GenericIndexedWriter.java +++ b/processing/src/main/java/org/apache/druid/segment/data/GenericIndexedWriter.java @@ -207,7 +207,7 @@ private static void writeBytesIntoSmooshedChannel( while (numBytesToPutInFile > 0) { int bytesRead = is.read(buffer, 0, Math.min(buffer.length, Ints.saturatedCast(numBytesToPutInFile))); if (bytesRead != -1) { - smooshChannel.write(holderBuffer.clear().limit(bytesRead)); + smooshChannel.write((ByteBuffer) holderBuffer.clear().limit(bytesRead)); numBytesToPutInFile -= bytesRead; } else { throw new ISE("Could not write [%d] bytes into smooshChannel.", numBytesToPutInFile); From e568bb33f58302a0b595da591d8ce37fa895e449 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 1 Sep 2023 11:57:34 +0000 Subject: [PATCH 11/16] putback cast --- .../org/apache/druid/hll/HyperLogLogCollectorBenchmark.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/hll/HyperLogLogCollectorBenchmark.java b/processing/src/test/java/org/apache/druid/hll/HyperLogLogCollectorBenchmark.java index c432175afac3..a04740a6027d 100644 --- a/processing/src/test/java/org/apache/druid/hll/HyperLogLogCollectorBenchmark.java +++ b/processing/src/test/java/org/apache/druid/hll/HyperLogLogCollectorBenchmark.java @@ -159,12 +159,12 @@ public double timeFold(int reps) final int size = sizes[i]; HyperLogLogCollector.makeCollector( - buf.duplicate().position(0).limit( + (ByteBuffer) buf.duplicate().position(0).limit( HyperLogLogCollector.getLatestNumBytesForDenseStorage() ) ).fold( HyperLogLogCollector.makeCollector( - chunk.duplicate().limit(pos + size).position(pos) + (ByteBuffer) chunk.duplicate().limit(pos + size).position(pos) ) ); } From b6355a7f59153c361151c6d7e4ff22379a991372 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 1 Sep 2023 12:47:36 +0000 Subject: [PATCH 12/16] disable hdr check --- codestyle/checkstyle.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codestyle/checkstyle.xml b/codestyle/checkstyle.xml index 6363b951c226..3426961e9a5d 100644 --- a/codestyle/checkstyle.xml +++ b/codestyle/checkstyle.xml @@ -24,11 +24,12 @@ "http://www.puppycrawl.com/dtds/configuration_1_2.dtd"> + From f92e3c48fea3089f4ccbd5dd6f3da45321baff8a Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 1 Sep 2023 13:14:39 +0000 Subject: [PATCH 13/16] checkstyle fixes --- .../common/aws/AWSCredentialsConfigTest.java | 1 + .../bloom/BaseBloomFilterAggregator.java | 1 + .../expressions/BloomFilterExpressions.java | 1 + .../sql/calcite/aggregation/Aggregation.java | 1 + .../sql/calcite/aggregation/Aggregations.java | 1 + .../ApproxCountDistinctSqlAggregator.java | 1 + .../builtin/ArrayConcatSqlAggregator.java | 1 + .../builtin/ArraySqlAggregator.java | 1 + .../aggregation/builtin/AvgSqlAggregator.java | 1 + .../ArrayContainsOperatorConversion.java | 1 + .../ArrayOverlapOperatorConversion.java | 1 + .../SchemaAwareUserDefinedTableMacro.java | 8 +-- .../MoveTimeFiltersToIntervals.java | 12 ++-- .../sql/calcite/planner/CalcitePlanner.java | 7 ++- .../sql/calcite/planner/IngestHandler.java | 3 +- .../druid/sql/calcite/rel/Windowing.java | 3 +- .../FilterJoinExcludePushToChildRule.java | 7 ++- .../DruidAggregateCaseToFilterRule.java | 60 +++++++++++++------ .../schema/AvailableSegmentMetadata.java | 1 + .../sql/calcite/schema/InformationSchema.java | 2 +- .../druid/sql/http/ArrayLinesWriter.java | 1 + .../apache/druid/sql/http/ArrayWriter.java | 1 + .../sql/calcite/CalciteArraysQueryTest.java | 6 +- 23 files changed, 88 insertions(+), 34 deletions(-) diff --git a/cloud/aws-common/src/test/java/org/apache/druid/common/aws/AWSCredentialsConfigTest.java b/cloud/aws-common/src/test/java/org/apache/druid/common/aws/AWSCredentialsConfigTest.java index 522ef1c0cd7c..ea66137e2efe 100644 --- a/cloud/aws-common/src/test/java/org/apache/druid/common/aws/AWSCredentialsConfigTest.java +++ b/cloud/aws-common/src/test/java/org/apache/druid/common/aws/AWSCredentialsConfigTest.java @@ -35,6 +35,7 @@ import javax.validation.Validation; import javax.validation.Validator; + import java.util.Properties; import java.util.UUID; diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BaseBloomFilterAggregator.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BaseBloomFilterAggregator.java index 8bc4bd323adc..7bf51f741d71 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BaseBloomFilterAggregator.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BaseBloomFilterAggregator.java @@ -25,6 +25,7 @@ import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import javax.annotation.Nullable; + import java.nio.ByteBuffer; /** diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/expressions/BloomFilterExpressions.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/expressions/BloomFilterExpressions.java index 9461a3635da8..1e497ec7f817 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/expressions/BloomFilterExpressions.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/expressions/BloomFilterExpressions.java @@ -32,6 +32,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; + import java.io.IOException; import java.util.List; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/Aggregation.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/Aggregation.java index 61cc58e12c03..b83a6001b858 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/Aggregation.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/Aggregation.java @@ -34,6 +34,7 @@ import org.apache.druid.sql.calcite.rel.VirtualColumnRegistry; import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/Aggregations.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/Aggregations.java index 3a3e43dd7b8a..9ab025d4f2fa 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/Aggregations.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/Aggregations.java @@ -30,6 +30,7 @@ import org.apache.druid.sql.calcite.planner.PlannerContext; import javax.annotation.Nullable; + import java.util.List; import java.util.Objects; import java.util.function.Function; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/ApproxCountDistinctSqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/ApproxCountDistinctSqlAggregator.java index 0ff7972657ea..9897439bd508 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/ApproxCountDistinctSqlAggregator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/ApproxCountDistinctSqlAggregator.java @@ -35,6 +35,7 @@ import org.apache.druid.sql.calcite.rel.VirtualColumnRegistry; import javax.annotation.Nullable; + import java.util.List; /** diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ArrayConcatSqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ArrayConcatSqlAggregator.java index ed6652181eb4..a64d7ec6c5a6 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ArrayConcatSqlAggregator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ArrayConcatSqlAggregator.java @@ -49,6 +49,7 @@ import org.apache.druid.sql.calcite.rel.VirtualColumnRegistry; import javax.annotation.Nullable; + import java.util.List; import java.util.stream.Collectors; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ArraySqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ArraySqlAggregator.java index 5136ed3c947b..8529e0b78523 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ArraySqlAggregator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ArraySqlAggregator.java @@ -51,6 +51,7 @@ import org.apache.druid.sql.calcite.rel.VirtualColumnRegistry; import javax.annotation.Nullable; + import java.util.List; import java.util.stream.Collectors; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/AvgSqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/AvgSqlAggregator.java index a938bdca0b84..977073ca5948 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/AvgSqlAggregator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/AvgSqlAggregator.java @@ -44,6 +44,7 @@ import org.apache.druid.sql.calcite.rel.VirtualColumnRegistry; import javax.annotation.Nullable; + import java.util.List; public class AvgSqlAggregator implements SqlAggregator diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayContainsOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayContainsOperatorConversion.java index cffee832535f..6063d63ee38c 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayContainsOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayContainsOperatorConversion.java @@ -41,6 +41,7 @@ import org.apache.druid.sql.calcite.rel.VirtualColumnRegistry; import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.List; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOverlapOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOverlapOperatorConversion.java index 8e09ea0c7340..2905ae80e7c4 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOverlapOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOverlapOperatorConversion.java @@ -41,6 +41,7 @@ import org.apache.druid.sql.calcite.rel.VirtualColumnRegistry; import javax.annotation.Nullable; + import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/external/SchemaAwareUserDefinedTableMacro.java b/sql/src/main/java/org/apache/druid/sql/calcite/external/SchemaAwareUserDefinedTableMacro.java index b5b0c3928332..be1ca2605662 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/external/SchemaAwareUserDefinedTableMacro.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/external/SchemaAwareUserDefinedTableMacro.java @@ -168,10 +168,10 @@ public Set computeResources(final SqlCall call, final boolean in Set resourceActions = new HashSet<>(); if (table instanceof ExternalTable && inputSourceTypeSecurityEnabled) { resourceActions.addAll(((ExternalTable) table) - .getInputSourceTypeSupplier().get().stream() - .map(inputSourceType -> - new ResourceAction(new Resource(inputSourceType, ResourceType.EXTERNAL), Action.READ)) - .collect(Collectors.toSet())); + .getInputSourceTypeSupplier().get().stream() + .map(inputSourceType -> + new ResourceAction(new Resource(inputSourceType, ResourceType.EXTERNAL), Action.READ)) + .collect(Collectors.toSet())); } else { resourceActions.addAll(base.computeResources(call, inputSourceTypeSecurityEnabled)); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/MoveTimeFiltersToIntervals.java b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/MoveTimeFiltersToIntervals.java index 1bee138c97a1..2262f41c527d 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/MoveTimeFiltersToIntervals.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/MoveTimeFiltersToIntervals.java @@ -179,8 +179,10 @@ private static Range toLongRange(final Range range) return Range.downTo(Long.parseLong(range.lowerEndpoint().getValue()), range.lowerBoundType()); } else { return Range.range( - Long.parseLong(range.lowerEndpoint().getValue()), range.lowerBoundType(), - Long.parseLong(range.upperEndpoint().getValue()), range.upperBoundType() + Long.parseLong(range.lowerEndpoint().getValue()), + range.lowerBoundType(), + Long.parseLong(range.upperEndpoint().getValue()), + range.upperBoundType() ); } } @@ -195,8 +197,10 @@ private static Range toLongRangeFromRange(final Range range) return Range.downTo(Numbers.parseLong(range.lowerEndpoint().getValue()), range.lowerBoundType()); } else { return Range.range( - Numbers.parseLong(range.lowerEndpoint().getValue()), range.lowerBoundType(), - Numbers.parseLong(range.upperEndpoint().getValue()), range.upperBoundType() + Numbers.parseLong(range.lowerEndpoint().getValue()), + range.lowerBoundType(), + Numbers.parseLong(range.upperEndpoint().getValue()), + range.upperBoundType() ); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlanner.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlanner.java index 1abec772e313..17dc6a34f2d6 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlanner.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlanner.java @@ -330,8 +330,11 @@ public RelRoot expandView( @Nullable List viewPath ) { - return CalcitePlanner.this.expandView(rowType, queryString, schemaPath, - viewPath + return CalcitePlanner.this.expandView( + rowType, + queryString, + schemaPath, + viewPath ); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/IngestHandler.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/IngestHandler.java index 3d38c6b3f2c1..c7708384b336 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/IngestHandler.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/IngestHandler.java @@ -163,7 +163,8 @@ private String validateAndGetDataSourceForIngest() throw InvalidSqlInput.exception( "Operation [%s] cannot be run with a target column list, given [%s (%s)]", operationName(), - insert.getTargetTable(), insert.getTargetColumnList() + insert.getTargetTable(), + insert.getTargetColumnList() ); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/Windowing.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/Windowing.java index e486a9588123..98aff17b1c6d 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/Windowing.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/Windowing.java @@ -237,7 +237,8 @@ public static Windowing fromCalciteStuff( // as defensive programming. final Mappings.TargetMapping mapping = Preconditions.checkNotNull( partialQuery.getWindowProject().getMapping(), - "mapping for windowProject[%s]", partialQuery.getWindowProject() + "mapping for windowProject[%s]", + partialQuery.getWindowProject() ); final List windowProjectOutputColumns = new ArrayList<>(); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/FilterJoinExcludePushToChildRule.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/FilterJoinExcludePushToChildRule.java index 4432b2824383..2a3613538afb 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rule/FilterJoinExcludePushToChildRule.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/FilterJoinExcludePushToChildRule.java @@ -226,8 +226,11 @@ protected void perform(RelOptRuleCall call, Filter filter, Join join) // create a FilterRel on top of the join if needed relBuilder.filter( - RexUtil.fixUp(rexBuilder, aboveFilters, - RelOptUtil.getFieldTypeList(relBuilder.peek().getRowType()))); + RexUtil.fixUp( + rexBuilder, + aboveFilters, + RelOptUtil.getFieldTypeList(relBuilder.peek().getRowType()) + )); call.transformTo(relBuilder.build()); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidAggregateCaseToFilterRule.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidAggregateCaseToFilterRule.java index 700f108f2902..ac5e6150a3e7 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidAggregateCaseToFilterRule.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidAggregateCaseToFilterRule.java @@ -204,8 +204,10 @@ private AggregateCall transform(AggregateCall aggregateCall, Project project, Li // latter is present. final RexNode filter; if (aggregateCall.filterArg >= 0) { - filter = rexBuilder.makeCall(SqlStdOperatorTable.AND, - project.getProjects().get(aggregateCall.filterArg), filterFromCase + filter = rexBuilder.makeCall( + SqlStdOperatorTable.AND, + project.getProjects().get(aggregateCall.filterArg), + filterFromCase ); } else { filter = filterFromCase; @@ -222,10 +224,16 @@ private AggregateCall transform(AggregateCall aggregateCall, Project project, Li && RexLiteral.isNullLiteral(arg2)) { newProjects.add(arg1); newProjects.add(filter); - return AggregateCall.create(SqlStdOperatorTable.COUNT, true, false, - false, ImmutableList.of(newProjects.size() - 2), - newProjects.size() - 1, RelCollations.EMPTY, - aggregateCall.getType(), aggregateCall.getName() + return AggregateCall.create( + SqlStdOperatorTable.COUNT, + true, + false, + false, + ImmutableList.of(newProjects.size() - 2), + newProjects.size() - 1, + RelCollations.EMPTY, + aggregateCall.getType(), + aggregateCall.getName() ); } newProjects.add(rexNode); @@ -263,10 +271,15 @@ private AggregateCall transform(AggregateCall aggregateCall, Project project, Li && !RexLiteral.isNullLiteral(arg1) && RexLiteral.isNullLiteral(arg2)) { newProjects.add(filter); - return AggregateCall.create(SqlStdOperatorTable.COUNT, false, false, - false, ImmutableList.of(), newProjects.size() - 1, - RelCollations.EMPTY, aggregateCall.getType(), - aggregateCall.getName() + return AggregateCall.create(SqlStdOperatorTable.COUNT, + false, + false, + false, + ImmutableList.of(), + newProjects.size() - 1, + RelCollations.EMPTY, + aggregateCall.getType(), + aggregateCall.getName() ); } else if (kind == SqlKind.SUM // Case B && isIntLiteral(arg1) && RexLiteral.intValue(arg1) == 1 @@ -277,9 +290,16 @@ && isIntLiteral(arg2) && RexLiteral.intValue(arg2) == 0) { final RelDataType dataType = typeFactory.createTypeWithNullability( typeFactory.createSqlType(SqlTypeName.BIGINT), false); - return AggregateCall.create(SqlStdOperatorTable.COUNT, false, false, - false, ImmutableList.of(), newProjects.size() - 1, - RelCollations.EMPTY, dataType, aggregateCall.getName() + return AggregateCall.create( + SqlStdOperatorTable.COUNT, + false, + false, + false, + ImmutableList.of(), + newProjects.size() - 1, + RelCollations.EMPTY, + dataType, + aggregateCall.getName() ); } else if ((RexLiteral.isNullLiteral(arg2) // Case A1 && aggregateCall.getAggregation().allowsFilter()) @@ -288,10 +308,16 @@ && isIntLiteral(arg2) && RexLiteral.intValue(arg2) == 0)) { newProjects.add(arg1); newProjects.add(filter); - return AggregateCall.create(aggregateCall.getAggregation(), false, - false, false, ImmutableList.of(newProjects.size() - 2), - newProjects.size() - 1, RelCollations.EMPTY, - aggregateCall.getType(), aggregateCall.getName() + return AggregateCall.create( + aggregateCall.getAggregation(), + false, + false, + false, + ImmutableList.of(newProjects.size() - 2), + newProjects.size() - 1, + RelCollations.EMPTY, + aggregateCall.getType(), + aggregateCall.getName() ); } else { newProjects.add(rexNode); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/AvailableSegmentMetadata.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/AvailableSegmentMetadata.java index 20d9b7e0ad9d..3cd307afcb93 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/AvailableSegmentMetadata.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/AvailableSegmentMetadata.java @@ -24,6 +24,7 @@ import org.apache.druid.timeline.DataSegment; import javax.annotation.Nullable; + import java.util.Set; /** diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/InformationSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/InformationSchema.java index f564d23413a6..449691a9bdaa 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/InformationSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/InformationSchema.java @@ -377,7 +377,7 @@ public Iterable apply(final String tableName) return generateColumnMetadata( schemaName, tableName, - table.getRowType(typeFactory), + table.getRowType(typeFactory), typeFactory ); } diff --git a/sql/src/main/java/org/apache/druid/sql/http/ArrayLinesWriter.java b/sql/src/main/java/org/apache/druid/sql/http/ArrayLinesWriter.java index beda6deceaba..4661754f265c 100644 --- a/sql/src/main/java/org/apache/druid/sql/http/ArrayLinesWriter.java +++ b/sql/src/main/java/org/apache/druid/sql/http/ArrayLinesWriter.java @@ -27,6 +27,7 @@ import org.apache.druid.java.util.common.jackson.JacksonUtils; import javax.annotation.Nullable; + import java.io.IOException; import java.io.OutputStream; diff --git a/sql/src/main/java/org/apache/druid/sql/http/ArrayWriter.java b/sql/src/main/java/org/apache/druid/sql/http/ArrayWriter.java index cd863d5bf175..1176796d12fa 100644 --- a/sql/src/main/java/org/apache/druid/sql/http/ArrayWriter.java +++ b/sql/src/main/java/org/apache/druid/sql/http/ArrayWriter.java @@ -29,6 +29,7 @@ import org.apache.druid.sql.calcite.table.RowSignatures; import javax.annotation.Nullable; + import java.io.IOException; import java.io.OutputStream; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index 9f25f4cd00f8..4a42152ffec2 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -2916,7 +2916,8 @@ public void testUnnestThriceWithFiltersOnDimAndUnnestCol() "_j0.unnest", "\"dimMultivalEnumerated\"", ColumnType.STRING - ), null + ), + null ), expressionVirtualColumn( "__j0.unnest", @@ -3112,7 +3113,8 @@ public void testUnnestThriceWithFiltersOnDimAndUnnestColumnsORCombinations() ); testQuery( sql, - QUERY_CONTEXT_UNNEST, expectedQuerySqlCom, + QUERY_CONTEXT_UNNEST, + expectedQuerySqlCom, ImmutableList.of( new Object[]{"27", "Baz", "Baz", "World"}, new Object[]{"27", "Baz", "Baz", "World"}, From de792497a77ba78f14bb13e9f8a1527897e5f315 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 1 Sep 2023 13:17:15 +0000 Subject: [PATCH 14/16] fix sql --- .../apache/druid/sql/calcite/CalciteArraysQueryTest.java | 3 ++- .../druid/sql/calcite/planner/DruidRexExecutorTest.java | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index 4a42152ffec2..c8ea97dff64c 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -3015,7 +3015,8 @@ public void testUnnestThriceWithFiltersOnDimAndAllUnnestColumns() "_j0.unnest", "\"dimMultivalEnumerated\"", ColumnType.STRING - ), equality("_j0.unnest", "Hello", ColumnType.STRING) + ), + equality("_j0.unnest", "Hello", ColumnType.STRING) ), expressionVirtualColumn( "__j0.unnest", diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java index 205f3379d71b..9155b58875b5 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java @@ -121,11 +121,13 @@ public void testLongsReduced() SqlStdOperatorTable.MULTIPLY, rexBuilder.makeLiteral( new BigDecimal(10L), - typeFactory.createSqlType(SqlTypeName.BIGINT), true + typeFactory.createSqlType(SqlTypeName.BIGINT), + true ), rexBuilder.makeLiteral( new BigDecimal(3L), - typeFactory.createSqlType(SqlTypeName.BIGINT), true + typeFactory.createSqlType(SqlTypeName.BIGINT), + true ) ); From 523da481923eb45702ab83b46d6cf905cc827c57 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 1 Sep 2023 13:19:08 +0000 Subject: [PATCH 15/16] add --- codestyle/checkstyle.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/codestyle/checkstyle.xml b/codestyle/checkstyle.xml index 3426961e9a5d..c708ed3fa41d 100644 --- a/codestyle/checkstyle.xml +++ b/codestyle/checkstyle.xml @@ -54,7 +54,7 @@ - + @@ -231,6 +231,7 @@ + From 6a6f844b0fb0487fd4433e1470d51a66e229e993 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 1 Sep 2023 13:19:55 +0000 Subject: [PATCH 16/16] enable back checkstyle plugins --- codestyle/checkstyle.xml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/codestyle/checkstyle.xml b/codestyle/checkstyle.xml index c708ed3fa41d..71891c0cdb32 100644 --- a/codestyle/checkstyle.xml +++ b/codestyle/checkstyle.xml @@ -24,12 +24,11 @@ "http://www.puppycrawl.com/dtds/configuration_1_2.dtd"> - + @@ -231,7 +230,6 @@ - +