From 870a74c6805866e4c24480be97268ef2576b2381 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 26 Mar 2025 10:54:19 +0000 Subject: [PATCH] Squashed commit of the following: commit 551d091d555625d9cef6aa687c1a9caf1abedc8b Author: Zoltan Haindrich Date: Wed Mar 26 10:53:00 2025 +0000 x commit d0c3d4a9bdb244720f648e3cf3d18fb39c0cda87 Author: Zoltan Haindrich Date: Wed Mar 26 07:40:46 2025 +0000 dont use dataset in disabled example commit cb5325d40b99e8bf690bd4d91a830aa6e2c1e7dc Author: Zoltan Haindrich Date: Tue Mar 25 14:43:29 2025 +0000 dont expose the if usage commit cb551e013967fbba1947bbf7f708bf4e0413b1dd Author: Zoltan Haindrich Date: Tue Mar 25 10:52:50 2025 +0000 use correct iterables commit 8e9b926bd18d6eee9b44fea0c790d131a235f06d Author: Zoltan Haindrich Date: Tue Mar 25 08:56:58 2025 +0000 update withThreadName; add test/etc commit ee079a85d8a4441ed11ac5f5e4ac3955a7084d21 Author: Zoltan Haindrich Date: Tue Mar 25 08:54:41 2025 +0000 fix coverage/etc commit ee905458769704255a85256af76c28cb97301e4d Author: Zoltan Haindrich Date: Tue Mar 25 07:15:05 2025 +0000 cleanup commit 87ad60c2de1df3de3431b8da19443aa9265c88ab Author: Zoltan Haindrich Date: Tue Mar 25 07:07:49 2025 +0000 cleanup commit 56dbe982731af5a3ddbfa9210b7a3548a58f5060 Author: Zoltan Haindrich Date: Tue Mar 25 06:58:21 2025 +0000 fixes commit d1d972023f6d1e0b9f1356ea5967db6018a3fd66 Author: Zoltan Haindrich Date: Mon Mar 24 18:32:48 2025 +0000 update test; respects rollup commit da422020a986bbfc447ffd1454828a5049c68c93 Author: Zoltan Haindrich Date: Mon Mar 24 14:15:07 2025 +0000 cleanup commit d586982b2415d69da8d2bf91bab069ca7d4a1693 Author: Zoltan Haindrich Date: Mon Mar 24 14:14:26 2025 +0000 enable if-s in iq tests commit bedb1db0e2438326db313532e4a4f882b64c566e Author: Zoltan Haindrich Date: Mon Mar 24 13:46:43 2025 +0000 make engine injected commit 413e3a53ed199532307b3ca1222c27ec7758aaf9 Author: Zoltan Haindrich Date: Mon Mar 24 13:37:29 2025 +0000 a commit 84b1f46305903aca500374d1f96782b4d7c29d23 Author: Zoltan Haindrich Date: Mon Mar 24 13:27:08 2025 +0000 updates commit da31fc4d301f96394930050bbe2cc10cc84e8149 Author: Zoltan Haindrich Date: Mon Mar 24 12:51:48 2025 +0000 fix rollup for dataset commit a8c057cc0f325bf415cefb99af98152a96882955 Author: Zoltan Haindrich Date: Mon Mar 24 12:31:11 2025 +0000 cleanup commit 0a5beb0b4dd9ccbf6be06fe5fd9a14f913061208 Author: Zoltan Haindrich Date: Mon Mar 24 12:14:12 2025 +0000 cleanup commit 961175938f2d19989cc9ccbef79f3b0b7a3f18a0 Author: Zoltan Haindrich Date: Mon Mar 24 11:49:29 2025 +0000 rename/cleanup/etc commit 741bddf5c33ff7ce4ce261dbc98ed7e9fa87f6dc Author: Zoltan Haindrich Date: Mon Mar 24 11:43:04 2025 +0000 add alldruidenginessupplier commit 8885e93b080a871946fc7d6dfb0ca8e50a92939c Author: Zoltan Haindrich Date: Mon Mar 24 11:31:12 2025 +0000 updates commit ccbfb405be32115753204682f057cd6759542303 Author: Zoltan Haindrich Date: Mon Mar 24 08:18:40 2025 +0000 changes --- .../druid/msq/quidem/MSQQuidemTest.java | 4 +- .../AbstractMSQComponentSupplierDelegate.java | 11 +- .../AllDruidEnginesComponentSupplier.java | 32 ++ .../druid/msq/test/CalciteMSQTestsHelper.java | 26 -- .../druid/msq/test/DartComponentSupplier.java | 10 +- .../org/apache/druid/concurrent/Threads.java | 15 +- .../druid/initialization/DruidModule.java | 33 ++ .../apache/druid/concurrent/ThreadsTest.java | 38 ++ .../druid/initialization/DruidModuleTest.java | 98 ++++ ...AsBrokerQueryComponentSupplierWrapper.java | 15 - .../java/org/apache/druid/quidem/QTest.java | 2 +- .../dart_with_datasets.iq | 13 - .../testframework_all_engines_dataset.iq | 17 + .../testframework_all_engines_disabled.iq | 20 + .../{dart1.iq => testframework_dart.iq} | 0 .../{example.iq => testframework_example.iq} | 0 .../guice/ServiceInjectorBuilderTest.java | 126 ++++++ .../datasets/InputSourceBasedTestDataset.java | 3 +- .../schema/DruidCalciteSchemaModule.java | 11 +- .../druid/sql/calcite/schema/DruidSchema.java | 3 +- .../calcite/schema/DruidSchemaManager.java | 11 +- .../org/apache/druid/sql/http/SqlQuery.java | 5 + .../druid/quidem/DruidAvaticaTestDriver.java | 11 +- .../druid/quidem/DruidConnectionExtras.java | 12 + .../quidem/DruidQuidemCommandHandler.java | 81 +++- .../quidem/DruidQuidemConnectionFactory.java | 53 ++- .../druid/quidem/DruidQuidemTestBase.java | 194 +++++++- .../apache/druid/quidem/SqlQuidemTest.java | 2 +- .../sql/calcite/BaseCalciteQueryTest.java | 5 + .../sql/calcite/CalciteIngestionDmlTest.java | 8 +- .../sql/calcite/CalciteScanSignatureTest.java | 19 +- .../sql/calcite/IngestionTestSqlEngine.java | 4 +- .../sql/calcite/MultiComponentSupplier.java | 173 ++++++++ .../apache/druid/sql/calcite/QTestCase.java | 3 +- .../sql/calcite/SqlTestFrameworkConfig.java | 33 +- .../druid/sql/calcite/util/CalciteTests.java | 15 +- .../calcite/util/DruidModuleCollection.java | 25 +- .../sql/calcite/util/QueryFrameworkUtils.java | 11 +- .../sql/calcite/util/SqlTestFramework.java | 418 ++++++++++-------- .../sql/calcite/util/TestDataBuilder.java | 6 +- .../customdataset.iq | 2 +- 41 files changed, 1208 insertions(+), 360 deletions(-) create mode 100644 extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/AllDruidEnginesComponentSupplier.java create mode 100644 processing/src/test/java/org/apache/druid/concurrent/ThreadsTest.java create mode 100644 processing/src/test/java/org/apache/druid/initialization/DruidModuleTest.java delete mode 100644 quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/dart_with_datasets.iq create mode 100644 quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/testframework_all_engines_dataset.iq create mode 100644 quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/testframework_all_engines_disabled.iq rename quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/{dart1.iq => testframework_dart.iq} (100%) rename quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/{example.iq => testframework_example.iq} (100%) create mode 100644 server/src/test/java/org/apache/druid/guice/ServiceInjectorBuilderTest.java create mode 100644 sql/src/test/java/org/apache/druid/sql/calcite/MultiComponentSupplier.java diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/quidem/MSQQuidemTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/quidem/MSQQuidemTest.java index 30deed4b6263..5f3ca89e4906 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/quidem/MSQQuidemTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/quidem/MSQQuidemTest.java @@ -19,15 +19,17 @@ package org.apache.druid.msq.quidem; +import org.apache.druid.quidem.DruidQuidemCommandHandler; import org.apache.druid.quidem.DruidQuidemTestBase; import org.apache.druid.quidem.ProjectPathUtils; + import java.io.File; public class MSQQuidemTest extends DruidQuidemTestBase { public MSQQuidemTest() { - super(); + super(new DruidQuidemCommandHandler()); } @Override diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/AbstractMSQComponentSupplierDelegate.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/AbstractMSQComponentSupplierDelegate.java index 42264ac46ba3..65634365dd62 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/AbstractMSQComponentSupplierDelegate.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/AbstractMSQComponentSupplierDelegate.java @@ -19,8 +19,6 @@ package org.apache.druid.msq.test; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.inject.Injector; import org.apache.druid.guice.IndexingServiceTuningConfigModule; import org.apache.druid.guice.JoinableFactoryModule; import org.apache.druid.initialization.DruidModule; @@ -29,7 +27,6 @@ import org.apache.druid.msq.guice.MSQIndexingModule; import org.apache.druid.msq.sql.MSQTaskSqlEngine; import org.apache.druid.msq.test.CalciteMSQTestsHelper.MSQTestModule; -import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.util.DruidModuleCollection; import org.apache.druid.sql.calcite.util.SqlTestFramework.QueryComponentSupplier; @@ -61,12 +58,9 @@ public DruidModule getCoreModule() } @Override - public SqlEngine createEngine( - QueryLifecycleFactory qlf, - ObjectMapper queryJsonMapper, - Injector injector) + public Class getSqlEngineClass() { - return injector.getInstance(MSQTaskSqlEngine.class); + return MSQTaskSqlEngine.class; } @Override @@ -74,4 +68,5 @@ public Boolean isExplainSupported() { return false; } + } diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/AllDruidEnginesComponentSupplier.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/AllDruidEnginesComponentSupplier.java new file mode 100644 index 000000000000..62799295908a --- /dev/null +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/AllDruidEnginesComponentSupplier.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.msq.test; + +import org.apache.druid.sql.calcite.MultiComponentSupplier; +import org.apache.druid.sql.calcite.MultiComponentSupplier.ComponentSuppliers; +import org.apache.druid.sql.calcite.util.SqlTestFramework.StandardComponentSupplier; + +@ComponentSuppliers({ + StandardComponentSupplier.class, + DartComponentSupplier.class, + StandardMSQComponentSupplier.class}) +public class AllDruidEnginesComponentSupplier extends MultiComponentSupplier +{ +} diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/CalciteMSQTestsHelper.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/CalciteMSQTestsHelper.java index 58f5a7b30624..b287458cf6b1 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/CalciteMSQTestsHelper.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/CalciteMSQTestsHelper.java @@ -45,12 +45,8 @@ import org.apache.druid.msq.guice.MSQExternalDataSourceModule; import org.apache.druid.msq.guice.MSQIndexingModule; import org.apache.druid.msq.querykit.DataSegmentProvider; -import org.apache.druid.query.DruidProcessingConfig; import org.apache.druid.query.ForwardingQueryProcessingPool; import org.apache.druid.query.QueryProcessingPool; -import org.apache.druid.query.groupby.GroupByQueryConfig; -import org.apache.druid.query.groupby.GroupByQueryRunnerTest; -import org.apache.druid.query.groupby.GroupingEngine; import org.apache.druid.query.groupby.TestGroupByBuffers; import org.apache.druid.segment.CompleteSegment; import org.apache.druid.segment.TestIndex; @@ -98,15 +94,6 @@ public void configure(Binder binder) { }).annotatedWith(Self.class).toInstance(ImmutableSet.of(NodeRole.PEON)); - DruidProcessingConfig druidProcessingConfig = new DruidProcessingConfig() - { - @Override - public String getFormatString() - { - return "test"; - } - }; - binder.bind(DruidProcessingConfig.class).toInstance(druidProcessingConfig); binder.bind(QueryProcessingPool.class) .toInstance(new ForwardingQueryProcessingPool(Execs.singleThreaded("Test-runner-processing-pool"))); @@ -174,7 +161,6 @@ public Supplier> fetchSegment( CompleteSegment a = walker.getSegment(segmentId); return () -> new ReferenceCountingResourceHolder<>(a, Closer.create()); } - } @Provides @@ -182,18 +168,6 @@ public DataServerQueryHandlerFactory provideDataServerQueryHandlerFactory() { return getTestDataServerQueryHandlerFactory(); } - - @Provides - @LazySingleton - GroupingEngine getGroupingEngine(GroupByQueryConfig groupByQueryConfig, TestGroupByBuffers groupByBuffers) - { - GroupingEngine groupingEngine = GroupByQueryRunnerTest.makeQueryRunnerFactory( - groupByQueryConfig, - groupByBuffers - ).getGroupingEngine(); - return groupingEngine; - } - } @Deprecated diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/DartComponentSupplier.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/DartComponentSupplier.java index 7cfac91a4ce5..9beabe7f6905 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/DartComponentSupplier.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/DartComponentSupplier.java @@ -19,9 +19,7 @@ package org.apache.druid.msq.test; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Binder; -import com.google.inject.Injector; import com.google.inject.Provides; import org.apache.druid.collections.NonBlockingPool; import org.apache.druid.discovery.DruidNodeDiscoveryProvider; @@ -41,7 +39,6 @@ import org.apache.druid.query.TestBufferPool; import org.apache.druid.rpc.ServiceClientFactory; import org.apache.druid.rpc.guice.ServiceClientModule; -import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.sql.avatica.DartDruidMeta; import org.apache.druid.sql.avatica.DruidMeta; import org.apache.druid.sql.calcite.TempDirProducer; @@ -90,12 +87,9 @@ public DruidModule getOverrideModule() } @Override - public SqlEngine createEngine( - QueryLifecycleFactory qlf, - ObjectMapper queryJsonMapper, - Injector injector) + public Class getSqlEngineClass() { - return injector.getInstance(DartSqlEngine.class); + return DartSqlEngine.class; } static class DartTestCoreModule implements DruidModule diff --git a/processing/src/main/java/org/apache/druid/concurrent/Threads.java b/processing/src/main/java/org/apache/druid/concurrent/Threads.java index 5dcb57c5cc14..825f4bfa23d4 100644 --- a/processing/src/main/java/org/apache/druid/concurrent/Threads.java +++ b/processing/src/main/java/org/apache/druid/concurrent/Threads.java @@ -55,7 +55,20 @@ public static void sleepFor(long sleepTime, TimeUnit unit) throws InterruptedExc } } - private Threads() + /** + * Temporarily renames the thread. + * + * Preferred usage should be via try-with-resources clauses. + * + * @return a {@link AutoCloseable} to rename the thread back to its original name. + */ + public static AutoCloseable withThreadName(String name) { + final Thread thread = Thread.currentThread(); + final String oldThreadName = thread.getName(); + thread.setName(name); + return () -> { + thread.setName(oldThreadName); + }; } } diff --git a/processing/src/main/java/org/apache/druid/initialization/DruidModule.java b/processing/src/main/java/org/apache/druid/initialization/DruidModule.java index 45d663cf5ce3..643dcf5789ef 100644 --- a/processing/src/main/java/org/apache/druid/initialization/DruidModule.java +++ b/processing/src/main/java/org/apache/druid/initialization/DruidModule.java @@ -20,6 +20,10 @@ package org.apache.druid.initialization; import com.fasterxml.jackson.databind.Module; +import com.google.inject.Binder; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.util.Modules; import org.apache.druid.guice.annotations.ExtensionPoint; import java.util.Collections; @@ -36,6 +40,35 @@ @ExtensionPoint public interface DruidModule extends com.google.inject.Module { + static DruidModule override(com.google.inject.Module baseModule, com.google.inject.Module overrideGuiceModule) + { + return new DruidModule() + { + @SuppressWarnings("unused") + @Inject + public void injectMe(Injector injector) + { + injector.injectMembers(baseModule); + injector.injectMembers(overrideGuiceModule); + } + + @Override + public void configure(Binder binder) + { + binder.install(Modules.override(baseModule).with(overrideGuiceModule)); + } + + @Override + public List getJacksonModules() + { + if (baseModule instanceof DruidModule) { + return ((DruidModule) baseModule).getJacksonModules(); + } + return Collections.emptyList(); + } + }; + } + default List getJacksonModules() { return Collections.emptyList(); diff --git a/processing/src/test/java/org/apache/druid/concurrent/ThreadsTest.java b/processing/src/test/java/org/apache/druid/concurrent/ThreadsTest.java new file mode 100644 index 000000000000..a8924b55f670 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/concurrent/ThreadsTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.concurrent; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ThreadsTest +{ + @Test + public void testThreadRename() throws Exception + { + String oldName = Thread.currentThread().getName(); + String newName = "testThreadRename-was:" + oldName; + try (AutoCloseable renameBack = Threads.withThreadName(newName)) { + assertEquals(newName, Thread.currentThread().getName()); + } + assertEquals(oldName, Thread.currentThread().getName()); + } +} diff --git a/processing/src/test/java/org/apache/druid/initialization/DruidModuleTest.java b/processing/src/test/java/org/apache/druid/initialization/DruidModuleTest.java new file mode 100644 index 000000000000..3d83dcd7d09a --- /dev/null +++ b/processing/src/test/java/org/apache/druid/initialization/DruidModuleTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.initialization; + +import com.fasterxml.jackson.databind.Module; +import com.google.inject.Binder; +import com.google.inject.Guice; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Provides; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class DruidModuleTest +{ + @Test + public void testOverride() + { + Injector initInjector = Guice.createInjector(new LocalProviderModule(10)); + DruidModule overrideModule = DruidModule.override(new LocalProviderModule(1), new LocalProviderModule(2)); + initInjector.injectMembers(overrideModule); + Injector injector = Guice.createInjector(overrideModule); + Integer val = injector.getInstance(Integer.class); + assertEquals(12, val); + } + + @Test + public void testOverrideJacksonModules() + { + Injector initInjector = Guice.createInjector(new LocalProviderModule(10)); + DruidModule overrideModule = DruidModule.override(new LocalProviderModule(1), new LocalProviderModule(2)); + initInjector.injectMembers(overrideModule); + List jacksonModules = overrideModule.getJacksonModules(); + // Even thru it was overridden, the base module should still provide these. + assertEquals(1, jacksonModules.size()); + } + + static class LocalProviderModule implements DruidModule + { + private int val; + private int baseVal = 0; + + @Inject + public void injectInteger(Integer baseVal) + { + this.baseVal = baseVal; + } + + public LocalProviderModule(int i) + { + val = i; + } + + @Override + public void configure(Binder binder) + { + } + + @Provides + public Integer getVal() + { + return val + baseVal; + } + + @Override + public List getJacksonModules() + { + List li = new ArrayList<>(); + for (int i = 0; i < val; i++) { + li.add(null); + } + return li; + } + + } + +} diff --git a/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java b/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java index 3cbe797be40d..0a2e496b09a9 100644 --- a/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java +++ b/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java @@ -30,11 +30,9 @@ import org.apache.druid.cli.CliBroker; import org.apache.druid.cli.QueryJettyServerInitializer; import org.apache.druid.client.BrokerSegmentWatcherConfig; -import org.apache.druid.client.BrokerServerView; import org.apache.druid.client.DirectDruidClientFactory; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.QueryableDruidServer; -import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.selector.CustomTierSelectorStrategyConfig; import org.apache.druid.client.selector.ServerSelectorStrategy; import org.apache.druid.client.selector.TierSelectorStrategy; @@ -42,7 +40,6 @@ import org.apache.druid.curator.discovery.DiscoveryModule; import org.apache.druid.discovery.DruidNodeDiscoveryProvider; import org.apache.druid.guice.AnnouncerModule; -import org.apache.druid.guice.BrokerProcessingModule; import org.apache.druid.guice.BrokerServiceModule; import org.apache.druid.guice.CoordinatorDiscoveryModule; import org.apache.druid.guice.ExpressionModule; @@ -59,7 +56,6 @@ import org.apache.druid.guice.SegmentWranglerModule; import org.apache.druid.guice.ServerViewModule; import org.apache.druid.guice.StartupLoggingModule; -import org.apache.druid.guice.StorageNodeModule; import org.apache.druid.guice.annotations.Client; import org.apache.druid.guice.annotations.EscalatedClient; import org.apache.druid.guice.http.HttpClientModule; @@ -78,12 +74,8 @@ import org.apache.druid.server.BrokerQueryResource; import org.apache.druid.server.ClientInfoResource; import org.apache.druid.server.DruidNode; -import org.apache.druid.server.SubqueryGuardrailHelper; -import org.apache.druid.server.SubqueryGuardrailHelperProvider; -import org.apache.druid.server.emitter.EmitterModule; import org.apache.druid.server.http.BrokerResource; import org.apache.druid.server.http.SelfDiscoveryResource; -import org.apache.druid.server.initialization.AuthorizerMapperModule; import org.apache.druid.server.initialization.ExternalStorageAccessSecurityModule; import org.apache.druid.server.initialization.jetty.JettyServerInitializer; import org.apache.druid.server.initialization.jetty.JettyServerModule; @@ -131,7 +123,6 @@ public DruidModule getCoreModule() modules.add(super.getCoreModule()); modules.addAll(forServerModules()); - modules.add(new BrokerProcessingModule()); modules.addAll(brokerModules()); modules.add(new QuidemCaptureModule()); @@ -181,7 +172,6 @@ private List forServerModules() new ExtensionsModule.SecondaryModule(), new DruidAuthModule(), new TLSCertificateCheckerModule(), - new EmitterModule(), HttpClientModule.global(), HttpClientModule.escalatedGlobal(), new HttpClientModule("druid.broker.http", Client.class, true), @@ -189,7 +179,6 @@ private List forServerModules() new CuratorModule(), new AnnouncerModule(), new SegmentWriteOutMediumModule(), - new StorageNodeModule(), new JettyServerModule(), new ExpressionModule(), new DiscoveryModule(), @@ -203,7 +192,6 @@ private List forServerModules() new JavaScriptModule(), new AuthenticatorModule(), new AuthorizerModule(), - new AuthorizerMapperModule(), new StartupLoggingModule(), new ExternalStorageAccessSecurityModule(), new ServiceClientModule(), @@ -231,8 +219,6 @@ static List brokerModules() binder.bindConstant().annotatedWith(Names.named("tlsServicePort")).to(8282); binder.bindConstant().annotatedWith(PruneLoadSpec.class).to(true); - binder.bind(TimelineServerView.class).to(BrokerServerView.class).in(LazySingleton.class); - JsonConfigProvider.bind(binder, "druid.broker.select", TierSelectorStrategy.class); JsonConfigProvider.bind(binder, "druid.broker.select.tier.custom", CustomTierSelectorStrategyConfig.class); JsonConfigProvider.bind(binder, "druid.broker.balancer", ServerSelectorStrategy.class); @@ -243,7 +229,6 @@ static List brokerModules() binder.bind(BrokerQueryResource.class).in(LazySingleton.class); Jerseys.addResource(binder, BrokerQueryResource.class); - binder.bind(SubqueryGuardrailHelper.class).toProvider(SubqueryGuardrailHelperProvider.class); binder.bind(QueryCountStatsProvider.class).to(BrokerQueryResource.class).in(LazySingleton.class); binder.bind(SubqueryCountStatsProvider.class).toInstance(new SubqueryCountStatsProvider()); Jerseys.addResource(binder, BrokerResource.class); diff --git a/quidem-ut/src/test/java/org/apache/druid/quidem/QTest.java b/quidem-ut/src/test/java/org/apache/druid/quidem/QTest.java index 83167a18a168..4eeb52ef7c61 100644 --- a/quidem-ut/src/test/java/org/apache/druid/quidem/QTest.java +++ b/quidem-ut/src/test/java/org/apache/druid/quidem/QTest.java @@ -31,7 +31,7 @@ public class QTest extends DruidQuidemTestBase { public QTest() { - super(); + super(new DruidQuidemCommandHandler()); } @Override diff --git a/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/dart_with_datasets.iq b/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/dart_with_datasets.iq deleted file mode 100644 index 65dbce0f087b..000000000000 --- a/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/dart_with_datasets.iq +++ /dev/null @@ -1,13 +0,0 @@ -!set dartQueryId 00000000-0000-0000-0000-000000000000 -!use druidtest://?componentSupplier=StandardMSQComponentSupplier&datasets=sql/src/test/quidem/sampledataset -!set outputformat mysql - -select count(1) from "rollup-tutorial"; -+--------+ -| EXPR$0 | -+--------+ -| 9 | -+--------+ -(1 row) - -!ok diff --git a/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/testframework_all_engines_dataset.iq b/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/testframework_all_engines_dataset.iq new file mode 100644 index 000000000000..dd085251ceec --- /dev/null +++ b/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/testframework_all_engines_dataset.iq @@ -0,0 +1,17 @@ +!set dartQueryId 00000000-0000-0000-0000-000000000000 +!use druidtest://?componentSupplier=AllDruidEnginesComponentSupplier&datasets=sql/src/test/quidem/sampledataset +!set outputformat mysql + +select * from "rollup-tutorial"; ++-------------------------+---------+---------+--------+-------+---------+ +| __time | srcIP | dstIP | bytes | count | packets | ++-------------------------+---------+---------+--------+-------+---------+ +| 2018-01-01 01:01:00.000 | 1.1.1.1 | 2.2.2.2 | 35937 | 3 | 286 | +| 2018-01-01 01:02:00.000 | 1.1.1.1 | 2.2.2.2 | 366260 | 2 | 415 | +| 2018-01-01 01:03:00.000 | 1.1.1.1 | 2.2.2.2 | 10204 | 1 | 49 | +| 2018-01-02 21:33:00.000 | 7.7.7.7 | 8.8.8.8 | 100288 | 2 | 161 | +| 2018-01-02 21:35:00.000 | 7.7.7.7 | 8.8.8.8 | 2818 | 1 | 12 | ++-------------------------+---------+---------+--------+-------+---------+ +(5 rows) + +!ok diff --git a/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/testframework_all_engines_disabled.iq b/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/testframework_all_engines_disabled.iq new file mode 100644 index 000000000000..c9c6a633dee4 --- /dev/null +++ b/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/testframework_all_engines_disabled.iq @@ -0,0 +1,20 @@ +!set dartQueryId 00000000-0000-0000-0000-000000000000 +!use druidtest://?componentSupplier=AllDruidEnginesComponentSupplier +!set outputformat mysql + +!disabled StandardComponentSupplier not supported on native engine: scan with order-by + +select dim1 from foo order by dim1; ++------+ +| dim1 | ++------+ +| | +| 1 | +| 10.1 | +| 2 | +| abc | +| def | ++------+ +(6 rows) + +!ok diff --git a/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/dart1.iq b/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/testframework_dart.iq similarity index 100% rename from quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/dart1.iq rename to quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/testframework_dart.iq diff --git a/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/example.iq b/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/testframework_example.iq similarity index 100% rename from quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/example.iq rename to quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/testframework_example.iq diff --git a/server/src/test/java/org/apache/druid/guice/ServiceInjectorBuilderTest.java b/server/src/test/java/org/apache/druid/guice/ServiceInjectorBuilderTest.java new file mode 100644 index 000000000000..cc11b1bc0e5e --- /dev/null +++ b/server/src/test/java/org/apache/druid/guice/ServiceInjectorBuilderTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Injector; +import com.google.inject.Provides; +import com.google.inject.util.Modules; +import org.apache.druid.initialization.CoreInjectorBuilder; +import org.apache.druid.initialization.DruidModule; +import org.apache.druid.initialization.ServiceInjectorBuilder; +import org.junit.jupiter.api.Test; + +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ServiceInjectorBuilderTest +{ + @Test + public void testNormalUsage() + { + Injector startupInjector = new StartupInjectorBuilder() + .withProperties(new Properties()) + .build(); + + CoreInjectorBuilder coreBuilder = new CoreInjectorBuilder(startupInjector); + coreBuilder.ignoreLoadScopes(); + coreBuilder.addModule( + new LocalProviderModule(1) + ); + + ServiceInjectorBuilder si = new ServiceInjectorBuilder(coreBuilder); + si.add( + new LocalProviderModule(2) + ); + Injector injector = si.build(); + Integer v = injector.getInstance(Integer.class); + assertEquals(2, v); + } + + @Test + public void testOverrideBelow() + { + Injector startupInjector = new StartupInjectorBuilder() + .withProperties(new Properties()) + .build(); + + CoreInjectorBuilder coreBuilder = new CoreInjectorBuilder(startupInjector); + coreBuilder.ignoreLoadScopes(); + coreBuilder.addModule( + Modules.override(new LocalProviderModule(0)) + .with(new LocalProviderModule(1)) + ); + + ServiceInjectorBuilder si = new ServiceInjectorBuilder(coreBuilder); + si.add( + new LocalProviderModule(2) + ); + Injector injector = si.build(); + Integer v = injector.getInstance(Integer.class); + assertEquals(2, v); + } + + @Test + public void testOverrideBoth() + { + Injector startupInjector = new StartupInjectorBuilder() + .withProperties(new Properties()) + .build(); + + CoreInjectorBuilder coreBuilder = new CoreInjectorBuilder(startupInjector); + coreBuilder.ignoreLoadScopes(); + coreBuilder.addModule( + Modules.override(new LocalProviderModule(0)) + .with(new LocalProviderModule(1)) + ); + + ServiceInjectorBuilder si = new ServiceInjectorBuilder(coreBuilder); + si.add( + Modules.override(new LocalProviderModule(-1)) + .with(new LocalProviderModule(2)) + ); + Injector injector = si.build(); + Integer v = injector.getInstance(Integer.class); + assertEquals(2, v); + } + + static class LocalProviderModule implements DruidModule + { + private int val; + + public LocalProviderModule(int i) + { + val = i; + } + + @Override + public void configure(Binder binder) + { + } + + @Provides + public Integer getVal() + { + return val; + } + } +} diff --git a/server/src/test/java/org/apache/druid/sql/calcite/util/datasets/InputSourceBasedTestDataset.java b/server/src/test/java/org/apache/druid/sql/calcite/util/datasets/InputSourceBasedTestDataset.java index 39cd46d52ac3..d5fee57fde62 100644 --- a/server/src/test/java/org/apache/druid/sql/calcite/util/datasets/InputSourceBasedTestDataset.java +++ b/server/src/test/java/org/apache/druid/sql/calcite/util/datasets/InputSourceBasedTestDataset.java @@ -86,7 +86,8 @@ public IncrementalIndexSchema getIndexSchema() .withTimestampSpec(getDataSchema().getTimestampSpec()) .withMetrics(getDataSchema().getAggregators()) .withDimensionsSpec(getDataSchema().getDimensionsSpec()) - .withRollup(false) + .withRollup(getDataSchema().getGranularitySpec().isRollup()) + .withQueryGranularity(getDataSchema().getGranularitySpec().getQueryGranularity()) .build(); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java index 26d038ef4a3b..14e1fc068824 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java @@ -22,7 +22,6 @@ import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Provides; -import com.google.inject.Scopes; import com.google.inject.name.Named; import com.google.inject.name.Names; import org.apache.druid.guice.LazySingleton; @@ -47,15 +46,15 @@ public void configure(Binder binder) binder.bind(DruidSchemaCatalog.class) .annotatedWith(Names.named(INCOMPLETE_SCHEMA)) .toProvider(RootSchemaProvider.class) - .in(Scopes.SINGLETON); + .in(LazySingleton.class); // BrokerSegmentMetadataCache needs to listen to changes for incoming segments LifecycleModule.register(binder, BrokerSegmentMetadataCache.class); - binder.bind(DruidSchema.class).in(Scopes.SINGLETON); - binder.bind(SystemSchema.class).in(Scopes.SINGLETON); - binder.bind(InformationSchema.class).in(Scopes.SINGLETON); - binder.bind(LookupSchema.class).in(Scopes.SINGLETON); + binder.bind(DruidSchema.class).in(LazySingleton.class); + binder.bind(SystemSchema.class).in(LazySingleton.class); + binder.bind(InformationSchema.class).in(LazySingleton.class); + binder.bind(LookupSchema.class).in(LazySingleton.class); // Binder to inject different schema to Calcite SqlBindings.addSchema(binder, NamedDruidSchema.class); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java index 5a8ae5ba9a8d..96cd166b1565 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java @@ -25,6 +25,7 @@ import org.apache.druid.sql.calcite.table.DruidTable; import javax.inject.Inject; + import java.util.Set; public class DruidSchema extends AbstractTableSchema @@ -74,7 +75,7 @@ public Table getTable(String name) public Set getTableNames() { if (druidSchemaManager != null) { - return druidSchemaManager.getTableNames(); + return druidSchemaManager.getTableNames(segmentMetadataCache); } else { return catalogResolver.getTableNames(segmentMetadataCache.getDatasourceNames()); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchemaManager.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchemaManager.java index 85808bb1af4a..4ea8dfb829cb 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchemaManager.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchemaManager.java @@ -44,8 +44,8 @@ public interface DruidSchemaManager /** * Return all tables known to this schema manager. Deprecated because getting * all tables is never actually needed in the current code. Calcite asks for - * the information for a single table (via {@link #getTable(String)}, or - * the list of all table names (via {@link #getTableNames()}. This + * the information for a single table (via {@link #getTable(String, BrokerSegmentMetadataCache)}, or + * the list of all table names (via {@link #getTableNames(BrokerSegmentMetadataCache)}. This * method was originally used to allow Calcite's {@link * org.apache.calcite.schema.impl.AbstractSchema AbstractSchema} class to do * the lookup. The current code implements the operations directly. For @@ -55,17 +55,12 @@ public interface DruidSchemaManager @Deprecated Map getTables(); - default DruidTable getTable(String name) - { - return getTables().get(name); - } - default DruidTable getTable(String name, BrokerSegmentMetadataCache segmentMetadataCache) { return getTables().get(name); } - default Set getTableNames() + default Set getTableNames(BrokerSegmentMetadataCache segmentMetadataCache) { return getTables().keySet(); } diff --git a/sql/src/main/java/org/apache/druid/sql/http/SqlQuery.java b/sql/src/main/java/org/apache/druid/sql/http/SqlQuery.java index 7b54ae39fadf..623570f3df23 100644 --- a/sql/src/main/java/org/apache/druid/sql/http/SqlQuery.java +++ b/sql/src/main/java/org/apache/druid/sql/http/SqlQuery.java @@ -195,4 +195,9 @@ public String toString() ", parameters=" + parameters + '}'; } + + public SqlQuery withQueryContext(Map newContext) + { + return new SqlQuery(query, resultFormat, header, typesHeader, sqlTypesHeader, newContext, parameters); + } } diff --git a/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java b/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java index 21e0cc7bd71a..df66887dfd4b 100644 --- a/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java +++ b/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java @@ -31,6 +31,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.server.DruidNode; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.avatica.AvaticaMonitor; import org.apache.druid.sql.avatica.DruidAvaticaJsonHandler; import org.apache.druid.sql.avatica.DruidMeta; @@ -86,7 +87,12 @@ public Connection connect(String url, Properties info) throws SQLException return server.getConnection(info); } catch (Exception e) { - throw new SQLException("Can't create testconnection", e); + if (e instanceof SQLException) { + throw (SQLException) e; + } + // We create an Error here so that the exception is certain to make it out of the Quidem runner because it + // captures SqlExceptions and makes the messages hard to find sometimes. + throw new Error("Can't create testconnection", e); } } @@ -100,6 +106,7 @@ public DruidConnectionExtras getConnectionExtras( ObjectMapper objectMapper, DruidHookDispatcher druidHookDispatcher, @Named("isExplainSupported") Boolean isExplainSupported, + SpecificSegmentsQuerySegmentWalker walker, Injector injector ) { @@ -107,6 +114,7 @@ public DruidConnectionExtras getConnectionExtras( objectMapper, druidHookDispatcher, isExplainSupported, + walker, injector ); } @@ -131,7 +139,6 @@ public void close() throws IOException { closer.close(); } - } static class AvaticaJettyServer implements Closeable diff --git a/sql/src/test/java/org/apache/druid/quidem/DruidConnectionExtras.java b/sql/src/test/java/org/apache/druid/quidem/DruidConnectionExtras.java index fa46e5f9d342..c87e0c968795 100644 --- a/sql/src/test/java/org/apache/druid/quidem/DruidConnectionExtras.java +++ b/sql/src/test/java/org/apache/druid/quidem/DruidConnectionExtras.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Injector; import org.apache.druid.error.DruidException; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.hook.DruidHookDispatcher; import java.sql.Connection; @@ -30,6 +31,8 @@ public interface DruidConnectionExtras { ObjectMapper getObjectMapper(); + SpecificSegmentsQuerySegmentWalker getWalker(); + DruidHookDispatcher getDruidHookDispatcher(); boolean isExplainSupported(); @@ -41,18 +44,21 @@ class DruidConnectionExtrasImpl implements DruidConnectionExtras private final ObjectMapper objectMapper; private final DruidHookDispatcher druidHookDispatcher; private final boolean isExplainSupported; + private final SpecificSegmentsQuerySegmentWalker walker; private final Injector injector; public DruidConnectionExtrasImpl( ObjectMapper objectMapper, DruidHookDispatcher druidHookDispatcher, boolean isExplainSupported, + SpecificSegmentsQuerySegmentWalker walker, Injector injector ) { this.objectMapper = objectMapper; this.druidHookDispatcher = druidHookDispatcher; this.isExplainSupported = isExplainSupported; + this.walker = walker; this.injector = injector; } @@ -74,6 +80,12 @@ public boolean isExplainSupported() return isExplainSupported; } + @Override + public SpecificSegmentsQuerySegmentWalker getWalker() + { + return walker; + } + @Override public Injector getInjector() { diff --git a/sql/src/test/java/org/apache/druid/quidem/DruidQuidemCommandHandler.java b/sql/src/test/java/org/apache/druid/quidem/DruidQuidemCommandHandler.java index 5bf6d18050f5..af52f4feb57e 100644 --- a/sql/src/test/java/org/apache/druid/quidem/DruidQuidemCommandHandler.java +++ b/sql/src/test/java/org/apache/druid/quidem/DruidQuidemCommandHandler.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; +import com.google.inject.Injector; import net.hydromatic.quidem.AbstractCommand; import net.hydromatic.quidem.Command; import net.hydromatic.quidem.CommandHandler; @@ -33,8 +34,10 @@ import org.apache.calcite.sql.SqlExplainFormat; import org.apache.calcite.sql.SqlExplainLevel; import org.apache.calcite.util.Util; +import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.Query; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; +import org.apache.druid.sql.calcite.SqlTestFrameworkConfig; import org.apache.druid.sql.calcite.rel.DruidRel; import org.apache.druid.sql.hook.DruidHook; import org.apache.druid.sql.hook.DruidHook.HookKey; @@ -46,6 +49,10 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assume.assumeFalse; public class DruidQuidemCommandHandler implements CommandHandler { @@ -70,15 +77,18 @@ public Command parseCommand(List lines, List content, String lin if (line.startsWith("msqPlan")) { return new MSQPlanCommand(lines, content); } + if (line.startsWith("disabled")) { + return new DisabledCommand(lines, content, line); + } return null; } - abstract static class AbstractPlanCommand extends AbstractCommand + public abstract static class AbstractPlanCommand extends AbstractCommand { private final List content; private final List lines; - AbstractPlanCommand(List lines, List content) + public AbstractPlanCommand(List lines, List content) { this.lines = ImmutableList.copyOf(lines); this.content = content; @@ -98,6 +108,9 @@ public final void execute(Context context, boolean execute) executeExplain(context); } catch (Exception e) { + // This is packaged as an Error because Quidem grabs anything else and pushes it into the output file + // believing that you are wanting to validate the error. By repackaging it, we get a nice failure in the + // runner instead throw new Error(e); } } else { @@ -110,9 +123,7 @@ protected final List executeExplainCollectHookValues(Context context, Hoo { DruidHookDispatcher dhp = unwrapDruidHookDispatcher(context); List logged = new ArrayList<>(); - try (Closeable unhook = dhp.withHook(hook, (key, value) -> { - logged.add(value); - })) { + try (Closeable unhook = dhp.withHook(hook, (key, value) -> logged.add(value))) { executeExplainQuery(context); } return logged; @@ -160,7 +171,9 @@ protected final DruidHookDispatcher unwrapDruidHookDispatcher(Context context) protected abstract void executeExplain(Context context) throws Exception; } - /** Command that prints the plan for the current query. */ + /** + * Command that prints the plan for the current query. + */ static class NativePlanCommand extends AbstractPlanCommand { NativePlanCommand(List lines, List content) @@ -304,6 +317,61 @@ public String getCollectedHintsAsString() } } + /** + * Usage: + * 
+   * !disabled StandardComponentSupplier reason for disabling
+   *
+ */ + static class DisabledCommand extends AbstractCommand + { + private String supplierName; + private String message; + private Pattern DISABLED_PATTERN = Pattern.compile("disabled\\s+([\\w\\d]+)(\\s+([^\\s].+))?\\s*"); + + private final List content; + private final List lines; + + DisabledCommand(List lines, List content, String line) + { + super(); + this.lines = ImmutableList.copyOf(lines); + this.content = content; + Matcher m = DISABLED_PATTERN.matcher(line); + if (!m.matches()) { + throw new IllegalArgumentException("usage: !disabled []"); + } + supplierName = m.group(1); + message = buildMessage(supplierName, m.group(3)); + } + + private static String buildMessage(String supplierName, String message) + { + return StringUtils.format("Test disabled on supplier [%s]. Reason: [%s]", supplierName, message); + } + + @Override + public final void execute(Context context, boolean execute) + { + if (execute) { + try { + Injector injector = DruidConnectionExtras.unwrapOrThrow(context.connection()).getInjector(); + SqlTestFrameworkConfig cfg = injector.getInstance(SqlTestFrameworkConfig.class); + assumeFalse(message, cfg.componentSupplier.getSimpleName().equals(supplierName)); + } + catch (Exception e) { + // This is packaged as an Error because Quidem grabs anything else and pushes it into the output file + // believing that you are wanting to validate the error. By repackaging it, we get a nice failure in the + // runner instead + throw new Error(e); + } + } else { + context.echo(content); + } + context.echo(lines); + } + } + static class ConvertedPlanCommand extends AbstractRelPlanCommand { ConvertedPlanCommand(List lines, List content) @@ -311,6 +379,7 @@ static class ConvertedPlanCommand extends AbstractRelPlanCommand super(lines, content, DruidHook.CONVERTED_PLAN); } } + static class MSQPlanCommand extends AbstractStringCaptureCommand { MSQPlanCommand(List lines, List content) diff --git a/sql/src/test/java/org/apache/druid/quidem/DruidQuidemConnectionFactory.java b/sql/src/test/java/org/apache/druid/quidem/DruidQuidemConnectionFactory.java index b9a7963f4d80..6a955caa4891 100644 --- a/sql/src/test/java/org/apache/druid/quidem/DruidQuidemConnectionFactory.java +++ b/sql/src/test/java/org/apache/druid/quidem/DruidQuidemConnectionFactory.java @@ -19,15 +19,24 @@ package org.apache.druid.quidem; +import com.google.inject.Injector; import net.hydromatic.quidem.Quidem.ConnectionFactory; import net.hydromatic.quidem.Quidem.PropertyHandler; +import org.apache.druid.sql.calcite.run.SqlEngine; +import org.apache.druid.sql.calcite.util.SqlTestFramework.QueryComponentSupplier; +import org.apache.http.client.utils.URIBuilder; + +import java.net.URI; import java.sql.Connection; import java.sql.DriverManager; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; public class DruidQuidemConnectionFactory implements ConnectionFactory, PropertyHandler { private Properties props = new Properties(); + private Map envs = new HashMap<>(); public DruidQuidemConnectionFactory() { @@ -38,12 +47,52 @@ public DruidQuidemConnectionFactory() @Override public Connection connect(String name, boolean reference) throws Exception { - if (name.startsWith("druidtest://")) { - return DriverManager.getConnection(name, props); + URI uri = URI.create(name); + if (uri.getScheme().endsWith("test")) { + // This property will be set by DruidQuidemTestBase to select the supplier + // for MultiComponentSupplier backed tests. + // note: caching will be based on the effective config: so if the same + // supplier is used from different MultiComponentSupplier it will be + // shared. + String customSupplier = props.getProperty("componentSupplier"); + if (customSupplier != null) { + URIBuilder builder = new URIBuilder(uri); + builder.addParameter("componentSupplier", customSupplier); + builder.setHost(""); + name = builder.build().toString(); + } + Connection connection = DriverManager.getConnection(name, props); + envs = buildEnvsForConnection(connection); + return connection; } throw new RuntimeException("unknown connection '" + name + "'"); } + private static Map buildEnvsForConnection(Connection connection) + { + Map envs = new HashMap<>(); + DruidConnectionExtras extras = DruidConnectionExtras.unwrapOrThrow(connection); + Injector injector = extras.getInjector(); + QueryComponentSupplier supplier = injector.getInstance(QueryComponentSupplier.class); + Class engineClazz = supplier.getSqlEngineClass(); + String engineName = engineClazz.getName(); + boolean isDart = engineName.contains("Dart"); + boolean isMSQ = engineName.contains("MSQ"); + boolean isNative = engineName.contains("Native"); + + envs.put("isDart", String.valueOf(isDart)); + envs.put("isMSQ", String.valueOf(isMSQ)); + envs.put("isNative", String.valueOf(isNative)); + envs.put("isTaskBased", String.valueOf(isDart || isMSQ)); + + return envs; + } + + public Object envLookup(String key) + { + return envs.get(key); + } + @Override public void onSet(String key, Object value) { diff --git a/sql/src/test/java/org/apache/druid/quidem/DruidQuidemTestBase.java b/sql/src/test/java/org/apache/druid/quidem/DruidQuidemTestBase.java index 8366d1dce9ef..7088d7b65953 100644 --- a/sql/src/test/java/org/apache/druid/quidem/DruidQuidemTestBase.java +++ b/sql/src/test/java/org/apache/druid/quidem/DruidQuidemTestBase.java @@ -19,6 +19,7 @@ package org.apache.druid.quidem; +import com.google.common.collect.Iterables; import com.google.common.io.Files; import net.hydromatic.quidem.CommandHandler; import net.hydromatic.quidem.Quidem; @@ -29,10 +30,16 @@ import org.apache.calcite.util.Util; import org.apache.commons.io.filefilter.TrueFileFilter; import org.apache.commons.io.filefilter.WildcardFileFilter; +import org.apache.druid.concurrent.Threads; +import org.apache.druid.error.DruidException; import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.RE; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.sql.calcite.MultiComponentSupplier; +import org.apache.druid.sql.calcite.SqlTestFrameworkConfig; +import org.apache.druid.sql.calcite.util.SqlTestFramework.QueryComponentSupplier; +import org.junit.AssumptionViolatedException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.params.ParameterizedTest; @@ -44,9 +51,14 @@ import java.io.IOException; import java.io.Reader; import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.function.Function; import static org.junit.jupiter.api.Assertions.fail; @@ -72,8 +84,7 @@ *
  • Copy over the .iq.out to .iq to accept the changes
    • * * - * To shorten the above 2 steps you can run the test with system property quiem.overwrite=true - * + * To shorten the above 2 steps you can run the test with system property quidem.overwrite=true */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) public abstract class DruidQuidemTestBase @@ -91,7 +102,12 @@ public abstract class DruidQuidemTestBase private DruidQuidemRunner druidQuidemRunner; - public DruidQuidemTestBase() + public DruidQuidemTestBase(CommandHandler commandHandler) + { + this(new DruidQuidemRunner(commandHandler, Quidem::new)); + } + + public DruidQuidemTestBase(DruidQuidemRunner druidQuidemRunner) { String filterStr = System.getProperty(PROPERTY_FILTER, null); if (filterStr != null) { @@ -100,58 +116,170 @@ public DruidQuidemTestBase() } filter = new WildcardFileFilter(filterStr); } - druidQuidemRunner = new DruidQuidemRunner(createCommandHandler()); + this.druidQuidemRunner = druidQuidemRunner; } - /** Creates a command handler. */ - protected CommandHandler createCommandHandler() + protected static class QuidemTestCaseConfiguration { - return new DruidQuidemCommandHandler(); + final String fileName; + final String componentSupplierName; + + public QuidemTestCaseConfiguration(String componentSupplierName, String fileName) + { + this.fileName = fileName; + this.componentSupplierName = componentSupplierName; + } + + public String getTestName() + { + if (componentSupplierName == null) { + return fileName; + } else { + return StringUtils.format("%s@%s", fileName, componentSupplierName); + } + } + + @Override + public String toString() + { + return getTestName(); + } } - @ParameterizedTest - @MethodSource("getFileNames") - public void test(String testFileName) throws Exception + public List getTestConfigs() throws IOException + { + List ret = new ArrayList<>(); + List fileNames = getFileNames(); + for (String file : fileNames) { + try { + ret.addAll(getConfigurationsFor(file)); + } + catch (Exception e) { + throw DruidException.defensive(e, "While processing configurations for quidem file [%s]", file); + } + } + return ret; + } + + private List getConfigurationsFor(String testFileName) throws IOException { File inFile = new File(getTestRoot(), testFileName); + List ret = new ArrayList<>(); + for (Class supplier : collectSuppliers(inFile)) { + String supplierName = supplier == null ? null : supplier.getSimpleName(); + ret.add(new QuidemTestCaseConfiguration(supplierName, testFileName)); + } + return ret; + } + + private List> collectSuppliers(File inFile) throws IOException + { + Set> metaSuppliers = collectMetaSuppliers(inFile); + + switch (metaSuppliers.size()) { + case 0: + return Collections.singletonList(null); + case 1: + return MultiComponentSupplier.getSuppliers(Iterables.getOnlyElement(metaSuppliers)); + default: + throw DruidException.defensive("Multiple MetaComponentSuppliers found [%s].", metaSuppliers); + } + } + + private Set> collectMetaSuppliers(File inFile) throws IOException + { + Set> metaSuppliers = new HashSet<>(); + + for (String line : Files.readLines(inFile, StandardCharsets.UTF_8)) { + if (line.startsWith("!use")) { + String[] parts = line.split(" "); + if (parts.length == 2) { + SqlTestFrameworkConfig cfg = SqlTestFrameworkConfig.fromURL(parts[1]); + validateFrameworkConfig(cfg); + if (MultiComponentSupplier.class.isAssignableFrom(cfg.componentSupplier)) { + metaSuppliers.add((Class) cfg.componentSupplier); + } + } + } + } + return metaSuppliers; + } + + @SuppressWarnings("unused") + protected void validateFrameworkConfig(SqlTestFrameworkConfig cfg) + { + // no-op + } + + @ParameterizedTest + @MethodSource("getTestConfigs") + public void test(QuidemTestCaseConfiguration testConfig) throws Exception + { + final String testName = testConfig.getTestName(); + + File inFile = new File(getTestRoot(), testConfig.fileName); + final File outFile = new File(inFile.getParentFile(), testName + ".iq.out"); + try (AutoCloseable closeable = Threads.withThreadName(testName)) { + druidQuidemRunner.run(inFile, outFile, testConfig.componentSupplierName); + } + catch (Error e) { + // This catch is needed to workaround the way Quidem currently handles AssumptionViolatedException + Throwable cause = e.getCause(); + if (cause != null && cause instanceof AssumptionViolatedException) { + AssumptionViolatedException assumptionViolatedException = (AssumptionViolatedException) cause; + throw assumptionViolatedException; + } + throw e; + } - final File outFile = new File(inFile.getParentFile(), inFile.getName() + ".out"); - druidQuidemRunner.run(inFile, outFile); } public static class DruidQuidemRunner { private CommandHandler commandHandler; + private Function quidemMaker; + + public DruidQuidemRunner() + { + this(new DruidQuidemCommandHandler(), Quidem::new); + } - public DruidQuidemRunner(CommandHandler commandHandler) + public DruidQuidemRunner(CommandHandler commandHandler, Function quidemMaker) { this.commandHandler = commandHandler; + this.quidemMaker = quidemMaker; } public void run(File inFile) throws Exception { File outFile = new File(inFile.getParent(), inFile.getName() + ".out"); - run(inFile, outFile); + run(inFile, outFile, null); } - public void run(File inFile, final File outFile) throws Exception + public void run(File inFile, final File outFile, String componentSupplier) throws Exception { FileUtils.mkdirp(outFile.getParentFile()); try (Reader reader = Util.reader(inFile); - Writer writer = Util.printWriter(outFile); - Closer closer = new Closer()) { + Writer writer = Util.printWriter(outFile); + Closer closer = new Closer()) { DruidQuidemConnectionFactory connectionFactory = new DruidQuidemConnectionFactory(); + if (componentSupplier != null) { + connectionFactory.onSet("componentSupplier", componentSupplier); + } ConfigBuilder configBuilder = Quidem.configBuilder() .withConnectionFactory(connectionFactory) .withPropertyHandler(connectionFactory) + .withEnv(connectionFactory::envLookup) .withCommandHandler(commandHandler); Config config = configBuilder .withReader(reader) - .withWriter(writer).build(); + .withWriter(writer) + .withStackLimit(-1) + .build(); - new Quidem(config).execute(); + quidemMaker.apply(config).execute(); } catch (Exception e) { throw new RE(e, "Encountered exception while running [%s]", inFile); @@ -162,17 +290,33 @@ public void run(File inFile, final File outFile) throws Exception if (!diff.isEmpty()) { if (isOverwrite()) { Files.copy(outFile, inFile); + } else if (isUnsupportedComponentSupplier(diff, componentSupplier)) { + System.out.println("Skipping verification of unsupported componentSupplier " + componentSupplier); } else { fail("Files differ: " + outFile + " " + inFile + "\n" + diff); } } } + public static boolean isOverwrite() { String property = System.getProperty(OVERWRITE_PROPERTY, "false"); return property.length() == 0 || Boolean.valueOf(property); } + + private static boolean isUnsupportedComponentSupplier(String diff, String componentSupplier) + { + return diff.contains(StringUtils.format( + "Unsupported componentSupplier[%s], skipping verification of diff.", + componentSupplier + )); + } + } + + protected CommandHandler getCommandHandler() + { + return new DruidQuidemCommandHandler(); } protected final List getFileNames() throws IOException @@ -183,8 +327,12 @@ protected final List getFileNames() throws IOException if (!testRoot.exists()) { throw new FileNotFoundException(StringUtils.format("testRoot [%s] doesn't exists!", testRoot)); } - for (File f : testRoot.listFiles(this::isTestIncluded)) { - ret.add(f.getName()); + + for (File f : Files.fileTraverser().breadthFirst(testRoot)) { + if (isTestIncluded(f)) { + Path relativePath = testRoot.toPath().relativize(f.toPath()); + ret.add(relativePath.toString()); + } } if (ret.isEmpty()) { throw new IAE( @@ -200,8 +348,8 @@ protected final List getFileNames() throws IOException private boolean isTestIncluded(File f) { return !f.isDirectory() - && f.getName().endsWith(IQ_SUFFIX) - && filter.accept(f); + && f.getName().endsWith(IQ_SUFFIX) + && filter.accept(f); } protected abstract File getTestRoot(); diff --git a/sql/src/test/java/org/apache/druid/quidem/SqlQuidemTest.java b/sql/src/test/java/org/apache/druid/quidem/SqlQuidemTest.java index 9e6a965891ce..f9ae79925752 100644 --- a/sql/src/test/java/org/apache/druid/quidem/SqlQuidemTest.java +++ b/sql/src/test/java/org/apache/druid/quidem/SqlQuidemTest.java @@ -25,7 +25,7 @@ public class SqlQuidemTest extends DruidQuidemTestBase { public SqlQuidemTest() { - super(); + super(new DruidQuidemCommandHandler()); } @Override diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index 52605f04dce7..aa8542d92bef 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -1234,6 +1234,11 @@ public static Query recursivelyClearContext(final Query query, ObjectM } final JsonNode newQueryNode = queryJsonMapper.valueToTree(newQuery); ((ObjectNode) newQueryNode).remove("context"); + JsonNode fc = ((ObjectNode) newQueryNode).get("searchFilterContext"); + if (fc != null) { + ((ObjectNode) fc).remove("nowMs"); + } + return queryJsonMapper.treeToValue(newQueryNode, Query.class); } catch (Exception e) { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteIngestionDmlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteIngestionDmlTest.java index 3b4c864fc659..2484c2deaf7f 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteIngestionDmlTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteIngestionDmlTest.java @@ -30,7 +30,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.inject.Binder; -import com.google.inject.Injector; import org.apache.druid.data.input.AbstractInputSource; import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.InputSplit; @@ -48,7 +47,6 @@ import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; -import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthenticationResult; import org.apache.druid.server.security.ResourceAction; @@ -127,7 +125,7 @@ public class CalciteIngestionDmlTest extends BaseCalciteQueryTest protected boolean didTest = false; - static class IngestionDmlComponentSupplier extends StandardComponentSupplier + public static class IngestionDmlComponentSupplier extends StandardComponentSupplier { public IngestionDmlComponentSupplier(TempDirProducer tempFolderProducer) { @@ -135,9 +133,9 @@ public IngestionDmlComponentSupplier(TempDirProducer tempFolderProducer) } @Override - public SqlEngine createEngine(QueryLifecycleFactory qlf, ObjectMapper queryJsonMapper, Injector injector) + public Class getSqlEngineClass() { - return IngestionTestSqlEngine.INSTANCE; + return IngestionTestSqlEngine.class; } @Override diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteScanSignatureTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteScanSignatureTest.java index 1f50b81b51ea..02a2a168034d 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteScanSignatureTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteScanSignatureTest.java @@ -19,21 +19,20 @@ package org.apache.druid.sql.calcite; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import com.google.inject.Injector; +import com.google.inject.Inject; import org.apache.calcite.rel.RelRoot; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.tools.ValidationException; import org.apache.druid.query.scan.ScanQuery; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.sql.calcite.CalciteScanSignatureTest.ScanSignatureComponentSupplier; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.rel.DruidQuery; import org.apache.druid.sql.calcite.run.EngineFeature; +import org.apache.druid.sql.calcite.run.NativeSqlEngine; import org.apache.druid.sql.calcite.run.QueryMaker; import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.util.CalciteTests; @@ -116,22 +115,20 @@ public ScanSignatureComponentSupplier(TempDirProducer tempFolderProducer) super(tempFolderProducer); } + @Override - public SqlEngine createEngine( - QueryLifecycleFactory qlf, - ObjectMapper queryJsonMapper, - Injector injector - ) + public Class getSqlEngineClass() { - // Create an engine that says yes to EngineFeature.SCAN_NEEDS_SIGNATURE. - return new ScanSignatureTestSqlEngine(super.createEngine(qlf, queryJsonMapper, injector)); + return ScanSignatureTestSqlEngine.class; } + // Create an engine that says yes to EngineFeature.SCAN_NEEDS_SIGNATURE. private static class ScanSignatureTestSqlEngine implements SqlEngine { private final SqlEngine parent; - public ScanSignatureTestSqlEngine(final SqlEngine parent) + @Inject + public ScanSignatureTestSqlEngine(final NativeSqlEngine parent) { this.parent = parent; } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/IngestionTestSqlEngine.java b/sql/src/test/java/org/apache/druid/sql/calcite/IngestionTestSqlEngine.java index 569598af1e4e..8cb2a974e6d1 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/IngestionTestSqlEngine.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/IngestionTestSqlEngine.java @@ -20,6 +20,7 @@ package org.apache.druid.sql.calcite; import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; import org.apache.calcite.rel.RelRoot; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; @@ -37,8 +38,7 @@ public class IngestionTestSqlEngine implements SqlEngine { - public static final IngestionTestSqlEngine INSTANCE = new IngestionTestSqlEngine(); - + @Inject private IngestionTestSqlEngine() { } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/MultiComponentSupplier.java b/sql/src/test/java/org/apache/druid/sql/calcite/MultiComponentSupplier.java new file mode 100644 index 000000000000..b6efbc9b351d --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/MultiComponentSupplier.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite; + +import com.google.inject.Module; +import org.apache.druid.error.DruidException; +import org.apache.druid.guice.DruidInjectorBuilder; +import org.apache.druid.initialization.DruidModule; +import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.query.QueryRunnerFactoryConglomerate; +import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; +import org.apache.druid.segment.join.JoinableFactoryWrapper; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.sql.calcite.run.SqlEngine; +import org.apache.druid.sql.calcite.util.SqlTestFramework; +import org.apache.druid.sql.calcite.util.SqlTestFramework.PlannerComponentSupplier; +import org.apache.druid.sql.calcite.util.SqlTestFramework.QueryComponentSupplier; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +/** + * Multi supplier to run the same testcase against multiple suppliers. + * + * Usage: + * + * 
    + * @ComponentSuppliers( {  RealSupplier1.class, RealSupplier2.class } )
+ * public class SomeSupplier extends MultiComponentSupplier {
+ * }
+ *
    + */ +public abstract class MultiComponentSupplier implements QueryComponentSupplier +{ + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.TYPE) + public @interface ComponentSuppliers + { + Class[] value(); + } + + private static final Map, Set>> + KNOWN_SUB_QUERY_COMPONENT_SUPPLIERS = new LinkedHashMap<>(); + + public static void registerComponentSupplier(Class clazz) + { + final List> subSuppliers = getSuppliers(clazz); + for (Class subSupplier : subSuppliers) { + KNOWN_SUB_QUERY_COMPONENT_SUPPLIERS.computeIfAbsent(subSupplier, ignore -> new LinkedHashSet<>()).add(clazz); + } + } + + public static Set> findParentSuppliers( + Class clazz + ) + { + return KNOWN_SUB_QUERY_COMPONENT_SUPPLIERS.get(clazz); + } + + public static List> getSuppliers(Class clazz) + { + ComponentSuppliers a = clazz.getAnnotation(ComponentSuppliers.class); + if (a == null || a.value().length == 0) { + throw DruidException.defensive("No component suppliers found [%s].", clazz.getName()); + } + return Arrays.asList(a.value()); + } + + private DruidException unsupportedException() + { + return DruidException.defensive("Unexpected call made to " + getClass().getName()); + } + + @Override + public void gatherProperties(Properties properties) + { + throw unsupportedException(); + } + + @Override + public DruidModule getCoreModule() + { + throw unsupportedException(); + } + + @Override + public DruidModule getOverrideModule() + { + throw unsupportedException(); + } + + @Override + public SpecificSegmentsQuerySegmentWalker addSegmentsToWalker(SpecificSegmentsQuerySegmentWalker walker) + { + throw unsupportedException(); + } + + @Override + public Class getSqlEngineClass() + { + throw unsupportedException(); + } + + @Override + public JoinableFactoryWrapper createJoinableFactoryWrapper(LookupExtractorFactoryContainerProvider lookupProvider) + { + throw unsupportedException(); + } + + @Override + public void finalizeTestFramework(SqlTestFramework sqlTestFramework) + { + throw unsupportedException(); + } + + @Override + public PlannerComponentSupplier getPlannerComponentSupplier() + { + throw unsupportedException(); + } + + @Override + public void configureGuice(DruidInjectorBuilder injectorBuilder, List overrideModules) + { + throw unsupportedException(); + } + + @Override + public Boolean isExplainSupported() + + { + throw unsupportedException(); + } + + @Override + public QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglomerate conglomerate, + Closer resourceCloser) + { + throw unsupportedException(); + } + + @Override + public TempDirProducer getTempDirProducer() + { + throw unsupportedException(); + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/QTestCase.java b/sql/src/test/java/org/apache/druid/sql/calcite/QTestCase.java index 01714525b664..67f5bf3c2eb7 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/QTestCase.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/QTestCase.java @@ -25,7 +25,6 @@ import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.quidem.DruidQTestInfo; -import org.apache.druid.quidem.DruidQuidemCommandHandler; import org.apache.druid.quidem.DruidQuidemTestBase; import org.apache.druid.quidem.DruidQuidemTestBase.DruidQuidemRunner; import org.apache.druid.sql.calcite.QueryTestRunner.QueryRunStep; @@ -70,7 +69,7 @@ public void run() isValidTestCaseFile(testInfo.getIQFile()); } - DruidQuidemRunner runner = new DruidQuidemTestBase.DruidQuidemRunner(new DruidQuidemCommandHandler()); + DruidQuidemRunner runner = new DruidQuidemTestBase.DruidQuidemRunner(); runner.run(testInfo.getIQFile()); } catch (Exception e) { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java b/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java index 8758871c3841..87919839a02d 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java @@ -27,10 +27,10 @@ import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import org.apache.druid.error.DruidException; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.topn.TopNQueryConfig; -import org.apache.druid.quidem.DruidAvaticaTestDriver; import org.apache.druid.server.QueryStackTests; import org.apache.druid.sql.calcite.util.CacheTestHelperModule.ResultCacheMode; import org.apache.druid.sql.calcite.util.FakeIndexTaskUtil; @@ -63,11 +63,11 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; -import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -458,20 +458,15 @@ private Map getNonDefaultMap() return map; } - public static SqlTestFrameworkConfig fromURL(String url) throws SQLException + public static SqlTestFrameworkConfig fromURL(String url) { Map queryParams; queryParams = new HashMap<>(); try { URI uri = new URI(url); - if (!DruidAvaticaTestDriver.SCHEME.equals(uri.getScheme())) { - throw new SQLException( - StringUtils.format("URI [%s] is invalid ; only scheme [%s] is supported.", url, DruidAvaticaTestDriver.SCHEME) - ); - } if (uri.getHost() != null || uri.getPort() != -1) { - throw new SQLException(StringUtils.format("URI [%s] is invalid ; only query parameters are supported.", url)); + throw DruidException.defensive("URI [%s] is invalid ; only query parameters are supported.", url); } List params = URLEncodedUtils.parse(uri, StandardCharsets.UTF_8); for (NameValuePair pair : params) { @@ -480,13 +475,13 @@ public static SqlTestFrameworkConfig fromURL(String url) throws SQLException // possible caveat: duplicate entries overwrite earlier ones } catch (URISyntaxException e) { - throw new SQLException("Can't decode URI", e); + throw DruidException.defensive(e, "Can't decode URI"); } return new SqlTestFrameworkConfig(queryParams); } - abstract static class ConfigOptionProcessor + public abstract static class ConfigOptionProcessor { final Class annotationClass; @@ -554,7 +549,21 @@ public Set> load(String pkg) .includePackage(pkg) .and(s -> s.contains("ComponentSupplier")) ); - return new Reflections(cfg).getSubTypesOf(QueryComponentSupplier.class); + final Set> baseComponentClazzes = + new Reflections(cfg).getSubTypesOf(QueryComponentSupplier.class); + LinkedHashSet> retVal = new LinkedHashSet<>(baseComponentClazzes); + + for (Class baseClazz : baseComponentClazzes) { + if (MultiComponentSupplier.class.isAssignableFrom(baseClazz) && baseClazz != MultiComponentSupplier.class) { + final Class multiClazz = + (Class) baseClazz; + + MultiComponentSupplier.registerComponentSupplier(multiClazz); + retVal.addAll(MultiComponentSupplier.getSuppliers(multiClazz)); + } + } + + return retVal; } }); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java index cb3ceaf442d7..b2c63d3b18b7 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java @@ -291,7 +291,11 @@ public static QueryLifecycleFactory createMockQueryLifecycleFactory( final QueryRunnerFactoryConglomerate conglomerate ) { - return QueryFrameworkUtils.createMockQueryLifecycleFactory(walker, conglomerate); + return QueryFrameworkUtils.createMockQueryLifecycleFactory( + walker, + conglomerate, + CalciteTests.TEST_AUTHORIZER_MAPPER + ); } public static SqlStatementFactory createSqlStatementFactory( @@ -396,6 +400,15 @@ NodeRole.COORDINATOR, new FakeDruidNodeDiscovery(ImmutableMap.of(NodeRole.COORDI return provider; } + public static SystemSchema createMockSystemSchema( + final DruidSchema druidSchema, + final SpecificSegmentsQuerySegmentWalker walker, + final AuthorizerMapper authorizerMapper + ) + { + return createMockSystemSchema(druidSchema, new TestTimelineServerView(walker.getSegments()), authorizerMapper); + } + public static SystemSchema createMockSystemSchema( final DruidSchema druidSchema, final TimelineServerView timelineServerView, diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/DruidModuleCollection.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/DruidModuleCollection.java index 764f190e41bf..ec728df96808 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/DruidModuleCollection.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/DruidModuleCollection.java @@ -70,12 +70,31 @@ public final void configure(Binder binder) public static DruidModule of(Module... modules) { - return new DruidModuleCollection(Arrays.asList(modules)); + return of(Arrays.asList(modules)); } - public static DruidModule of(List modules) + public static DruidModule of(List modules) { - return new DruidModuleCollection(modules); + return new DruidModuleCollection(flatten(modules)); + } + + public static List flatten(Module... modules) + { + return flatten(Arrays.asList(modules)); + } + + public static List flatten(List modules) + { + ArrayList flattenedModules = new ArrayList<>(modules.size()); + for (Module module : modules) { + if (module instanceof DruidModuleCollection) { + DruidModuleCollection moduleCollection = (DruidModuleCollection) module; + flattenedModules.addAll(moduleCollection.subModules); + } else { + flattenedModules.add(module); + } + } + return flattenedModules; } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java index f61f52343987..67b5b9d11e22 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java @@ -85,7 +85,8 @@ public class QueryFrameworkUtils public static QueryLifecycleFactory createMockQueryLifecycleFactory( final QuerySegmentWalker walker, - final QueryRunnerFactoryConglomerate conglomerate + final QueryRunnerFactoryConglomerate conglomerate, + final AuthorizerMapper authorizerMapper ) { return new QueryLifecycleFactory( @@ -95,7 +96,7 @@ public static QueryLifecycleFactory createMockQueryLifecycleFactory( new ServiceEmitter("dummy", "dummy", new NoopEmitter()), new NoopRequestLogger(), new AuthConfig(), - CalciteTests.TEST_AUTHORIZER_MAPPER, + authorizerMapper, Suppliers.ofInstance(new DefaultQueryConfig(ImmutableMap.of())) ); } @@ -179,7 +180,7 @@ public static DruidSchemaCatalog createMockRootSchema( ); } - static DruidSchemaCatalog createMockRootSchema( + public static DruidSchemaCatalog createMockRootSchema( final PlannerConfig plannerConfig, final ViewManager viewManager, final AuthorizerMapper authorizerMapper, @@ -243,7 +244,7 @@ public static DruidSchemaCatalog createMockRootSchema( ); } - static DruidSchema createMockSchema( + public static DruidSchema createMockSchema( final Injector injector, final QueryRunnerFactoryConglomerate conglomerate, final SpecificSegmentsQuerySegmentWalker walker, @@ -253,7 +254,7 @@ static DruidSchema createMockSchema( ) { final BrokerSegmentMetadataCache cache = new BrokerSegmentMetadataCache( - createMockQueryLifecycleFactory(walker, conglomerate), + createMockQueryLifecycleFactory(walker, conglomerate, CalciteTests.TEST_AUTHORIZER_MAPPER), timelineServerView, BrokerSegmentMetadataCacheConfig.create(), CalciteTests.TEST_AUTHENTICATOR_ESCALATOR, diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index e99550d08c15..7803b9082aae 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -28,7 +28,6 @@ import com.google.inject.Injector; import com.google.inject.Module; import com.google.inject.Provides; -import com.google.inject.TypeLiteral; import org.apache.druid.client.TestHttpClient; import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.cache.Cache; @@ -45,6 +44,7 @@ import org.apache.druid.guice.SegmentWranglerModule; import org.apache.druid.guice.ServerModule; import org.apache.druid.guice.StartupInjectorBuilder; +import org.apache.druid.guice.StorageNodeModule; import org.apache.druid.guice.annotations.Global; import org.apache.druid.guice.annotations.Merging; import org.apache.druid.guice.annotations.Self; @@ -56,9 +56,7 @@ import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.java.util.http.client.HttpClient; import org.apache.druid.math.expr.ExprMacroTable; -import org.apache.druid.query.DefaultGenericQueryMetricsFactory; import org.apache.druid.query.DruidProcessingConfig; -import org.apache.druid.query.GenericQueryMetricsFactory; import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QueryRunnerTestHelper; @@ -77,8 +75,6 @@ import org.apache.druid.query.topn.TopNQueryConfig; import org.apache.druid.quidem.ProjectPathUtils; import org.apache.druid.quidem.TestSqlModule; -import org.apache.druid.segment.DefaultColumnFormatConfig; -import org.apache.druid.segment.column.ColumnConfig; import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.realtime.ChatHandlerProvider; import org.apache.druid.segment.realtime.NoopChatHandlerProvider; @@ -87,7 +83,6 @@ import org.apache.druid.server.LocalQuerySegmentWalker; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; -import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.SubqueryGuardrailHelper; @@ -128,7 +123,6 @@ import org.apache.druid.utils.JvmUtils; import javax.inject.Named; - import java.io.Closeable; import java.io.File; import java.io.IOException; @@ -201,6 +195,8 @@ public interface QueryComponentSupplier extends Closeable */ void gatherProperties(Properties properties); + Class getSqlEngineClass(); + SpecificSegmentsQuerySegmentWalker addSegmentsToWalker(SpecificSegmentsQuerySegmentWalker walker); /** @@ -213,12 +209,6 @@ public interface QueryComponentSupplier extends Closeable */ DruidModule getOverrideModule(); - SqlEngine createEngine( - QueryLifecycleFactory qlf, - ObjectMapper objectMapper, - Injector injector - ); - default CatalogResolver createCatalogResolver() { return CatalogResolver.NULL_RESOLVER; @@ -226,8 +216,6 @@ default CatalogResolver createCatalogResolver() /** * Configure the JSON mapper. - * - * @see #configureGuice(DruidInjectorBuilder) for the preferred solution. */ @Deprecated default void configureJsonMapper(ObjectMapper mapper) @@ -239,6 +227,7 @@ default void configureJsonMapper(ObjectMapper mapper) void finalizeTestFramework(SqlTestFramework sqlTestFramework); PlannerComponentSupplier getPlannerComponentSupplier(); + @Override default void close() throws IOException { @@ -292,15 +281,6 @@ public SpecificSegmentsQuerySegmentWalker addSegmentsToWalker(SpecificSegmentsQu return delegate.addSegmentsToWalker(walker); } - @Override - public SqlEngine createEngine( - QueryLifecycleFactory qlf, - ObjectMapper objectMapper, - Injector injector) - { - return delegate.createEngine(qlf, objectMapper, injector); - } - @Override public void configureJsonMapper(ObjectMapper mapper) { @@ -338,8 +318,10 @@ public Boolean isExplainSupported() } @Override - public QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglomerate conglomerate, - Closer resourceCloser) + public QueryRunnerFactoryConglomerate wrapConglomerate( + QueryRunnerFactoryConglomerate conglomerate, + Closer resourceCloser + ) { return delegate.wrapConglomerate(conglomerate, resourceCloser); } @@ -356,6 +338,12 @@ public DruidModule getOverrideModule() return delegate.getOverrideModule(); } + @Override + public Class getSqlEngineClass() + { + return delegate.getSqlEngineClass(); + } + @Override public TempDirProducer getTempDirProducer() { @@ -417,23 +405,180 @@ public DruidModule getCoreModule() new LookylooModule(), new SegmentWranglerModule(), new ExpressionModule(), - new QueryRunnerFactoryModule(), + DruidModule.override( + new QueryRunnerFactoryModule(), + new Module() + { + @Override + public void configure(Binder binder) + { + + } + + @Provides + @Named("isExplainSupported") + public Boolean isExplainSupported(Builder builder) + { + return builder.componentSupplier.isExplainSupported(); + } + + @Provides + public QueryComponentSupplier getQueryComponentSupplier(Builder builder) + { + return builder.componentSupplier; + } + + @Provides + @LazySingleton + GroupByQueryMetricsFactory groupByQueryMetricsFactory() + { + return DefaultGroupByQueryMetricsFactory.instance(); + } + + @Provides + @LazySingleton + public TopNQueryConfig makeTopNQueryConfig(Builder builder) + { + return new TopNQueryConfig() + { + @Override + public int getMinTopNThreshold() + { + return builder.minTopNThreshold; + } + }; + } + + @Provides + public QueryWatcher getQueryWatcher() + { + return QueryRunnerTestHelper.NOOP_QUERYWATCHER; + } + } + ), new BuiltInTypesModule(), new TestSqlModule(), - new ServerModule(), + DruidModule.override( + new ServerModule(), + new Module() + { + @Provides + @Self + @LazySingleton + public DruidNode makeSelfDruidNode() + { + return new DruidNode("druid/broker", "local-test-host", false, 12345, 443, true, false); + } + + @Override + public void configure(Binder binder) + { + } + } + ), new LifecycleModule(), - new QueryableModule(), - new SqlModule() + DruidModule.override( + new QueryableModule(), + binder -> { + TestRequestLogger testRequestLogger = new TestRequestLogger(); + binder.bind(RequestLogger.class).toInstance(testRequestLogger); + } + ), + DruidModule.override( + new SqlModule(), + new Module() + { + @Override + public void configure(Binder binder) + { + } + + @Provides + @LazySingleton + ViewManager createViewManager(Builder builder) + { + return builder.componentSupplier.getPlannerComponentSupplier().createViewManager(); + } + + @Provides + @LazySingleton + private DruidSchema makeDruidSchema( + final Injector injector, + QueryRunnerFactoryConglomerate conglomerate, + QuerySegmentWalker walker, + Builder builder, + TimelineServerView timelineServerView + ) + { + return QueryFrameworkUtils.createMockSchema( + injector, + conglomerate, + (SpecificSegmentsQuerySegmentWalker) walker, + builder.componentSupplier.getPlannerComponentSupplier().createSchemaManager(), + builder.catalogResolver, + timelineServerView + ); + } + + @Provides + @LazySingleton + private SystemSchema makeSystemSchema( + AuthorizerMapper authorizerMapper, + DruidSchema druidSchema, + TimelineServerView timelineServerView) + { + return CalciteTests.createMockSystemSchema(druidSchema, timelineServerView, authorizerMapper); + } + + @Provides + @LazySingleton + private TimelineServerView makeTimelineServerView(SpecificSegmentsQuerySegmentWalker walker) + { + return new TestTimelineServerView(walker.getSegments()); + } + + @Provides + @LazySingleton + private LookupSchema makeLookupSchema(final Injector injector) + { + return QueryFrameworkUtils.createMockLookupSchema(injector); + } + + @Provides + @LazySingleton + private DruidSchemaCatalog makeCatalog( + final PlannerConfig plannerConfig, + final ViewManager viewManager, + AuthorizerMapper authorizerMapper, + DruidSchema druidSchema, + SystemSchema systemSchema, + LookupSchema lookupSchema, + DruidOperatorTable createOperatorTable + ) + { + final DruidSchemaCatalog rootSchema = QueryFrameworkUtils.createMockRootSchema( + plannerConfig, + viewManager, + authorizerMapper, + druidSchema, + systemSchema, + lookupSchema, + createOperatorTable + ); + return rootSchema; + } + } + ), + new TestSetupModule(), + new TestSchemaSetupModule(), + new StorageNodeModule() ); } @Override public DruidModule getOverrideModule() { - return DruidModuleCollection.of( - new TestSetupModule(), - new TestSchemaSetupModule() - ); + return DruidModuleCollection.of(); } /** @@ -460,16 +605,9 @@ public SpecificSegmentsQuerySegmentWalker addSegmentsToWalker(SpecificSegmentsQu } @Override - public SqlEngine createEngine( - QueryLifecycleFactory qlf, - ObjectMapper objectMapper, - Injector injector - ) + public Class getSqlEngineClass() { - return new NativeSqlEngine( - qlf, - objectMapper - ); + return NativeSqlEngine.class; } @Override @@ -508,8 +646,10 @@ public Boolean isExplainSupported() } @Override - public QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglomerate conglomerate, - Closer resourceCloser) + public QueryRunnerFactoryConglomerate wrapConglomerate( + QueryRunnerFactoryConglomerate conglomerate, + Closer resourceCloser + ) { return conglomerate; } @@ -648,6 +788,21 @@ public Builder withConfig(SqlTestFrameworkConfig config) this.config = config; return this; } + + public QueryComponentSupplier getComponentSupplier() + { + return componentSupplier; + } + + public CatalogResolver getCatalogResolver() + { + return catalogResolver; + } + + public Closer getResourceCloser() + { + return resourceCloser; + } } /** @@ -761,74 +916,52 @@ ChatHandlerProvider getChatHandlerProvider() return new NoopChatHandlerProvider(); } - @Provides - GenericQueryMetricsFactory getGenericQueryMetricsFactory() - { - return DefaultGenericQueryMetricsFactory.instance(); - } - @Override public void configure(Binder binder) { binder.bind(DruidOperatorTable.class).in(LazySingleton.class); binder.bind(DataSegment.PruneSpecsHolder.class).toInstance(DataSegment.PruneSpecsHolder.DEFAULT); - binder.bind(DefaultColumnFormatConfig.class).toInstance(new DefaultColumnFormatConfig(null, null)); - - binder.bind(new TypeLiteral>(){}) - .annotatedWith(Global.class) - .to(TestBufferPool.class); - - binder.bind(new TypeLiteral>(){}) - .annotatedWith(Merging.class) - .to(TestBufferPool.class); - - TestRequestLogger testRequestLogger = new TestRequestLogger(); - binder.bind(RequestLogger.class).toInstance(testRequestLogger); } @Provides - @Self - @LazySingleton - public DruidNode makeSelfDruidNode() + @Global + NonBlockingPool getGlobalPool(TestBufferPool pool) { - return new DruidNode("druid/broker", "local-test-host", false, 12345, 443, true, false); + return pool; } @Provides - AuthorizerMapper getAuthorizerMapper() + @Merging + BlockingPool getMergingPool(TestBufferPool pool) { - return AuthTestUtils.TEST_AUTHORIZER_MAPPER; + return pool; } @Provides - @LazySingleton - public TopNQueryConfig makeTopNQueryConfig(Builder builder) - { - return new TopNQueryConfig() - { - @Override - public int getMinTopNThreshold() - { - return builder.minTopNThreshold; - } - }; + AuthorizerMapper getAuthorizerMapper() + { + return AuthTestUtils.TEST_AUTHORIZER_MAPPER; } @Provides @LazySingleton private GroupByResourcesReservationPool makeGroupByResourcesReservationPool( final GroupByQueryConfig config, - final TestGroupByBuffers bufferPools) + final TestGroupByBuffers bufferPools + ) { return new GroupByResourcesReservationPool(bufferPools.getMergePool(), config); } @Provides @LazySingleton - private GroupingEngine makeGroupingEngine(final ObjectMapper mapper, final DruidProcessingConfig processingConfig, + private GroupingEngine makeGroupingEngine( + final ObjectMapper mapper, + final DruidProcessingConfig processingConfig, final GroupByStatsProvider statsProvider, final GroupByQueryConfig config, - final GroupByResourcesReservationPool groupByResourcesReservationPool) + final GroupByResourcesReservationPool groupByResourcesReservationPool + ) { final Supplier configSupplier = Suppliers.ofInstance(config); return new GroupingEngine( @@ -842,19 +975,12 @@ private GroupingEngine makeGroupingEngine(final ObjectMapper mapper, final Druid ); } - @Provides - @LazySingleton - GroupByQueryMetricsFactory groupByQueryMetricsFactory() - { - return DefaultGroupByQueryMetricsFactory.instance(); - } - - @Provides @LazySingleton @Merging GroupByResourcesReservationPool makeMergingGroupByResourcesReservationPool( - final GroupByResourcesReservationPool groupByResourcesReservationPool) + final GroupByResourcesReservationPool groupByResourcesReservationPool + ) { return groupByResourcesReservationPool; } @@ -895,17 +1021,11 @@ public QueryLifecycleFactory queryLifecycleFactory(final Injector injector) { return QueryFrameworkUtils.createMockQueryLifecycleFactory( injector.getInstance(QuerySegmentWalker.class), - injector.getInstance(QueryRunnerFactoryConglomerate.class) + injector.getInstance(QueryRunnerFactoryConglomerate.class), + injector.getInstance(AuthorizerMapper.class) ); } - @Provides - @LazySingleton - ViewManager createViewManager(Builder builder) - { - return builder.componentSupplier.getPlannerComponentSupplier().createViewManager(); - } - @Provides SqlTestFrameworkConfig getTestConfig(Builder builder) { @@ -918,91 +1038,10 @@ public URI getDruidTestURI(SqlTestFrameworkConfig config) { return config.getDruidTestURI(); } - - @Provides - @Named("isExplainSupported") - public Boolean isExplainSupported(Builder builder) - { - return builder.componentSupplier.isExplainSupported(); - } - - @Provides - public QueryWatcher getQueryWatcher() - { - return QueryRunnerTestHelper.NOOP_QUERYWATCHER; - } } public static class TestSchemaSetupModule implements DruidModule { - - @Provides - @LazySingleton - private DruidSchema makeDruidSchema( - final Injector injector, - QueryRunnerFactoryConglomerate conglomerate, - QuerySegmentWalker walker, - Builder builder, - TimelineServerView timelineServerView) - { - return QueryFrameworkUtils.createMockSchema( - injector, - conglomerate, - (SpecificSegmentsQuerySegmentWalker) walker, - builder.componentSupplier.getPlannerComponentSupplier().createSchemaManager(), - builder.catalogResolver, - timelineServerView - ); - } - - @Provides - @LazySingleton - private SystemSchema makeSystemSchema(AuthorizerMapper authorizerMapper, DruidSchema druidSchema, - TimelineServerView timelineServerView) - { - return CalciteTests.createMockSystemSchema(druidSchema, timelineServerView, authorizerMapper); - } - - @Provides - @LazySingleton - private TimelineServerView makeTimelineServerView(SpecificSegmentsQuerySegmentWalker walker) - { - return new TestTimelineServerView(walker.getSegments()); - } - - - @Provides - @LazySingleton - private ColumnConfig getColumnConfig() - { - return ColumnConfig.DEFAULT; - } - - @Provides - @LazySingleton - private LookupSchema makeLookupSchema(final Injector injector) - { - return QueryFrameworkUtils.createMockLookupSchema(injector); - } - - @Provides - @LazySingleton - private DruidSchemaCatalog makeCatalog(final PlannerConfig plannerConfig, final ViewManager viewManager, - AuthorizerMapper authorizerMapper, DruidSchema druidSchema, SystemSchema systemSchema, - LookupSchema lookupSchema, DruidOperatorTable createOperatorTable) - { - final DruidSchemaCatalog rootSchema = QueryFrameworkUtils.createMockRootSchema( - plannerConfig, - viewManager, - authorizerMapper, - druidSchema, - systemSchema, - lookupSchema, - createOperatorTable - ); - return rootSchema; - } - @Provides @LazySingleton public SpecificSegmentsQuerySegmentWalker specificSegmentsQuerySegmentWalker( @@ -1117,12 +1156,6 @@ public SubqueryGuardrailHelper makeSubqueryGuardrailHelper() return new SubqueryGuardrailHelper(null, JvmUtils.getRuntimeInfo().getMaxHeapSizeBytes(), 1); } - @Provides - public QueryScheduler makeQueryScheduler() - { - return QueryStackTests.DEFAULT_NOOP_SCHEDULER; - } - @Override public void configure(Binder binder) { @@ -1152,18 +1185,19 @@ private SqlTestFramework(Builder builder) // not the Druid node to which the module is scoped. .ignoreLoadScopes(); - ArrayList overrideModules = new ArrayList<>(builder.overrideModules); - - injectorBuilder.add(componentSupplier.getCoreModule()); + injectorBuilder.addAll(DruidModuleCollection.flatten(componentSupplier.getCoreModule())); injectorBuilder.addModule(binder -> binder.bind(Builder.class).toInstance(builder)); - overrideModules.add(componentSupplier.getOverrideModule()); + + ArrayList overrideModules = new ArrayList<>(builder.overrideModules); + overrideModules.addAll(DruidModuleCollection.flatten(componentSupplier.getOverrideModule())); builder.componentSupplier.configureGuice(injectorBuilder, overrideModules); ServiceInjectorBuilder serviceInjector = new ServiceInjectorBuilder(injectorBuilder); serviceInjector.addAll(overrideModules); this.injector = serviceInjector.build(); - this.engine = builder.componentSupplier.createEngine(queryLifecycleFactory(), queryJsonMapper(), injector); + this.engine = injector.getInstance(componentSupplier.getSqlEngineClass()); + componentSupplier.configureJsonMapper(queryJsonMapper()); componentSupplier.finalizeTestFramework(this); } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java index 4fd90a95c32b..309030f0280e 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java @@ -519,7 +519,7 @@ public Optional build( ) ); - private static List USER_VISIT_ROWS = ImmutableList.of( + public static List USER_VISIT_ROWS = ImmutableList.of( toRow( "2021-01-01T01:00:00Z", USER_VISIT_DIMS, @@ -1158,12 +1158,12 @@ private static MapBasedInputRow toRow(String time, List dimensions, Map< return new MapBasedInputRow(DateTimes.ISO_DATE_OPTIONAL_TIME.parse(time), dimensions, event); } - public static InputRow createRow(final ImmutableMap map) + public static InputRow createRow(final Map map) { return MapInputRowParser.parse(FOO_SCHEMA, (Map) map); } - public static InputRow createRow(final ImmutableMap map, InputRowSchema inputRowSchema) + public static InputRow createRow(final Map map, InputRowSchema inputRowSchema) { return MapInputRowParser.parse(inputRowSchema, (Map) map); } diff --git a/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/customdataset.iq b/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/customdataset.iq index dd3b0d43bcf4..d467cf3da286 100644 --- a/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/customdataset.iq +++ b/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/customdataset.iq @@ -6,7 +6,7 @@ select count(1) from "rollup-tutorial"; +--------+ | EXPR$0 | +--------+ -| 9 | +| 5 | +--------+ (1 row)