From 675c565333661a2d680f9c75ae3433223ac4cdcd Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 13 Nov 2025 12:13:30 -0800 Subject: [PATCH 01/10] fix msq compaction to apply security policies --- .../auth/AllowAllWithPolicyAuthResource.java | 183 ++++++++++++++++++ .../compact/CompactionSupervisorTest.java | 31 ++- ...rg.apache.druid.initialization.DruidModule | 1 + .../msq/indexing/MSQCompactionRunner.java | 28 +++ .../server/security/AuthorizationUtils.java | 2 +- 5 files changed, 243 insertions(+), 2 deletions(-) create mode 100644 embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AllowAllWithPolicyAuthResource.java diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AllowAllWithPolicyAuthResource.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AllowAllWithPolicyAuthResource.java new file mode 100644 index 000000000000..51094d0f3f8a --- /dev/null +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AllowAllWithPolicyAuthResource.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testing.embedded.auth; + +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.inject.Binder; +import com.google.inject.Key; +import com.google.inject.multibindings.MapBinder; +import org.apache.druid.guice.LazySingleton; +import org.apache.druid.guice.PolyBind; +import org.apache.druid.initialization.DruidModule; +import org.apache.druid.query.policy.NoRestrictionPolicy; +import org.apache.druid.server.security.Access; +import org.apache.druid.server.security.Action; +import org.apache.druid.server.security.AllowAllAuthenticator; +import org.apache.druid.server.security.AuthConfig; +import org.apache.druid.server.security.AuthenticationResult; +import org.apache.druid.server.security.Authenticator; +import org.apache.druid.server.security.AuthorizationUtils; +import org.apache.druid.server.security.Authorizer; +import org.apache.druid.server.security.NoopEscalator; +import org.apache.druid.server.security.Resource; +import org.apache.druid.testing.embedded.EmbeddedDruidCluster; +import org.apache.druid.testing.embedded.EmbeddedResource; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class AllowAllWithPolicyAuthResource implements EmbeddedResource +{ + private static final String AUTH_TYPE = "actually-allow-all"; + + @Override + public void start() throws Exception + { + // Do nothing + } + + @Override + public void stop() throws Exception + { + // Do nothing + } + + @Override + public void beforeStart(EmbeddedDruidCluster cluster) + { + cluster.addExtension(AllowAllWithPolicyAuthModule.class) + .addCommonProperty("druid.auth.authenticatorChain", "[\"test\"]") + .addCommonProperty("druid.auth.authenticator.test.type", "actually-allow-all") + .addCommonProperty("druid.auth.authorizers", "[\"test\"]") + .addCommonProperty("druid.escalator.type", "actually-allow-all") + .addCommonProperty("druid.auth.authorizer.test.type", "actually-allow-all") + .addCommonProperty("druid.policy.enforcer.type", "restrictAllTables") + .addCommonProperty( + "druid.policy.enforcer.allowedPolicies", + "[\"org.apache.druid.query.policy.NoRestrictionPolicy\"]" + ); + } + + @JsonTypeName(AUTH_TYPE) + static class AllowAllWithPolicyAuthenticator extends AllowAllAuthenticator + { + private static final AuthenticationResult RESULT = new AuthenticationResult("anon", "test", "test", null); + + @Override + public Filter getFilter() + { + return new Filter() + { + @Override + public void init(FilterConfig filterConfig) throws ServletException + { + + } + + @Override + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) + throws IOException, ServletException + { + servletRequest.setAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT, RESULT); + filterChain.doFilter(servletRequest, servletResponse); + } + + @Override + public void destroy() + { + + } + }; + } + + @Override + public AuthenticationResult authenticateJDBCContext(Map context) + { + return RESULT; + } + } + + @JsonTypeName(AUTH_TYPE) + static class AllowAllWithPolicyAuthorizer implements Authorizer + { + @Override + public Access authorize(AuthenticationResult authenticationResult, Resource resource, Action action) + { + if (AuthorizationUtils.RESTRICTION_APPLICABLE_RESOURCE_TYPES.contains(resource.getType()) && Action.READ.equals(action)) { + return Access.allowWithRestriction(NoRestrictionPolicy.instance()); + } + return Access.OK; + } + } + + @JsonTypeName(AUTH_TYPE) + static class AllowAllWithPolicyEscalator extends NoopEscalator + { + @Override + public AuthenticationResult createEscalatedAuthenticationResult() + { + return AllowAllWithPolicyAuthenticator.RESULT; + } + } + + @JsonTypeName(AUTH_TYPE) + public static class AllowAllWithPolicyAuthModule implements DruidModule + { + @Override + public void configure(Binder binder) + { + final MapBinder authenticatorMapBinder = PolyBind.optionBinder( + binder, + Key.get(Authenticator.class) + ); + authenticatorMapBinder.addBinding(AUTH_TYPE) + .to(AllowAllWithPolicyAuthenticator.class) + .in(LazySingleton.class); + final MapBinder authorizerMapBinder = PolyBind.optionBinder( + binder, + Key.get(Authorizer.class) + ); + authorizerMapBinder.addBinding(AUTH_TYPE) + .to(AllowAllWithPolicyAuthorizer.class) + .in(LazySingleton.class); + } + + @Override + public List getJacksonModules() + { + return List.of( + new SimpleModule().registerSubtypes( + AllowAllWithPolicyAuthenticator.class, + AllowAllWithPolicyAuthorizer.class, + AllowAllWithPolicyEscalator.class + ) + ); + } + } +} diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java index e11563c03897..034e4940cc01 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java @@ -22,6 +22,8 @@ import org.apache.druid.catalog.guice.CatalogClientModule; import org.apache.druid.catalog.guice.CatalogCoordinatorModule; import org.apache.druid.common.utils.IdUtils; +import org.apache.druid.indexer.CompactionEngine; +import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec; import org.apache.druid.indexing.common.task.IndexTask; import org.apache.druid.indexing.compact.CompactionSupervisorSpec; import org.apache.druid.indexing.overlord.Segments; @@ -38,6 +40,7 @@ import org.apache.druid.server.coordinator.DataSourceCompactionConfig; import org.apache.druid.server.coordinator.InlineSchemaDataSourceCompactionConfig; import org.apache.druid.server.coordinator.UserCompactionTaskGranularityConfig; +import org.apache.druid.server.coordinator.UserCompactionTaskQueryTuningConfig; import org.apache.druid.testing.embedded.EmbeddedBroker; import org.apache.druid.testing.embedded.EmbeddedCoordinator; import org.apache.druid.testing.embedded.EmbeddedDruidCluster; @@ -45,6 +48,7 @@ import org.apache.druid.testing.embedded.EmbeddedIndexer; import org.apache.druid.testing.embedded.EmbeddedOverlord; import org.apache.druid.testing.embedded.EmbeddedRouter; +import org.apache.druid.testing.embedded.auth.AllowAllWithPolicyAuthResource; import org.apache.druid.testing.embedded.indexing.MoreResources; import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase; import org.hamcrest.Matcher; @@ -54,6 +58,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import java.util.List; import java.util.Map; /** @@ -87,6 +92,7 @@ public EmbeddedDruidCluster createCluster() MSQSqlModule.class, SqlTaskModule.class ) + .addResource(new AllowAllWithPolicyAuthResource()) .addServer(coordinator) .addServer(overlord) .addServer(indexer) @@ -99,7 +105,7 @@ public EmbeddedDruidCluster createCluster() public void enableCompactionSupervisors() { final UpdateResponse updateResponse = cluster.callApi().onLeaderOverlord( - o -> o.updateClusterCompactionConfig(new ClusterCompactionConfig(1.0, 100, null, true, null)) + o -> o.updateClusterCompactionConfig(new ClusterCompactionConfig(1.0, 100, null, true, CompactionEngine.MSQ)) ); Assertions.assertTrue(updateResponse.isSuccess()); } @@ -125,6 +131,29 @@ public void test_ingestDayGranularity_andCompactToMonthGranularity_withInlineCon .withGranularitySpec( new UserCompactionTaskGranularityConfig(Granularities.MONTH, null, null) ) + .withTuningConfig( + new UserCompactionTaskQueryTuningConfig( + null, + null, + null, + null, + null, + new DimensionRangePartitionsSpec(null, 5000, List.of("item"), false), + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ) .build(); runCompactionWithSpec(compactionConfig); diff --git a/embedded-tests/src/test/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/embedded-tests/src/test/resources/META-INF/services/org.apache.druid.initialization.DruidModule index c38b51514eae..2ffbcd69ed6d 100644 --- a/embedded-tests/src/test/resources/META-INF/services/org.apache.druid.initialization.DruidModule +++ b/embedded-tests/src/test/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -18,3 +18,4 @@ # org.apache.druid.testing.embedded.gcs.GoogleStorageTestModule +org.apache.druid.testing.embedded.auth.AllowAllWithPolicyAuthResource$AllowAllWithPolicyAuthModule diff --git a/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java b/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java index a4adb11ccc6e..ab552e43793a 100644 --- a/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java +++ b/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java @@ -63,6 +63,7 @@ import org.apache.druid.query.groupby.GroupByQuery; import org.apache.druid.query.groupby.GroupByQueryConfig; import org.apache.druid.query.groupby.orderby.OrderByColumnSpec; +import org.apache.druid.query.policy.PolicyEnforcer; import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.VirtualColumns; @@ -73,6 +74,14 @@ import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.server.coordinator.CompactionConfigValidationResult; +import org.apache.druid.server.security.Action; +import org.apache.druid.server.security.AuthorizationResult; +import org.apache.druid.server.security.AuthorizationUtils; +import org.apache.druid.server.security.AuthorizerMapper; +import org.apache.druid.server.security.Escalator; +import org.apache.druid.server.security.Resource; +import org.apache.druid.server.security.ResourceAction; +import org.apache.druid.server.security.ResourceType; import org.apache.druid.sql.calcite.parser.DruidSqlInsert; import org.apache.druid.sql.calcite.planner.ColumnMapping; import org.apache.druid.sql.calcite.planner.ColumnMappings; @@ -263,6 +272,9 @@ public List createMsqControllerTasks( query = buildScanQuery(compactionTask, interval, dataSchema, inputColToVirtualCol); } QueryContext compactionTaskContext = new QueryContext(compactionTask.getContext()); + + query = maybeApplyPolicy(dataSchema, query); + DataSourceMSQDestination destination = buildMSQDestination(compactionTask, dataSchema); boolean isReindex = MSQControllerTask.isReplaceInputDataSourceTask(query, destination); @@ -293,6 +305,22 @@ public List createMsqControllerTasks( return msqControllerTasks; } + private Query maybeApplyPolicy(DataSchema dataSchema, Query query) + { + final Escalator escalator = injector.getInstance(Escalator.class); + if (escalator != null) { + final AuthorizerMapper authorizerMapper = injector.getInstance(AuthorizerMapper.class); + final PolicyEnforcer policyEnforcer = injector.getInstance(PolicyEnforcer.class); + final AuthorizationResult authResult = AuthorizationUtils.authorizeAllResourceActions( + escalator.createEscalatedAuthenticationResult(), + List.of(new ResourceAction(new Resource(dataSchema.getDataSource(), ResourceType.DATASOURCE), Action.READ)), + authorizerMapper + ); + query = query.withDataSource(query.getDataSource().withPolicies(authResult.getPolicyMap(), policyEnforcer)); + } + return query; + } + private static DataSourceMSQDestination buildMSQDestination( CompactionTask compactionTask, DataSchema dataSchema diff --git a/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java b/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java index 2bdd0d586b75..73ae57930634 100644 --- a/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java +++ b/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java @@ -46,7 +46,7 @@ */ public class AuthorizationUtils { - static final ImmutableSet RESTRICTION_APPLICABLE_RESOURCE_TYPES = ImmutableSet.of( + public static final ImmutableSet RESTRICTION_APPLICABLE_RESOURCE_TYPES = ImmutableSet.of( ResourceType.DATASOURCE ); From 80699b6b7cfc5451a0d4ddd135d92497bf4675c4 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 13 Nov 2025 13:30:04 -0800 Subject: [PATCH 02/10] trick style bot --- .../testing/embedded/auth/AllowAllWithPolicyAuthResource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AllowAllWithPolicyAuthResource.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AllowAllWithPolicyAuthResource.java index 51094d0f3f8a..1c41ce22eb69 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AllowAllWithPolicyAuthResource.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AllowAllWithPolicyAuthResource.java @@ -77,11 +77,11 @@ public void beforeStart(EmbeddedDruidCluster cluster) .addCommonProperty("druid.auth.authorizers", "[\"test\"]") .addCommonProperty("druid.escalator.type", "actually-allow-all") .addCommonProperty("druid.auth.authorizer.test.type", "actually-allow-all") - .addCommonProperty("druid.policy.enforcer.type", "restrictAllTables") .addCommonProperty( "druid.policy.enforcer.allowedPolicies", "[\"org.apache.druid.query.policy.NoRestrictionPolicy\"]" - ); + ) + .addCommonProperty("druid.policy.enforcer.type", "restrictAllTables"); } @JsonTypeName(AUTH_TYPE) From 5eac83f1f6d697c996e581efcec51ddaaff5e3c9 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 13 Nov 2025 13:57:23 -0800 Subject: [PATCH 03/10] dependencies --- embedded-tests/pom.xml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/embedded-tests/pom.xml b/embedded-tests/pom.xml index 9a3fbd90765e..4cb86bcedbe9 100644 --- a/embedded-tests/pom.xml +++ b/embedded-tests/pom.xml @@ -547,7 +547,11 @@ 5.5 test - + + javax.servlet + javax.servlet-api + test + From a6388d20fba0287a7f5083b836efb7791522ed8c Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 13 Nov 2025 14:31:34 -0800 Subject: [PATCH 04/10] bots bots bots --- .../testing/embedded/auth/AllowAllWithPolicyAuthResource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AllowAllWithPolicyAuthResource.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AllowAllWithPolicyAuthResource.java index 1c41ce22eb69..60543409cd13 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AllowAllWithPolicyAuthResource.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AllowAllWithPolicyAuthResource.java @@ -75,8 +75,8 @@ public void beforeStart(EmbeddedDruidCluster cluster) .addCommonProperty("druid.auth.authenticatorChain", "[\"test\"]") .addCommonProperty("druid.auth.authenticator.test.type", "actually-allow-all") .addCommonProperty("druid.auth.authorizers", "[\"test\"]") - .addCommonProperty("druid.escalator.type", "actually-allow-all") .addCommonProperty("druid.auth.authorizer.test.type", "actually-allow-all") + .addCommonProperty("druid.escalator.type", "actually-allow-all") .addCommonProperty( "druid.policy.enforcer.allowedPolicies", "[\"org.apache.druid.query.policy.NoRestrictionPolicy\"]" @@ -95,7 +95,7 @@ public Filter getFilter() return new Filter() { @Override - public void init(FilterConfig filterConfig) throws ServletException + public void init(FilterConfig filterConfig) { } From 0637446ab3e8f405f2ff14cf5afba59ca1b622ce Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 13 Nov 2025 20:03:42 -0800 Subject: [PATCH 05/10] fix test --- .../msq/indexing/MSQCompactionRunnerTest.java | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java b/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java index ef417a8b838f..c9bf384569d7 100644 --- a/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java +++ b/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java @@ -32,6 +32,9 @@ import org.apache.druid.data.input.impl.LongDimensionSchema; import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.data.input.impl.TimestampSpec; +import org.apache.druid.guice.StartupInjectorBuilder; +import org.apache.druid.guice.security.EscalatorModule; +import org.apache.druid.guice.security.PolicyModule; import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexer.granularity.UniformGranularitySpec; import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec; @@ -41,6 +44,7 @@ import org.apache.druid.indexing.common.task.CompactionIntervalSpec; import org.apache.druid.indexing.common.task.CompactionTask; import org.apache.druid.indexing.common.task.TuningConfigBuilder; +import org.apache.druid.initialization.CoreInjectorBuilder; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; @@ -74,6 +78,7 @@ import org.apache.druid.segment.transform.CompactionTransformSpec; import org.apache.druid.segment.transform.TransformSpec; import org.apache.druid.server.coordinator.CompactionConfigValidationResult; +import org.apache.druid.server.initialization.AuthorizerMapperModule; import org.apache.druid.sql.calcite.parser.DruidSqlInsert; import org.joda.time.Interval; import org.junit.Assert; @@ -143,19 +148,6 @@ public class MSQCompactionRunnerTest ImmutableSet.of(MV_STRING_DIMENSION.getName()) ) ); - private static final Map INTERVAL_DATASCHEMAS_WITH_PROJECTION = ImmutableMap.of( - COMPACTION_INTERVAL, - new CombinedDataSchema( - DATA_SOURCE, - new TimestampSpec(TIMESTAMP_COLUMN, null, null), - new DimensionsSpec(DIMENSIONS), - null, - null, - null, - ImmutableList.of(PROJECTION_SPEC), - ImmutableSet.of(MV_STRING_DIMENSION.getName()) - ) - ); private static final ObjectMapper JSON_MAPPER = new DefaultObjectMapper(); private static final AggregatorFactory AGG1 = new CountAggregatorFactory("agg_0"); private static final AggregatorFactory AGG2 = new LongSumAggregatorFactory("sum_added", "sum_added"); @@ -163,7 +155,11 @@ public class MSQCompactionRunnerTest private static final MSQCompactionRunner MSQ_COMPACTION_RUNNER = new MSQCompactionRunner( JSON_MAPPER, TestExprMacroTable.INSTANCE, - null + new CoreInjectorBuilder(new StartupInjectorBuilder().forTests().build()).addModules( + new EscalatorModule(), + new AuthorizerMapperModule(), + new PolicyModule() + ).build() ); private static final List PARTITION_DIMENSIONS = Collections.singletonList(STRING_DIMENSION.getName()); From 6dd22abb96c3a253025b8149894d8d832da5e002 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 14 Nov 2025 13:32:55 -0800 Subject: [PATCH 06/10] allowAll now can allowAll by optionally setting a policy, delete test only auth stuff since no longer needed --- .../auth/AllowAllWithPolicyAuthResource.java | 183 ------------------ .../compact/CompactionSupervisorTest.java | 33 +++- ...rg.apache.druid.initialization.DruidModule | 1 - .../http/OverlordCompactionResourceTest.java | 2 +- .../msq/indexing/MSQCompactionRunner.java | 5 + .../guice/security/AuthorizerModule.java | 30 +-- .../AuthorizerMapperModule.java | 2 +- .../server/security/AllowAllAuthorizer.java | 25 +++ .../druid/server/security/AuthTestUtils.java | 2 +- .../server/security/AuthorizationUtils.java | 8 +- .../AsyncManagementForwardingServletTest.java | 2 +- .../security/AuthorizationUtilsTest.java | 2 +- .../AsyncQueryForwardingServletTest.java | 2 +- 13 files changed, 85 insertions(+), 212 deletions(-) delete mode 100644 embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AllowAllWithPolicyAuthResource.java diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AllowAllWithPolicyAuthResource.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AllowAllWithPolicyAuthResource.java deleted file mode 100644 index 60543409cd13..000000000000 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AllowAllWithPolicyAuthResource.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.testing.embedded.auth; - -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.fasterxml.jackson.databind.Module; -import com.fasterxml.jackson.databind.module.SimpleModule; -import com.google.inject.Binder; -import com.google.inject.Key; -import com.google.inject.multibindings.MapBinder; -import org.apache.druid.guice.LazySingleton; -import org.apache.druid.guice.PolyBind; -import org.apache.druid.initialization.DruidModule; -import org.apache.druid.query.policy.NoRestrictionPolicy; -import org.apache.druid.server.security.Access; -import org.apache.druid.server.security.Action; -import org.apache.druid.server.security.AllowAllAuthenticator; -import org.apache.druid.server.security.AuthConfig; -import org.apache.druid.server.security.AuthenticationResult; -import org.apache.druid.server.security.Authenticator; -import org.apache.druid.server.security.AuthorizationUtils; -import org.apache.druid.server.security.Authorizer; -import org.apache.druid.server.security.NoopEscalator; -import org.apache.druid.server.security.Resource; -import org.apache.druid.testing.embedded.EmbeddedDruidCluster; -import org.apache.druid.testing.embedded.EmbeddedResource; - -import javax.servlet.Filter; -import javax.servlet.FilterChain; -import javax.servlet.FilterConfig; -import javax.servlet.ServletException; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; -import java.io.IOException; -import java.util.List; -import java.util.Map; - -public class AllowAllWithPolicyAuthResource implements EmbeddedResource -{ - private static final String AUTH_TYPE = "actually-allow-all"; - - @Override - public void start() throws Exception - { - // Do nothing - } - - @Override - public void stop() throws Exception - { - // Do nothing - } - - @Override - public void beforeStart(EmbeddedDruidCluster cluster) - { - cluster.addExtension(AllowAllWithPolicyAuthModule.class) - .addCommonProperty("druid.auth.authenticatorChain", "[\"test\"]") - .addCommonProperty("druid.auth.authenticator.test.type", "actually-allow-all") - .addCommonProperty("druid.auth.authorizers", "[\"test\"]") - .addCommonProperty("druid.auth.authorizer.test.type", "actually-allow-all") - .addCommonProperty("druid.escalator.type", "actually-allow-all") - .addCommonProperty( - "druid.policy.enforcer.allowedPolicies", - "[\"org.apache.druid.query.policy.NoRestrictionPolicy\"]" - ) - .addCommonProperty("druid.policy.enforcer.type", "restrictAllTables"); - } - - @JsonTypeName(AUTH_TYPE) - static class AllowAllWithPolicyAuthenticator extends AllowAllAuthenticator - { - private static final AuthenticationResult RESULT = new AuthenticationResult("anon", "test", "test", null); - - @Override - public Filter getFilter() - { - return new Filter() - { - @Override - public void init(FilterConfig filterConfig) - { - - } - - @Override - public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) - throws IOException, ServletException - { - servletRequest.setAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT, RESULT); - filterChain.doFilter(servletRequest, servletResponse); - } - - @Override - public void destroy() - { - - } - }; - } - - @Override - public AuthenticationResult authenticateJDBCContext(Map context) - { - return RESULT; - } - } - - @JsonTypeName(AUTH_TYPE) - static class AllowAllWithPolicyAuthorizer implements Authorizer - { - @Override - public Access authorize(AuthenticationResult authenticationResult, Resource resource, Action action) - { - if (AuthorizationUtils.RESTRICTION_APPLICABLE_RESOURCE_TYPES.contains(resource.getType()) && Action.READ.equals(action)) { - return Access.allowWithRestriction(NoRestrictionPolicy.instance()); - } - return Access.OK; - } - } - - @JsonTypeName(AUTH_TYPE) - static class AllowAllWithPolicyEscalator extends NoopEscalator - { - @Override - public AuthenticationResult createEscalatedAuthenticationResult() - { - return AllowAllWithPolicyAuthenticator.RESULT; - } - } - - @JsonTypeName(AUTH_TYPE) - public static class AllowAllWithPolicyAuthModule implements DruidModule - { - @Override - public void configure(Binder binder) - { - final MapBinder authenticatorMapBinder = PolyBind.optionBinder( - binder, - Key.get(Authenticator.class) - ); - authenticatorMapBinder.addBinding(AUTH_TYPE) - .to(AllowAllWithPolicyAuthenticator.class) - .in(LazySingleton.class); - final MapBinder authorizerMapBinder = PolyBind.optionBinder( - binder, - Key.get(Authorizer.class) - ); - authorizerMapBinder.addBinding(AUTH_TYPE) - .to(AllowAllWithPolicyAuthorizer.class) - .in(LazySingleton.class); - } - - @Override - public List getJacksonModules() - { - return List.of( - new SimpleModule().registerSubtypes( - AllowAllWithPolicyAuthenticator.class, - AllowAllWithPolicyAuthorizer.class, - AllowAllWithPolicyEscalator.class - ) - ); - } - } -} diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java index 034e4940cc01..65f7c75aa113 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java @@ -48,15 +48,14 @@ import org.apache.druid.testing.embedded.EmbeddedIndexer; import org.apache.druid.testing.embedded.EmbeddedOverlord; import org.apache.druid.testing.embedded.EmbeddedRouter; -import org.apache.druid.testing.embedded.auth.AllowAllWithPolicyAuthResource; import org.apache.druid.testing.embedded.indexing.MoreResources; import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase; import org.hamcrest.Matcher; import org.hamcrest.Matchers; import org.joda.time.Period; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import java.util.List; import java.util.Map; @@ -83,6 +82,14 @@ public EmbeddedDruidCluster createCluster() return EmbeddedDruidCluster.withEmbeddedDerbyAndZookeeper() .useLatchableEmitter() .useDefaultTimeoutForLatchableEmitter(600) + .addCommonProperty("druid.auth.authorizers", "[\"allowAll\"]") + .addCommonProperty("druid.auth.authorizer.allowAll.type", "allowAll") + .addCommonProperty("druid.auth.authorizer.allowAll.policy.type", "noRestriction") + .addCommonProperty( + "druid.policy.enforcer.allowedPolicies", + "[\"org.apache.druid.query.policy.NoRestrictionPolicy\"]" + ) + .addCommonProperty("druid.policy.enforcer.type", "restrictAllTables") .addExtensions( CatalogClientModule.class, CatalogCoordinatorModule.class, @@ -92,7 +99,6 @@ public EmbeddedDruidCluster createCluster() MSQSqlModule.class, SqlTaskModule.class ) - .addResource(new AllowAllWithPolicyAuthResource()) .addServer(coordinator) .addServer(overlord) .addServer(indexer) @@ -101,18 +107,21 @@ public EmbeddedDruidCluster createCluster() .addServer(new EmbeddedRouter()); } - @BeforeAll - public void enableCompactionSupervisors() + + private void configureCompaction(CompactionEngine compactionEngine) { final UpdateResponse updateResponse = cluster.callApi().onLeaderOverlord( - o -> o.updateClusterCompactionConfig(new ClusterCompactionConfig(1.0, 100, null, true, CompactionEngine.MSQ)) + o -> o.updateClusterCompactionConfig(new ClusterCompactionConfig(1.0, 100, null, true, compactionEngine)) ); Assertions.assertTrue(updateResponse.isSuccess()); } - @Test - public void test_ingestDayGranularity_andCompactToMonthGranularity_withInlineConfig() + @MethodSource("getEngine") + @ParameterizedTest(name = "compactionEngine={0}") + public void test_ingestDayGranularity_andCompactToMonthGranularity_withInlineConfig(CompactionEngine compactionEngine) { + configureCompaction(compactionEngine); + // Ingest data at DAY granularity and verify runIngestionAtGranularity( "DAY", @@ -221,4 +230,10 @@ private void runIngestionAtGranularity( .withId(IdUtils.getRandomId()); cluster.callApi().runTask(task, overlord); } + + public static List getEngine() + { + return List.of(CompactionEngine.NATIVE, CompactionEngine.MSQ); + } + } diff --git a/embedded-tests/src/test/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/embedded-tests/src/test/resources/META-INF/services/org.apache.druid.initialization.DruidModule index 2ffbcd69ed6d..c38b51514eae 100644 --- a/embedded-tests/src/test/resources/META-INF/services/org.apache.druid.initialization.DruidModule +++ b/embedded-tests/src/test/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -18,4 +18,3 @@ # org.apache.druid.testing.embedded.gcs.GoogleStorageTestModule -org.apache.druid.testing.embedded.auth.AllowAllWithPolicyAuthResource$AllowAllWithPolicyAuthModule diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResourceTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResourceTest.java index fc20739af156..a54459ad8891 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResourceTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResourceTest.java @@ -99,7 +99,7 @@ public void setUp() httpRequest = EasyMock.createStrictMock(HttpServletRequest.class); authorizerMapper = EasyMock.createStrictMock(AuthorizerMapper.class); - EasyMock.expect(authorizerMapper.getAuthorizer("druid")).andReturn(new AllowAllAuthorizer()).anyTimes(); + EasyMock.expect(authorizerMapper.getAuthorizer("druid")).andReturn(new AllowAllAuthorizer(null)).anyTimes(); taskMaster = EasyMock.createStrictMock(TaskMaster.class); EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.of(supervisorManager)).anyTimes(); diff --git a/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java b/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java index ab552e43793a..f243e2870975 100644 --- a/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java +++ b/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java @@ -305,6 +305,11 @@ public List createMsqControllerTasks( return msqControllerTasks; } + /** + * Creates a system {@link AuthorizationResult} using an {@link Escalator} and applies any resulting + * {@link org.apache.druid.query.policy.Policy} to rewrite the query with + * {@link org.apache.druid.query.DataSource#withPolicies(Map, PolicyEnforcer)} + */ private Query maybeApplyPolicy(DataSchema dataSchema, Query query) { final Escalator escalator = injector.getInstance(Escalator.class); diff --git a/server/src/main/java/org/apache/druid/guice/security/AuthorizerModule.java b/server/src/main/java/org/apache/druid/guice/security/AuthorizerModule.java index 95ac8fcf5c39..36463442a41a 100644 --- a/server/src/main/java/org/apache/druid/guice/security/AuthorizerModule.java +++ b/server/src/main/java/org/apache/druid/guice/security/AuthorizerModule.java @@ -19,34 +19,42 @@ package org.apache.druid.guice.security; +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.inject.Binder; -import com.google.inject.Key; -import com.google.inject.Module; import com.google.inject.Provides; -import com.google.inject.multibindings.MapBinder; import com.google.inject.name.Named; -import org.apache.druid.guice.LazySingleton; -import org.apache.druid.guice.PolyBind; +import org.apache.druid.initialization.DruidModule; import org.apache.druid.server.security.AllowAllAuthorizer; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.Authorizer; -public class AuthorizerModule implements Module +import java.util.List; + +public class AuthorizerModule implements DruidModule { @Override public void configure(Binder binder) { - final MapBinder authorizerMapBinder = PolyBind.optionBinder( - binder, - Key.get(Authorizer.class) +// final MapBinder authorizerMapBinder = PolyBind.optionBinder( +// binder, +// Key.get(Authorizer.class) +// ); +// authorizerMapBinder.addBinding(AuthConfig.ALLOW_ALL_NAME).to(AllowAllAuthorizer.class).in(LazySingleton.class); + } + + @Override + public List getJacksonModules() + { + return List.of( + new SimpleModule().registerSubtypes(AllowAllAuthorizer.class) ); - authorizerMapBinder.addBinding(AuthConfig.ALLOW_ALL_NAME).to(AllowAllAuthorizer.class).in(LazySingleton.class); } @Provides @Named(AuthConfig.ALLOW_ALL_NAME) public Authorizer getAuthorizer() { - return new AllowAllAuthorizer(); + return new AllowAllAuthorizer(null); } } diff --git a/server/src/main/java/org/apache/druid/server/initialization/AuthorizerMapperModule.java b/server/src/main/java/org/apache/druid/server/initialization/AuthorizerMapperModule.java index 3d3507b0c36e..9e35b32d8f19 100644 --- a/server/src/main/java/org/apache/druid/server/initialization/AuthorizerMapperModule.java +++ b/server/src/main/java/org/apache/druid/server/initialization/AuthorizerMapperModule.java @@ -85,7 +85,7 @@ public AuthorizerMapper get() // Default is allow all if (authorizers == null) { - AllowAllAuthorizer allowAllAuthorizer = new AllowAllAuthorizer(); + AllowAllAuthorizer allowAllAuthorizer = new AllowAllAuthorizer(null); authorizerMap.put(AuthConfig.ALLOW_ALL_NAME, allowAllAuthorizer); return new AuthorizerMapper(null) diff --git a/server/src/main/java/org/apache/druid/server/security/AllowAllAuthorizer.java b/server/src/main/java/org/apache/druid/server/security/AllowAllAuthorizer.java index 6271228b33de..34f8fabda679 100644 --- a/server/src/main/java/org/apache/druid/server/security/AllowAllAuthorizer.java +++ b/server/src/main/java/org/apache/druid/server/security/AllowAllAuthorizer.java @@ -19,11 +19,36 @@ package org.apache.druid.server.security; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.query.policy.Policy; + +import javax.annotation.Nullable; + public class AllowAllAuthorizer implements Authorizer { + @Nullable + private final Policy policy; + + @JsonCreator + public AllowAllAuthorizer( + @JsonProperty("policy") @Nullable Policy policy + ) + { + this.policy = policy; + } + @Override public Access authorize(AuthenticationResult authenticationResult, Resource resource, Action action) { + if (shouldApplyPolicy(resource, action)) { + return Access.allowWithRestriction(policy); + } return Access.OK; } + + private boolean shouldApplyPolicy(Resource resource, Action action) + { + return policy != null && AuthorizationUtils.shouldApplyPolicy(resource, action); + } } diff --git a/server/src/main/java/org/apache/druid/server/security/AuthTestUtils.java b/server/src/main/java/org/apache/druid/server/security/AuthTestUtils.java index 5922a5d234c4..5ba4b6fb47bc 100644 --- a/server/src/main/java/org/apache/druid/server/security/AuthTestUtils.java +++ b/server/src/main/java/org/apache/druid/server/security/AuthTestUtils.java @@ -36,7 +36,7 @@ public class AuthTestUtils @Override public Authorizer getAuthorizer(String name) { - return new AllowAllAuthorizer(); + return new AllowAllAuthorizer(null); } }; } diff --git a/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java b/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java index 73ae57930634..cb142ef5c06d 100644 --- a/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java +++ b/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java @@ -204,8 +204,7 @@ public static AuthorizationResult authorizeAllResourceActions( return AuthorizationResult.deny(access.getMessage()); } else { resultCache.add(resourceAction); - if (resourceAction.getAction().equals(Action.READ) - && RESTRICTION_APPLICABLE_RESOURCE_TYPES.contains(resourceAction.getResource().getType())) { + if (shouldApplyPolicy(resourceAction.getResource(), resourceAction.getAction())) { // For every table read, we check on the policy returned from authorizer and add it to the map. policyFilters.put(resourceAction.getResource().getName(), access.getPolicy()); } else if (access.getPolicy().isPresent()) { @@ -222,6 +221,11 @@ public static AuthorizationResult authorizeAllResourceActions( return AuthorizationResult.allowWithRestriction(policyFilters); } + public static boolean shouldApplyPolicy(Resource resource, Action action) + { + return Action.READ.equals(action) || RESTRICTION_APPLICABLE_RESOURCE_TYPES.contains(resource.getType()); + } + /** * Performs authorization check on a list of resource-actions based on the authentication fields from the request. diff --git a/server/src/test/java/org/apache/druid/server/AsyncManagementForwardingServletTest.java b/server/src/test/java/org/apache/druid/server/AsyncManagementForwardingServletTest.java index c576f966cf65..5b414d6f3cb0 100644 --- a/server/src/test/java/org/apache/druid/server/AsyncManagementForwardingServletTest.java +++ b/server/src/test/java/org/apache/druid/server/AsyncManagementForwardingServletTest.java @@ -518,7 +518,7 @@ public String getCurrentLeader() injector.getInstance(DruidHttpClientConfig.class), coordinatorLeaderSelector, overlordLeaderSelector, - new AuthorizerMapper(ImmutableMap.of("allowAll", new AllowAllAuthorizer())) + new AuthorizerMapper(ImmutableMap.of("allowAll", new AllowAllAuthorizer(null))) ) ); diff --git a/server/src/test/java/org/apache/druid/server/security/AuthorizationUtilsTest.java b/server/src/test/java/org/apache/druid/server/security/AuthorizationUtilsTest.java index 2d61ba586a7a..fd07ae170537 100644 --- a/server/src/test/java/org/apache/druid/server/security/AuthorizationUtilsTest.java +++ b/server/src/test/java/org/apache/druid/server/security/AuthorizationUtilsTest.java @@ -66,7 +66,7 @@ public Iterable apply(@Nullable String input) }; Map authorizerMap = new HashMap<>(); - authorizerMap.put(authorizerName, new AllowAllAuthorizer()); + authorizerMap.put(authorizerName, new AllowAllAuthorizer(null)); AuthorizerMapper mapper = new AuthorizerMapper(authorizerMap); diff --git a/services/src/test/java/org/apache/druid/server/AsyncQueryForwardingServletTest.java b/services/src/test/java/org/apache/druid/server/AsyncQueryForwardingServletTest.java index 372ad96174c7..24416859ea2f 100644 --- a/services/src/test/java/org/apache/druid/server/AsyncQueryForwardingServletTest.java +++ b/services/src/test/java/org/apache/druid/server/AsyncQueryForwardingServletTest.java @@ -170,7 +170,7 @@ public void configure(Binder binder) @Override public Authorizer getAuthorizer(String name) { - return new AllowAllAuthorizer(); + return new AllowAllAuthorizer(null); } } ); From 4ba10dbb0d938ba4b249f17e4cb6ff583a032d0a Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 14 Nov 2025 13:35:11 -0800 Subject: [PATCH 07/10] oops, cleanup --- .../org/apache/druid/guice/security/AuthorizerModule.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/server/src/main/java/org/apache/druid/guice/security/AuthorizerModule.java b/server/src/main/java/org/apache/druid/guice/security/AuthorizerModule.java index 36463442a41a..eccfb23b99d3 100644 --- a/server/src/main/java/org/apache/druid/guice/security/AuthorizerModule.java +++ b/server/src/main/java/org/apache/druid/guice/security/AuthorizerModule.java @@ -36,11 +36,6 @@ public class AuthorizerModule implements DruidModule @Override public void configure(Binder binder) { -// final MapBinder authorizerMapBinder = PolyBind.optionBinder( -// binder, -// Key.get(Authorizer.class) -// ); -// authorizerMapBinder.addBinding(AuthConfig.ALLOW_ALL_NAME).to(AllowAllAuthorizer.class).in(LazySingleton.class); } @Override From a686f0d697da2545e0eabd26dff010668b731350 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 14 Nov 2025 13:36:47 -0800 Subject: [PATCH 08/10] unused dependency --- embedded-tests/pom.xml | 5 ----- .../testing/embedded/compact/CompactionSupervisorTest.java | 1 - 2 files changed, 6 deletions(-) diff --git a/embedded-tests/pom.xml b/embedded-tests/pom.xml index 4cb86bcedbe9..72ada2ed4ce5 100644 --- a/embedded-tests/pom.xml +++ b/embedded-tests/pom.xml @@ -547,11 +547,6 @@ 5.5 test - - javax.servlet - javax.servlet-api - test - diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java index 65f7c75aa113..9019a0cb5ec5 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java @@ -235,5 +235,4 @@ public static List getEngine() { return List.of(CompactionEngine.NATIVE, CompactionEngine.MSQ); } - } From 729c7cf1fbdb5ab62d1df26c6808dad133189b44 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 14 Nov 2025 13:53:29 -0800 Subject: [PATCH 09/10] nicer --- .../msq/indexing/MSQCompactionRunner.java | 53 ++++++++++--------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java b/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java index f243e2870975..50cb8481f47c 100644 --- a/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java +++ b/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java @@ -47,6 +47,7 @@ import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.msq.indexing.destination.DataSourceMSQDestination; import org.apache.druid.msq.util.MultiStageQueryContext; +import org.apache.druid.query.DataSource; import org.apache.druid.query.Druids; import org.apache.druid.query.Order; import org.apache.druid.query.OrderBy; @@ -271,9 +272,8 @@ public List createMsqControllerTasks( } else { query = buildScanQuery(compactionTask, interval, dataSchema, inputColToVirtualCol); } - QueryContext compactionTaskContext = new QueryContext(compactionTask.getContext()); - query = maybeApplyPolicy(dataSchema, query); + QueryContext compactionTaskContext = new QueryContext(compactionTask.getContext()); DataSourceMSQDestination destination = buildMSQDestination(compactionTask, dataSchema); @@ -305,27 +305,6 @@ public List createMsqControllerTasks( return msqControllerTasks; } - /** - * Creates a system {@link AuthorizationResult} using an {@link Escalator} and applies any resulting - * {@link org.apache.druid.query.policy.Policy} to rewrite the query with - * {@link org.apache.druid.query.DataSource#withPolicies(Map, PolicyEnforcer)} - */ - private Query maybeApplyPolicy(DataSchema dataSchema, Query query) - { - final Escalator escalator = injector.getInstance(Escalator.class); - if (escalator != null) { - final AuthorizerMapper authorizerMapper = injector.getInstance(AuthorizerMapper.class); - final PolicyEnforcer policyEnforcer = injector.getInstance(PolicyEnforcer.class); - final AuthorizationResult authResult = AuthorizationUtils.authorizeAllResourceActions( - escalator.createEscalatedAuthenticationResult(), - List.of(new ResourceAction(new Resource(dataSchema.getDataSource(), ResourceType.DATASOURCE), Action.READ)), - authorizerMapper - ); - query = query.withDataSource(query.getDataSource().withPolicies(authResult.getPolicyMap(), policyEnforcer)); - } - return query; - } - private static DataSourceMSQDestination buildMSQDestination( CompactionTask compactionTask, DataSchema dataSchema @@ -403,6 +382,28 @@ private static RowSignature getRowSignature(DataSchema dataSchema) return rowSignatureBuilder.build(); } + /** + * Creates a {@link DataSource} and uses 'system' {@link AuthorizationResult} using an {@link Escalator} and + * {@link AuthorizerMapper} and applies any resulting {@link org.apache.druid.query.policy.Policy} to it using + * {@link DataSource#withPolicies(Map, PolicyEnforcer)} + */ + private DataSource getDataSource(String name) + { + TableDataSource dataSource = new TableDataSource(name); + final Escalator escalator = injector.getInstance(Escalator.class); + if (escalator != null) { + final AuthorizerMapper authorizerMapper = injector.getInstance(AuthorizerMapper.class); + final PolicyEnforcer policyEnforcer = injector.getInstance(PolicyEnforcer.class); + final AuthorizationResult authResult = AuthorizationUtils.authorizeAllResourceActions( + escalator.createEscalatedAuthenticationResult(), + List.of(new ResourceAction(new Resource(name, ResourceType.DATASOURCE), Action.READ)), + authorizerMapper + ); + return dataSource.withPolicies(authResult.getPolicyMap(), policyEnforcer); + } + return dataSource; + } + private static List getAggregateDimensions( DataSchema dataSchema, Map inputColToVirtualCol @@ -490,7 +491,7 @@ private static Map buildQueryContext( return queryContext; } - private static Query buildScanQuery( + private Query buildScanQuery( CompactionTask compactionTask, Interval interval, DataSchema dataSchema, @@ -500,7 +501,7 @@ private static Query buildScanQuery( RowSignature rowSignature = getRowSignature(dataSchema); VirtualColumns virtualColumns = VirtualColumns.create(new ArrayList<>(inputColToVirtualCol.values())); Druids.ScanQueryBuilder scanQueryBuilder = new Druids.ScanQueryBuilder() - .dataSource(dataSchema.getDataSource()) + .dataSource(getDataSource(dataSchema.getDataSource())) .columns(rowSignature.getColumnNames()) .virtualColumns(virtualColumns) .columnTypes(rowSignature.getColumnTypes()) @@ -651,7 +652,7 @@ private Query buildGroupByQuery( .collect(Collectors.toList()); GroupByQuery.Builder builder = new GroupByQuery.Builder() - .setDataSource(new TableDataSource(compactionTask.getDataSource())) + .setDataSource(getDataSource(compactionTask.getDataSource())) .setVirtualColumns(virtualColumns) .setDimFilter(dimFilter) .setGranularity(new AllGranularity()) From 0666516e6b7ff288cca9f66d15a14c51e1949cd8 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 14 Nov 2025 13:59:47 -0800 Subject: [PATCH 10/10] better name --- embedded-tests/pom.xml | 1 + .../org/apache/druid/msq/indexing/MSQCompactionRunner.java | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/embedded-tests/pom.xml b/embedded-tests/pom.xml index 72ada2ed4ce5..9a3fbd90765e 100644 --- a/embedded-tests/pom.xml +++ b/embedded-tests/pom.xml @@ -547,6 +547,7 @@ 5.5 test + diff --git a/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java b/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java index 50cb8481f47c..745b86cdd163 100644 --- a/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java +++ b/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java @@ -387,7 +387,7 @@ private static RowSignature getRowSignature(DataSchema dataSchema) * {@link AuthorizerMapper} and applies any resulting {@link org.apache.druid.query.policy.Policy} to it using * {@link DataSource#withPolicies(Map, PolicyEnforcer)} */ - private DataSource getDataSource(String name) + private DataSource getInputDataSource(String name) { TableDataSource dataSource = new TableDataSource(name); final Escalator escalator = injector.getInstance(Escalator.class); @@ -501,7 +501,7 @@ private Query buildScanQuery( RowSignature rowSignature = getRowSignature(dataSchema); VirtualColumns virtualColumns = VirtualColumns.create(new ArrayList<>(inputColToVirtualCol.values())); Druids.ScanQueryBuilder scanQueryBuilder = new Druids.ScanQueryBuilder() - .dataSource(getDataSource(dataSchema.getDataSource())) + .dataSource(getInputDataSource(dataSchema.getDataSource())) .columns(rowSignature.getColumnNames()) .virtualColumns(virtualColumns) .columnTypes(rowSignature.getColumnTypes()) @@ -652,7 +652,7 @@ private Query buildGroupByQuery( .collect(Collectors.toList()); GroupByQuery.Builder builder = new GroupByQuery.Builder() - .setDataSource(getDataSource(compactionTask.getDataSource())) + .setDataSource(getInputDataSource(compactionTask.getDataSource())) .setVirtualColumns(virtualColumns) .setDimFilter(dimFilter) .setGranularity(new AllGranularity())