diff --git a/distribution/bin/check-licenses.py b/distribution/bin/check-licenses.py index f90ecd817b3b..5704adc0ba21 100755 --- a/distribution/bin/check-licenses.py +++ b/distribution/bin/check-licenses.py @@ -224,6 +224,7 @@ def build_compatible_license_names(): compatible_licenses['Apache License Version 2'] = 'Apache License version 2.0' compatible_licenses['Apache License v2.0'] = 'Apache License version 2.0' compatible_licenses['Apache License, version 2.0'] = 'Apache License version 2.0' + compatible_licenses['Apache 2.0 License'] = 'Apache License version 2.0' compatible_licenses['Public Domain'] = 'Public Domain' @@ -241,6 +242,7 @@ def build_compatible_license_names(): compatible_licenses['Revised BSD'] = 'BSD-3-Clause License' compatible_licenses['New BSD License'] = 'BSD-3-Clause License' compatible_licenses['3-Clause BSD License'] = 'BSD-3-Clause License' + compatible_licenses['BSD 3-Clause'] = 'BSD-3-Clause License' compatible_licenses['ICU License'] = 'ICU License' @@ -256,9 +258,11 @@ def build_compatible_license_names(): compatible_licenses['The Eclipse Public License, Version 1.0'] = 'Eclipse Public License 1.0' compatible_licenses['Eclipse Public License - Version 1.0'] = 'Eclipse Public License 1.0' compatible_licenses['Eclipse Public License, Version 1.0'] = 'Eclipse Public License 1.0' + compatible_licenses['Eclipse Public License v1.0'] = 'Eclipse Public License 1.0' compatible_licenses['Eclipse Distribution License 1.0'] = 'Eclipse Distribution License 1.0' compatible_licenses['Eclipse Distribution License - v 1.0'] = 'Eclipse Distribution License 1.0' + compatible_licenses['Eclipse Distribution License v. 1.0'] = 'Eclipse Distribution License 1.0' compatible_licenses['EDL 1.0'] = 'Eclipse Distribution License 1.0' compatible_licenses['Mozilla Public License Version 2.0'] = 'Mozilla Public License Version 2.0' diff --git a/distribution/pom.xml b/distribution/pom.xml index 4ecc7c38e417..a47ead33057b 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -242,6 +242,8 @@ org.apache.druid.extensions:druid-basic-security -c org.apache.druid.extensions:druid-pac4j + -c + org.apache.druid.extensions:druid-ranger-security ${druid.distribution.pulldeps.opts} diff --git a/docs/development/extensions-core/druid-ranger-security.md b/docs/development/extensions-core/druid-ranger-security.md new file mode 100644 index 000000000000..374458f0794f --- /dev/null +++ b/docs/development/extensions-core/druid-ranger-security.md @@ -0,0 +1,158 @@ +--- +id: druid-ranger-security +title: "Apache Ranger Security" +--- + + + +This Apache Druid extension adds: + +- an Authorizer which implements access control for the Druid metastore against Apache Ranger + +Make sure to [include](../../development/extensions.md#loading-extensions) `druid-ranger-security` as an extension. + +Please see [Authentication and Authorization](../../design/auth.md) for more information on the extension interfaces being implemented. + +## Configuration + +Support for Apache Ranger authorization consists of three elements: configuration of the extension +in Apache Druid, configuring the connection to Apache Ranger and providing the service definition for Druid to Apache Ranger. + +### Enabling the extension +Ensure that you have a valid authentication chain and escalator set in your `common.runtime.properties`. For every +authenticator your wish to use the authorizer for set `druid.auth.authenticator..authorizerName` +to the name you will give the authorizer, e.g. `ranger`. + +Then add the following and amend to your needs (in case you use multiple authorizers): + +``` +druid.auth.authorizers=["ranger"] +druid.auth.authorizer.ranger.type=ranger +``` + +The following is an example that uses `druid-basic-security` for authentication and `druid-ranger-security` for +authorization. + +``` +druid.auth.authenticatorChain=["basic"] +druid.auth.authenticator.basic.type=basic +druid.auth.authenticator.basic.initialAdminPassword=password1 +druid.auth.authenticator.basic.initialInternalClientPassword=password2 +druid.auth.authenticator.basic.credentialsValidator.type=metadata +druid.auth.authenticator.basic.skipOnFailure=false +druid.auth.authenticator.basic.enableCacheNotifications=true +druid.auth.authenticator.basic.authorizerName=ranger + +druid.auth.authorizers=["ranger"] +druid.auth.authorizer.ranger.type=ranger + +# Escalator +druid.escalator.type=basic +druid.escalator.internalClientUsername=druid_system +druid.escalator.internalClientPassword=password2 +druid.escalator.authorizerName=ranger +``` + +--- +**NOTE** + +Contrary to the documentation of `druid-basic-auth` Ranger does not automatically provision a highly privileged +system system user and you will need to do this yourself. This system user in case of `druid-basic-auth` is named +`druid_system` and for the escalator it is configurable as shown above. Make sure to take note of these user names and +configure `READ` access to `state:STATE` and to `config:security` in your ranger policies, +otherwise system services will not work properly. +--- + +#### Properties to configure the extension in Apache Druid +|Property|Description|Default|required| +|--------|-----------|-------|--------| +|`druid.auth.ranger.keytab`|Defines the keytab to be used while authenticating against Apache Ranger to obtain policies and provide auditing|null|No| +|`druid.auth.ranger.principal`|Defines the principal to be used while authenticating against Apache Ranger to obtain policies and provide auditing|null|No| +|`druid.auth.ranger.use_ugi`|Determines if groups that the authenticated user belongs to should be obtained from Hadoop's `UserGroupInformation`|null|No| + +### Configuring the connection to Apache Ranger + +The Apache Ranger authorization extension will read several configuration files. Discussing the +the contents of those files is beyond the scope of this document. Depending on your needs you will +need to create them. The minimum you will need to have is a `ranger-druid-security.xml` file +that you will need to put in the classpath (e.g. `_common`). For auditing, the configuration is +in `ranger-druid-audit.xml`. + +### Adding the service definition for Apache Druid to Apache Ranger + +At the time of writing of this document Apache Ranger (2.0) does not include a service and +service definition yet. You can add the service definition to Apache Ranger by entering the following +command: + +`curl -u : -d "@ranger-servicedef-druid.json" -X POST -H "Accept: application/json" -H "Content-Type: application/json" http://localhost:6080/service/public/v2/api/servicedef/` + +You should get back `json` describing the service definition you just added. You can now go to the web +interface of Apache Ranger which should now include a widget for "Druid". Click the plus sign an create +the new service. Ensure your service name is equal to what you configured in `ranger-druid-security.xml`. + +#### Configuring Apache Ranger policies + +When installing a new Druid service inside Apache Ranger for the first time, Ranger will provision the policies +to allow the administrative user `read/write` access to all properties and data sources. You might want to limit this. +Do not forget to add the correct policies for the `druid_system` user and the `internalClientUserName` of the escalator. + +--- +**NOTE** + +Loading new data sources requires `write` access to the `datasource` prior to the loading itself. So if you +want to create a datasource `wikipedia` you are required to have an `allow` policy inside Apache Ranger before +trying to load the spec. +--- + +## Usage + +### HTTP methods + +For information on what HTTP methods are supported on a particular request endpoint, please refer to the [API documentation](../../operations/api-reference.md). + +GET requires READ permission, while POST and DELETE require WRITE permission. + +### SQL Permissions + +Queries on Druid datasources require DATASOURCE READ permissions for the specified datasource. + +Queries on the [INFORMATION_SCHEMA tables](../../querying/sql.html#information-schema) will +return information about datasources that the caller has DATASOURCE READ access to. Other +datasources will be omitted. + +Queries on the [system schema tables](../../querying/sql.html#system-schema) require the following permissions: +- `segments`: Segments will be filtered based on DATASOURCE READ permissions. +- `servers`: The user requires STATE READ permissions. +- `server_segments`: The user requires STATE READ permissions and segments will be filtered based on DATASOURCE READ permissions. +- `tasks`: Tasks will be filtered based on DATASOURCE READ permissions. + + +### Debugging + +If you face difficulty grasping why access is denied to certain elements and the `audit` section in +Apache Ranger does not give you any detail, you can enable debug logging for `org.apache.druid.security.ranger`. +To do so add the following in your `log4j2.xml`: + +```xml + + + + +``` diff --git a/docs/development/extensions.md b/docs/development/extensions.md index 21bd8508b770..54a50ffb98c5 100644 --- a/docs/development/extensions.md +++ b/docs/development/extensions.md @@ -54,6 +54,7 @@ Core extensions are maintained by Druid committers. |druid-orc-extensions|Support for data in Apache Orc data format.|[link](../development/extensions-core/orc.md)| |druid-parquet-extensions|Support for data in Apache Parquet data format. Requires druid-avro-extensions to be loaded.|[link](../development/extensions-core/parquet.md)| |druid-protobuf-extensions| Support for data in Protobuf data format.|[link](../development/extensions-core/protobuf.md)| +|druid-ranger-security|Support for access control through Apache Ranger.|[link](../development/extensions-core/druid-ranger-security.md)| |druid-s3-extensions|Interfacing with data in AWS S3, and using S3 as deep storage.|[link](../development/extensions-core/s3.md)| |druid-ec2-extensions|Interfacing with AWS EC2 for autoscaling middle managers|UNDOCUMENTED| |druid-stats|Statistics related module including variance and standard deviation.|[link](../development/extensions-core/stats.md)| diff --git a/extensions-core/druid-ranger-security/pom.xml b/extensions-core/druid-ranger-security/pom.xml new file mode 100644 index 000000000000..fa6c8251f577 --- /dev/null +++ b/extensions-core/druid-ranger-security/pom.xml @@ -0,0 +1,406 @@ + + + + + + 4.0.0 + + org.apache.druid.extensions + druid-ranger-security + druid-ranger-security + druid-ranger-security + + + org.apache.druid + druid + 0.19.0-SNAPSHOT + ../../pom.xml + + + + + org.apache.druid + druid-core + ${project.parent.version} + provided + + + org.apache.druid + druid-services + ${project.parent.version} + provided + + + org.apache.druid + druid-server + ${project.parent.version} + provided + + + org.jdbi + jdbi + provided + + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + provided + + + com.google.code.findbugs + jsr305 + provided + + + joda-time + joda-time + provided + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + com.google.inject + guice + provided + + + io.netty + netty + provided + + + com.fasterxml.jackson.core + jackson-databind + provided + + + javax.servlet + javax.servlet-api + provided + + + com.fasterxml.jackson.core + jackson-core + provided + + + com.sun.jersey + jersey-server + provided + + + org.apache.commons + commons-lang3 + provided + + + com.google.guava + guava + provided + + + javax.ws.rs + jsr311-api + provided + + + org.apache.ranger + ranger-plugins-common + ${apache.ranger.version} + compile + + + org.apache.ranger + ranger-plugins-audit + ${apache.ranger.version} + compile + + + com.google.code.gson + gson + ${apache.ranger.gson.version} + compile + + + org.apache.hadoop + hadoop-client + runtime + + + commons-cli + commons-cli + + + log4j + log4j + + + commons-codec + commons-codec + + + commons-logging + commons-logging + + + commons-io + commons-io + + + commons-lang + commons-lang + + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpcore + + + org.apache.zookeeper + zookeeper + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-log4j12 + + + javax.ws.rs + jsr311-api + + + com.google.code.findbugs + jsr305 + + + org.mortbay.jetty + jetty-util + + + org.apache.hadoop + hadoop-annotations + + + javax.activation + activation + + + com.google.protobuf + protobuf-java + + + com.sun.jersey + jersey-core + + + org.apache.curator + curator-client + + + org.apache.curator + curator-framework + + + org.apache.curator + curator-recipes + + + org.apache.commons + commons-math3 + + + com.google.guava + guava + + + + commons-beanutils + commons-beanutils-core + + + + + org.apache.hadoop + hadoop-common + ${hadoop.compile.version} + compile + + + commons-cli + commons-cli + + + log4j + log4j + + + commons-codec + commons-codec + + + commons-logging + commons-logging + + + commons-io + commons-io + + + commons-lang + commons-lang + + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpcore + + + org.apache.zookeeper + zookeeper + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-log4j12 + + + javax.ws.rs + jsr311-api + + + com.google.code.findbugs + jsr305 + + + org.mortbay.jetty + jetty-util + + + com.google.protobuf + protobuf-java + + + com.sun.jersey + jersey-core + + + org.apache.curator + curator-client + + + org.apache.commons + commons-math3 + + + com.google.guava + guava + + + org.apache.avro + avro + + + net.java.dev.jets3t + jets3t + + + com.sun.jersey + jersey-json + + + com.jcraft + jsch + + + org.mortbay.jetty + jetty + + + com.sun.jersey + jersey-server + + + + commons-beanutils + commons-beanutils-core + + + + + + + junit + junit + test + + + org.easymock + easymock + test + + + org.apache.druid + druid-server + ${project.parent.version} + test-jar + test + + + + + + src/test/resources + + **/* + + true + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + + org.apache.ranger:ranger-plugins-audit + + + + + + diff --git a/extensions-core/druid-ranger-security/src/main/java/org/apache/druid/security/ranger/authorizer/RangerAuthorizer.java b/extensions-core/druid-ranger-security/src/main/java/org/apache/druid/security/ranger/authorizer/RangerAuthorizer.java new file mode 100644 index 000000000000..330a7c0250d5 --- /dev/null +++ b/extensions-core/druid-ranger-security/src/main/java/org/apache/druid/security/ranger/authorizer/RangerAuthorizer.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.security.ranger.authorizer; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.security.ranger.authorizer.guice.Ranger; +import org.apache.druid.server.security.Access; +import org.apache.druid.server.security.Action; +import org.apache.druid.server.security.AuthenticationResult; +import org.apache.druid.server.security.Authorizer; +import org.apache.druid.server.security.Resource; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler; +import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; +import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; +import org.apache.ranger.plugin.policyengine.RangerAccessResult; +import org.apache.ranger.plugin.service.RangerBasePlugin; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Date; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; + +@JsonTypeName("ranger") +public class RangerAuthorizer implements Authorizer +{ + private static final Logger log = new Logger(RangerAuthorizer.class); + + public static final String RANGER_DRUID_SERVICETYPE = "druid"; + public static final String RANGER_DRUID_APPID = "druid"; + + private final RangerBasePlugin rangerPlugin; + private final boolean useUgi; + + @JsonCreator + public RangerAuthorizer( + @JsonProperty("keytab") String keytab, + @JsonProperty("principal") String principal, + @JsonProperty("use_ugi") boolean useUgi, + @JacksonInject @Ranger Configuration conf) + { + this.useUgi = useUgi; + + UserGroupInformation.setConfiguration(conf); + + if (keytab != null && principal != null) { + try { + UserGroupInformation.loginUserFromKeytab(principal, keytab); + } + catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + + rangerPlugin = new RangerBasePlugin(RANGER_DRUID_SERVICETYPE, RANGER_DRUID_APPID); + rangerPlugin.init(); + rangerPlugin.setResultProcessor(new RangerDefaultAuditHandler()); + + } + + @Override + public Access authorize(AuthenticationResult authenticationResult, Resource resource, Action action) + { + if (authenticationResult == null) { + throw new IAE("authenticationResult is null where it should never be."); + } + + Set userGroups = null; + if (useUgi) { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(authenticationResult.getIdentity()); + String[] groups = ugi != null ? ugi.getGroupNames() : null; + if (groups != null && groups.length > 0) { + userGroups = new HashSet<>(Arrays.asList(groups)); + } + } + + RangerDruidResource rangerDruidResource = new RangerDruidResource(resource); + RangerDruidAccessRequest request = new RangerDruidAccessRequest( + rangerDruidResource, + authenticationResult.getIdentity(), + userGroups, + action + ); + + RangerAccessResult result = rangerPlugin.isAccessAllowed(request); + if (log.isDebugEnabled()) { + log.debug("==> authorize: %s, allowed: %s", + request.toString(), + result != null ? result.getIsAllowed() : null); + } + + if (result != null && result.getIsAllowed()) { + return new Access(true); + } + + return new Access(false); + } +} + +class RangerDruidResource extends RangerAccessResourceImpl +{ + public RangerDruidResource(Resource resource) + { + setValue(resource.getType().name().toLowerCase(Locale.ENGLISH), resource.getName()); + } +} + +class RangerDruidAccessRequest extends RangerAccessRequestImpl +{ + public RangerDruidAccessRequest(RangerDruidResource resource, String user, Set userGroups, Action action) + { + super(resource, action.name().toLowerCase(Locale.ENGLISH), user, userGroups); + setAccessTime(new Date()); + } +} diff --git a/extensions-core/druid-ranger-security/src/main/java/org/apache/druid/security/ranger/authorizer/RangerSecurityDruidModule.java b/extensions-core/druid-ranger-security/src/main/java/org/apache/druid/security/ranger/authorizer/RangerSecurityDruidModule.java new file mode 100644 index 000000000000..f10ddebdf8f9 --- /dev/null +++ b/extensions-core/druid-ranger-security/src/main/java/org/apache/druid/security/ranger/authorizer/RangerSecurityDruidModule.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.security.ranger.authorizer; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.common.collect.ImmutableList; +import com.google.inject.Binder; +import com.google.inject.Inject; +import org.apache.druid.initialization.DruidModule; +import org.apache.druid.security.ranger.authorizer.guice.Ranger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; + +import java.io.IOException; +import java.util.List; +import java.util.Properties; + +public class RangerSecurityDruidModule implements DruidModule +{ + private Properties props = null; + + @Override + public void configure(Binder binder) + { + // this block of code is common among extensions that use Hadoop things but are not running in Hadoop, in order + // to properly initialize everything + + final Configuration conf = new Configuration(); + + // Set explicit CL. Otherwise it'll try to use thread context CL, which may not have all of our dependencies. + conf.setClassLoader(getClass().getClassLoader()); + + // Ensure that FileSystem class level initialization happens with correct CL + // See https://github.com/apache/druid/issues/1714 + ClassLoader currCtxCl = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); + FileSystem.get(conf); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + finally { + Thread.currentThread().setContextClassLoader(currCtxCl); + } + + if (props != null) { + for (String propName : props.stringPropertyNames()) { + if (propName.startsWith("hadoop.")) { + conf.set(propName.substring("hadoop.".length()), props.getProperty(propName)); + } + } + } + + binder.bind(Configuration.class).annotatedWith(Ranger.class).toInstance(conf); + } + + @Override + public List getJacksonModules() + { + return ImmutableList.of( + new SimpleModule("RangerDruidSecurity").registerSubtypes(RangerAuthorizer.class) + ); + } + + @Inject + public void setProperties(Properties props) + { + this.props = props; + } + +} diff --git a/extensions-core/druid-ranger-security/src/main/java/org/apache/druid/security/ranger/authorizer/guice/Ranger.java b/extensions-core/druid-ranger-security/src/main/java/org/apache/druid/security/ranger/authorizer/guice/Ranger.java new file mode 100644 index 000000000000..811a00b25645 --- /dev/null +++ b/extensions-core/druid-ranger-security/src/main/java/org/apache/druid/security/ranger/authorizer/guice/Ranger.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.security.ranger.authorizer.guice; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@BindingAnnotation +public @interface Ranger +{ +} diff --git a/extensions-core/druid-ranger-security/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-core/druid-ranger-security/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100644 index 000000000000..62de1db99870 --- /dev/null +++ b/extensions-core/druid-ranger-security/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.druid.security.ranger.authorizer.RangerSecurityDruidModule diff --git a/extensions-core/druid-ranger-security/src/test/java/org/apache/druid/security/ranger/authorizer/RangerAdminClientImpl.java b/extensions-core/druid-ranger-security/src/test/java/org/apache/druid/security/ranger/authorizer/RangerAdminClientImpl.java new file mode 100644 index 000000000000..c28bdc330c6c --- /dev/null +++ b/extensions-core/druid-ranger-security/src/test/java/org/apache/druid/security/ranger/authorizer/RangerAdminClientImpl.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.security.ranger.authorizer; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.ranger.admin.client.AbstractRangerAdminClient; +import org.apache.ranger.plugin.util.ServicePolicies; + +import java.io.File; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; + +public class RangerAdminClientImpl extends AbstractRangerAdminClient +{ + private static final Logger LOG = new Logger(RangerAdminClientImpl.class); + private static final String CACHE_FILE_NAME = "druid-policies.json"; + + protected Gson gson; + + @Override + public void init(String serviceName, String appId, String configPropertyPrefix) + { + super.init(serviceName, appId, configPropertyPrefix); + + try { + gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create(); + } + catch (Throwable excp) { + LOG.error(excp, "AbstractRangerAdminClient: failed to create GsonBuilder object"); + } + } + + @Override + public ServicePolicies getServicePoliciesIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception + { + + String basedir = System.getProperty("basedir"); + if (basedir == null) { + basedir = new File(".").getCanonicalPath(); + } + + Path cachePath = FileSystems.getDefault().getPath(basedir, "/src/test/resources/" + CACHE_FILE_NAME); + byte[] cacheBytes = Files.readAllBytes(cachePath); + + return gson.fromJson(new String(cacheBytes, "UTF8"), ServicePolicies.class); + } + +} diff --git a/extensions-core/druid-ranger-security/src/test/java/org/apache/druid/security/ranger/authorizer/RangerAuthorizerTest.java b/extensions-core/druid-ranger-security/src/test/java/org/apache/druid/security/ranger/authorizer/RangerAuthorizerTest.java new file mode 100644 index 000000000000..46fef6ca77c8 --- /dev/null +++ b/extensions-core/druid-ranger-security/src/test/java/org/apache/druid/security/ranger/authorizer/RangerAuthorizerTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.security.ranger.authorizer; + +import org.apache.druid.server.security.Action; +import org.apache.druid.server.security.AuthenticationResult; +import org.apache.druid.server.security.Resource; +import org.apache.druid.server.security.ResourceType; +import org.apache.hadoop.conf.Configuration; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class RangerAuthorizerTest +{ + static RangerAuthorizer rangerAuthorizer = null; + + private static final AuthenticationResult alice = new AuthenticationResult("alice", null, null, null); + private static final AuthenticationResult bob = new AuthenticationResult("bob", null, null, null); + + private static final Resource aliceDatasource = new Resource("alice-datasource", ResourceType.DATASOURCE); + private static final Resource aliceConfig = new Resource("config", ResourceType.CONFIG); + private static final Resource aliceState = new Resource("state", ResourceType.STATE); + + @BeforeClass + public static void setupBeforeClass() + { + rangerAuthorizer = new RangerAuthorizer(null, null, false, new Configuration()); + } + + @Test + public void testOperations() + { + Assert.assertTrue(rangerAuthorizer.authorize(alice, aliceDatasource, Action.READ).isAllowed()); + Assert.assertTrue(rangerAuthorizer.authorize(alice, aliceDatasource, Action.READ).isAllowed()); + Assert.assertTrue(rangerAuthorizer.authorize(alice, aliceConfig, Action.READ).isAllowed()); + Assert.assertTrue(rangerAuthorizer.authorize(alice, aliceConfig, Action.WRITE).isAllowed()); + Assert.assertTrue(rangerAuthorizer.authorize(alice, aliceState, Action.READ).isAllowed()); + Assert.assertTrue(rangerAuthorizer.authorize(alice, aliceState, Action.WRITE).isAllowed()); + + Assert.assertFalse(rangerAuthorizer.authorize(bob, aliceDatasource, Action.READ).isAllowed()); + } +} diff --git a/extensions-core/druid-ranger-security/src/test/resources/druid-policies.json b/extensions-core/druid-ranger-security/src/test/resources/druid-policies.json new file mode 100644 index 000000000000..abdf85ac894e --- /dev/null +++ b/extensions-core/druid-ranger-security/src/test/resources/druid-policies.json @@ -0,0 +1,294 @@ +{ + "serviceName": "cl1_druid", + "serviceId": 16, + "policyUpdateTime": "20180304-09:49:38.000-+0000", + "policyVersion": "1", + "policies": [ + { + "service": "cl1_druid", + "name": "alice-test", + "policyType": 0, + "policyPriority": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "datasource": { + "values": [ + "alice-datasource" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "read", + "isAllowed": true + } + ], + "users": [ + "alice" + ], + "groups": [], + "roles": [], + "conditions": [], + "delegateAdmin": false + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "serviceType": "druid", + "options": {}, + "validitySchedules": [], + "policyLabels": [], + "zoneName": "", + "isDenyAllElse": false, + "id": 62, + "guid": "850fe929-26ce-4641-be06-3a771b969e54", + "isEnabled": true, + "version": 1 + }, + { + "service": "cl1_druid", + "name": "alice-config", + "policyType": 0, + "policyPriority": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "config": { + "values": [ + "CONFIG" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "read", + "isAllowed": true + }, + { + "type": "write", + "isAllowed": true + } + ], + "users": [ + "alice" + ], + "groups": [], + "roles": [], + "conditions": [], + "delegateAdmin": false + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "serviceType": "druid", + "options": {}, + "validitySchedules": [], + "policyLabels": [], + "zoneName": "", + "isDenyAllElse": false, + "id": 63, + "guid": "27725030-4a53-46a7-b58e-0d39763c019d", + "isEnabled": true, + "version": 1 + }, + { + "service": "cl1_druid", + "name": "alice-state", + "policyType": 0, + "policyPriority": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "state": { + "values": [ + "STATE" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "read", + "isAllowed": true + }, + { + "type": "write", + "isAllowed": true + } + ], + "users": [ + "alice" + ], + "groups": [], + "roles": [], + "conditions": [], + "delegateAdmin": false + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "serviceType": "druid", + "options": {}, + "validitySchedules": [], + "policyLabels": [], + "zoneName": "", + "isDenyAllElse": false, + "id": 64, + "guid": "bddd74af-b505-452d-9930-c20971337f48", + "isEnabled": true, + "version": 1 + } + ], + "startIndex": 0, + "pageSize": 0, + "totalCount": 0, + "resultSize": 0, + "queryTimeMS": 1585396899372, + "serviceDef": { + "id": 18, + "name": "druid", + "implClass": "org.apache.ranger.service.druid.RangerDruidService", + "label": "Druid", + "description": "Apache Druid", + "resources": [ + { + "itemId": 10, + "name": "datasource", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Datasource", + "description": "Druid Datasource" + }, + { + "itemId": 20, + "name": "config", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Config", + "description": "Druid Config" + }, + { + "itemId": 30, + "name": "state", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "State", + "description": "Druid State" + } + ], + "accessTypes": [ + { + "itemId": 1, + "name": "read", + "label": "Read" + }, + { + "itemId": 2, + "name": "write", + "label": "Write" + } + ], + "configs": [ + { + "itemId": 1, + "name": "username", + "type": "string", + "mandatory": true, + "label": "Username" + }, + { + "itemId": 2, + "name": "password", + "type": "password", + "mandatory": true, + "label": "Password" + }, + { + "itemId": 3, + "name": "druid.broker.url", + "type": "string", + "mandatory": true, + "defaultValue": "http://localhost:8082", + "label": "Druid broker host:port" + } + ], + "enums": [ + ], + "contextEnrichers": [ + ], + "policyConditions": [ + { + "itemId": 1, + "name": "ip-range", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerIpMatcher", + "evaluatorOptions": { + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "IP Address Range", + "description": "IP Address Range" + } + ] + } +} diff --git a/extensions-core/druid-ranger-security/src/test/resources/ranger-druid-security.xml b/extensions-core/druid-ranger-security/src/test/resources/ranger-druid-security.xml new file mode 100644 index 000000000000..86981395642e --- /dev/null +++ b/extensions-core/druid-ranger-security/src/test/resources/ranger-druid-security.xml @@ -0,0 +1,52 @@ + + + + + + ranger.plugin.druid.service.name + cl1_druid + + Name of the Ranger service containing policies for this SampleApp instance + + + + + ranger.plugin.druid.policy.source.impl + org.apache.druid.security.ranger.authorizer.RangerAdminClientImpl + + Policy source. + + + + + ranger.plugin.druid.policy.pollIntervalMs + 30000 + + How often to poll for changes in policies? + + + + + ranger.plugin.druid.policy.cache.dir + ${project.build.directory} + + Directory where Ranger policies are cached after successful retrieval from the source + + + + diff --git a/licenses.yaml b/licenses.yaml index 7af752619474..68d11f5e0ac1 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -4211,6 +4211,238 @@ libraries: --- +name: org.apache.ranger ranger-plugins-audit +license_category: binary +version: 2.0.0 +module: druid-ranger-security +license_name: Apache License version 2.0 +libraries: + - org.apache.ranger: ranger-plugins-audit + +--- + +name: org.apache.ranger ranger-plugins-common +license_category: binary +version: 2.0.0 +module: druid-ranger-security +license_name: Apache License version 2.0 +libraries: + - org.apache.ranger: ranger-plugins-common + +--- + +name: com.101tec zkclient +license_category: binary +version: '0.10' +module: druid-ranger-security +license_name: Apache License version 2.0 +libraries: + - com.101tec: zkclient + +--- + +name: com.kstruct gethostname4j +license_category: binary +version: 0.0.2 +module: druid-ranger-security +license_name: MIT License +libraries: + - com.kstruct: gethostname4j + +--- + +name: com.sun.jersey jersey-bundle +license_category: binary +version: 1.19.3 +module: druid-ranger-security +license_name: CDDL 1.1 +libraries: + - com.sun.jersey: jersey-bundle + +--- + +name: net.java.dev.jna jna-platform +license_category: binary +version: 5.2.0 +module: druid-ranger-security +license_name: Apache License version 2.0 +libraries: + - net.java.dev.jna: jna-platform + +--- + +name: JOpt Simple +license_category: binary +version: 5.0.4 +module: druid-ranger-security +license_name: MIT License +libraries: + - net.sf.jopt-simple: jopt-simple +copyright: Paul R. Holser, Jr. + +--- + +name: org.apache.httpcomponents httpmime +license_category: binary +version: 4.5.3 +module: druid-ranger-security +license_name: Apache License version 2.0 +libraries: + - org.apache.httpcomponents: httpmime + +--- + +name: Apache Kafka +license_category: binary +version: 2.0.0 +module: druid-ranger-security +license_name: Apache License version 2.0 +libraries: + - org.apache.kafka: kafka-clients +notices: + - kafka-clients: 'Apache Kafka Copyright 2019 The Apache Software Foundation. + +This distribution has a binary dependency on jersey, which is available under +the CDDL License. The source code of jersey can be found at https://github.com/jersey/jersey/.' + +--- + +name: org.apache.kafka kafka_2.11 +license_category: binary +version: 2.0.0 +module: druid-ranger-security +license_name: Apache License version 2.0 +libraries: + - org.apache.kafka: kafka_2.11 + +--- + +name: org.apache.ranger ranger-plugins-cred +license_category: binary +version: 2.0.0 +module: druid-ranger-security +license_name: Apache License version 2.0 +libraries: + - org.apache.ranger: ranger-plugins-cred + +--- + +name: org.apache.solr solr-solrj +license_category: binary +version: 7.7.1 +module: druid-ranger-security +license_name: Apache License version 2.0 +libraries: + - org.apache.solr: solr-solrj + +--- + +name: org.codehaus.woodstox stax2-api +license_category: binary +version: 3.1.4 +module: druid-ranger-security +license_name: BSD-3-Clause License +libraries: + - org.codehaus.woodstox: stax2-api + +--- + +name: org.codehaus.woodstox woodstox-core-asl +license_category: binary +version: 4.4.1 +module: druid-ranger-security +license_name: Apache License version 2.0 +libraries: + - org.codehaus.woodstox: woodstox-core-asl + +--- + +name: org.eclipse.persistence commonj.sdo +license_category: binary +version: 2.1.1 +module: druid-ranger-security +license_name: Eclipse Distribution License 1.0 +libraries: + - org.eclipse.persistence: commonj.sdo + +--- + +name: org.eclipse.persistence eclipselink +license_category: binary +version: 2.5.2 +module: druid-ranger-security +license_name: Eclipse Distribution License 1.0 +libraries: + - org.eclipse.persistence: eclipselink + +--- + +name: org.eclipse.persistence javax.persistence +license_category: binary +version: 2.1.0 +module: druid-ranger-security +license_name: Eclipse Distribution License 1.0 +libraries: + - org.eclipse.persistence: javax.persistence + +--- + +name: org.noggit noggit +license_category: binary +version: '0.8' +module: druid-ranger-security +license_name: Apache License version 2.0 +libraries: + - org.noggit: noggit + +--- + +name: Scala Library +license_category: binary +version: 2.11.12 +module: druid-ranger-security +license_name: BSD-3-Clause License +libraries: + - org.scala-lang: scala-library +copyright: LAMP/EPFL and Lightbend, Inc. + +--- + +name: org.scala-lang scala-reflect +license_category: binary +version: 2.11.12 +module: druid-ranger-security +license_name: BSD-3-Clause License +libraries: + - org.scala-lang: scala-reflect + +--- + +name: snappy-java +license_category: binary +version: 1.1.7.1 +module: druid-ranger-security +license_name: Apache License version 2.0 +libraries: + - org.xerial.snappy: snappy-java +notices: + - snappy-java: | + This product includes software developed by Google + Snappy: http://code.google.com/p/snappy/ (New BSD License) + + + This library containd statically linked libstdc++. This inclusion is allowed by + "GCC RUntime Library Exception" + http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html + + == Contributors == + * Tatu Saloranta + * Providing benchmark suite + * Alec Wysoker + * Performance and memory usage improvement + +--- + # Web console modules start name: "@babel/runtime" license_category: binary diff --git a/pom.xml b/pom.xml index 6e43316e21d9..0305e8797d32 100644 --- a/pom.xml +++ b/pom.xml @@ -79,6 +79,8 @@ 4.1.0 2.12.0 2.2.2 + 2.0.0 + 2.2.4 1.15.0 1.9.2 1.21.0 @@ -162,6 +164,7 @@ extensions-core/simple-client-sslcontext extensions-core/druid-basic-security extensions-core/google-extensions + extensions-core/druid-ranger-security extensions-contrib/influx-extensions extensions-contrib/cassandra-storage diff --git a/website/.spelling b/website/.spelling index 26aac1c9f794..b3f1deae329f 100644 --- a/website/.spelling +++ b/website/.spelling @@ -1722,4 +1722,8 @@ isRobot isUnpatrolled metroCode regionIsoCode -regionName \ No newline at end of file +regionName + - ../docs/development/extensions-core/druid-ranger-security.md +json +metastore +UserGroupInformation \ No newline at end of file