From 1d5694c95f91996b28b2b4cdde3400ebefdaca58 Mon Sep 17 00:00:00 2001 From: Demogorgon314 Date: Mon, 18 Apr 2022 23:40:21 +0800 Subject: [PATCH 1/8] Add KoP OAuth example --- kop/java/README.md | 127 ++++++++++++++++++ kop/java/pom.xml | 6 + .../examples/kafka/OAuthConsumer.java | 74 ++++++++++ .../examples/kafka/OAuthProducer.java | 65 +++++++++ .../main/resources/hydra/docker-compose.yml | 35 +++++ .../resources/hydra/keys/private_key.json | 12 ++ .../main/resources/hydra/keys/public_key.json | 9 ++ .../main/resources/init_hydra_oauth_server.sh | 78 +++++++++++ kop/java/src/main/resources/oauth.properties | 20 +++ .../oauth_credentials/credentials.json | 4 + .../simple_credentials_file.json | 4 + 11 files changed, 434 insertions(+) create mode 100644 kop/java/src/main/java/io/streamnative/examples/kafka/OAuthConsumer.java create mode 100644 kop/java/src/main/java/io/streamnative/examples/kafka/OAuthProducer.java create mode 100644 kop/java/src/main/resources/hydra/docker-compose.yml create mode 100644 kop/java/src/main/resources/hydra/keys/private_key.json create mode 100644 kop/java/src/main/resources/hydra/keys/public_key.json create mode 100755 kop/java/src/main/resources/init_hydra_oauth_server.sh create mode 100644 kop/java/src/main/resources/oauth.properties create mode 100644 kop/java/src/main/resources/oauth_credentials/credentials.json create mode 100644 kop/java/src/main/resources/oauth_credentials/simple_credentials_file.json diff --git a/kop/java/README.md b/kop/java/README.md index ff8ebaa..47eade8 100644 --- a/kop/java/README.md +++ b/kop/java/README.md @@ -13,6 +13,8 @@ This document describes how to produce messages to and consume messages from a K # Example +## Example: Token authentication + See [KoP Security](https://github.com/streamnative/kop/blob/master/docs/security.md) for how to configure KoP with token authentication. This example takes a topic named `my-topic` under `public/default` namespace as reference. 1. Grant produce and consume permissions to the specific role. @@ -64,3 +66,128 @@ See [KoP Security](https://github.com/streamnative/kop/blob/master/docs/security ``` Receive record: hello from persistent://public/default/my-topic-0@0 ``` + +## Example: OAuth2 authentication + +See [KoP Security](https://github.com/streamnative/kop/blob/master/docs/security.md#oauthbearer) for how to configure KoP with oauth authentication. This example takes a topic named `my-topic` under `public/default` namespace as reference. +1. Start the hydra OAuth2 Server + +Start hydra server +```shell +docker-compose -f $(git rev-parse --show-toplevel)/kop/java/src/main/resources/hydra/docker-compose.yml up -d +``` +Init the hydra server +```shell +./$(git rev-parse --show-toplevel)/kop/java/src/main/resources/init_hydra_oauth_server.sh +``` + +2. Configure the oauth in Pulsar + +In this example it will use the follow value: +> Need replace the privateKey value to your local credentials_file.json path. +```properties +# Enable the authentication +authenticationEnabled=true +authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken +superUserRoles=simple_client_id +brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 +brokerClientAuthenticationParameters={"type":"client_credentials","privateKey":"file:///path/to/simple_credentials_file.json","issuerUrl":"http://localhost:4444","audience":"http://example.com/api/v2/"} +tokenPublicKey=data:;base64,MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA4g8rgGslfLNGdfh94KbfsMPjgX17nnEHnCLhrlVyA+jxSThiQyQVQCkZfav9k4cLCiKdoqxKtLV0RA3hWXGHE0qUNUJWVN3vz3NOI7ccEHBJHzbDk24NYxsW7M6zNfBfTc6ZrJr5XENy7emscODn8HJ2Qf1UkMUeze5EirJ2lsB9Zzo1GIw9ZU65W9HWWcgS5sL9eHlDRbVLmgph7jRzkQJGm2hOeyiE+ufUOWkBQH49BhKaNGfjZ8BOJ1WRsbIIVtwhS7m+HSIKmglboG+onNd5LYAmngbkCuhwjJajBQayxkeBeumvRQACC1+mKC5KaW40JmVRKFFHDcf892t6GX6c7PaVWPqvf2l6nYRbYT9nl4fQK1aUTiCqrPf2+WjEH1JIEwTfFZKTwpTtlr3ejGJMT7wH2L4uFbpguKawTo4lYHWN3IsryDfUVvNbb7l8KMqiuDIy+5R6WezajsCYI/GzvLGCYO1EnRTDFdEmipfbNT2/D91OPKNGmZLVUkVVlL0z+1iQtwfRamn2oRNHzMYMAplGikxrQld/IPUIbKjgtLWPDnfskoWvuCIDQdRzMpxAXa3O/cq5uQRpu2o8xZ8RYWixxrIGc1/8m+QQLy7DwcmVd0dGU29S+fnfOzWr43KWlyWfGsBLFxUkltjY6gx6oB6tsQVC3Cy5Eku8FdcCAwEAAQ== + +# Use the KoP's built-in handler +kopOauth2AuthenticateCallbackHandler=io.streamnative.pulsar.handlers.kop.security.oauth.OauthValidatorCallbackHandler + +# Java property configuration file of OauthValidatorCallbackHandler +kopOauth2ConfigFile=conf/kop-handler.properties + +# Enable the authorization for test +authorizationEnabled=true +authorizationProvider=org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider +``` + +3. Create a new oauth client + +```shell +docker run --rm \ + --network hydra_default \ + oryd/hydra:v1.11.7 \ + clients create \ + --endpoint http://hydra:4445 \ + --id test_role \ + --secret test_secret \ + --grant-types client_credentials \ + --response-types token,code \ + --token-endpoint-auth-method client_secret_post \ + --audience http://example.com/api/v2/ +``` + +4. Create the credentials file json calls `credentials.json` + +```properties +{ + "client_id":"test_role", + "client_secret":"test_secret" +} + +``` + +5. Grant produce and consume permissions to the specific role. + + ```bash + bin/pulsar-admin namespaces grant-permission public/default \ + --role test_role \ + --actions produce,consume + ``` + +> **NOTE** +> +> The `conf/client.conf` should be configured. For details, see [Configure CLI Tools](http://pulsar.apache.org/docs/en/security-jwt/#cli-tools). + +6. Configure the oauth in [oauth.properties](src/main/resources/oauth.properties). + + ```properties + bootstrap.servers=localhost:9092 + topic=persistent://public/default/my-topic + group=my-group + issuerUrl=http://localhost:4444 + credentialsUrl=file:///path/to/credentials.json + audience=http://example.com/api/v2/ + ``` + +7. Compile the project. + + ``` + mvn clean compile + ``` + +8. Run a Kafka producer to produce a `hello` message. + + ```bash + mvn exec:java -Dexec.mainClass=io.streamnative.examples.kafka.OAuthProducer + ``` + + **Output:** + + ``` + Send hello to persistent://public/default/my-topic-0@0 + ``` + +9. Run a Kafka consumer to consume some messages. + + ```bash + mvn exec:java -Dexec.mainClass=io.streamnative.examples.kafka.OAuthConsumer + ``` + + **Output:** + + ``` + Receive record: hello from persistent://public/default/my-topic-0@0 + ``` + +10. Stop the hydra server + +```shell +docker-compose -f $(git rev-parse --show-toplevel)/kop/java/src/main/resources/hydra/docker-compose.yml down -d +``` + +11. Stop the Pulsar server \ No newline at end of file diff --git a/kop/java/pom.xml b/kop/java/pom.xml index 25b458d..9bf5e2c 100644 --- a/kop/java/pom.xml +++ b/kop/java/pom.xml @@ -41,6 +41,12 @@ ${slf4j.simple.version} runtime + + + io.streamnative.pulsar.handlers + oauth-client + 2.9.2.8 + diff --git a/kop/java/src/main/java/io/streamnative/examples/kafka/OAuthConsumer.java b/kop/java/src/main/java/io/streamnative/examples/kafka/OAuthConsumer.java new file mode 100644 index 0000000..51bcc00 --- /dev/null +++ b/kop/java/src/main/java/io/streamnative/examples/kafka/OAuthConsumer.java @@ -0,0 +1,74 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.streamnative.examples.kafka; + +import io.streamnative.pulsar.handlers.kop.security.oauth.OauthLoginCallbackHandler; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.serialization.StringDeserializer; +import java.io.IOException; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.Collections; +import java.util.Properties; + +public class OAuthConsumer { + + public static void main(String[] args) throws IOException { + // 1. Get the configured parameters from oauth.properties + final Properties properties = new Properties(); + properties.load(TokenProducer.class.getClassLoader().getResourceAsStream("oauth.properties")); + String bootstrapServers = properties.getProperty("bootstrap.servers"); + String topic = properties.getProperty("topic"); + String group = properties.getProperty("group"); + String issuerUrl = properties.getProperty("issuerUrl"); + String credentialsUrl = properties.getProperty("credentialsUrl"); + String audience = properties.getProperty("audience"); + + // 2. Create a consumer with OAuth authentication. + final Properties props = new Properties(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + props.put(ConsumerConfig.GROUP_ID_CONFIG, group); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + props.setProperty("sasl.login.callback.handler.class", OauthLoginCallbackHandler.class.getName()); + props.setProperty("security.protocol", "SASL_PLAINTEXT"); + props.setProperty("sasl.mechanism", "OAUTHBEARER"); + final String jaasTemplate = "org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required" + + " oauth.issuer.url=\"%s\"" + + " oauth.credentials.url=\"%s\"" + + " oauth.audience=\"%s\";"; + props.setProperty("sasl.jaas.config", String.format(jaasTemplate, + issuerUrl, + "file://" + Paths.get(credentialsUrl).toAbsolutePath(), + audience + )); + final KafkaConsumer consumer = new KafkaConsumer<>(props); + consumer.subscribe(Collections.singleton(topic)); + + // 3. Consume some messages and quit immediately + boolean running = true; + while (running) { + final ConsumerRecords records = consumer.poll(Duration.ofSeconds(1)); + if (!records.isEmpty()) { + records.forEach(record -> System.out.println("Receive record: " + record.value() + " from " + + record.topic() + "-" + record.partition() + "@" + record.offset())); + running = false; + } + } + consumer.close(); + } +} diff --git a/kop/java/src/main/java/io/streamnative/examples/kafka/OAuthProducer.java b/kop/java/src/main/java/io/streamnative/examples/kafka/OAuthProducer.java new file mode 100644 index 0000000..1600fb3 --- /dev/null +++ b/kop/java/src/main/java/io/streamnative/examples/kafka/OAuthProducer.java @@ -0,0 +1,65 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.streamnative.examples.kafka; + +import io.streamnative.pulsar.handlers.kop.security.oauth.OauthLoginCallbackHandler; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.serialization.StringSerializer; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +public class OAuthProducer { + + public static void main(String[] args) throws IOException, ExecutionException, InterruptedException { + // 1. Get the configured parameters from oauth.properties + final Properties properties = new Properties(); + properties.load(OAuthProducer.class.getClassLoader().getResourceAsStream("oauth.properties")); + String bootstrapServers = properties.getProperty("bootstrap.servers"); + String topic = properties.getProperty("topic"); + String issuerUrl = properties.getProperty("issuerUrl"); + String credentialsUrl = properties.getProperty("credentialsUrl"); + String audience = properties.getProperty("audience"); + + // 2. Create a producer with OAuth authentication. + final Properties props = new Properties(); + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + props.setProperty("sasl.login.callback.handler.class", OauthLoginCallbackHandler.class.getName()); + props.setProperty("security.protocol", "SASL_PLAINTEXT"); + props.setProperty("sasl.mechanism", "OAUTHBEARER"); + final String jaasTemplate = "org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required" + + " oauth.issuer.url=\"%s\"" + + " oauth.credentials.url=\"%s\"" + + " oauth.audience=\"%s\";"; + props.setProperty("sasl.jaas.config", String.format(jaasTemplate, + issuerUrl, + "file://" + Paths.get(credentialsUrl).toAbsolutePath(), + audience + )); + final KafkaProducer producer = new KafkaProducer<>(props); + + // 3. Produce one message + final Future recordMetadataFuture = producer.send(new ProducerRecord<>(topic, "hello")); + final RecordMetadata recordMetadata = recordMetadataFuture.get(); + System.out.println("Send hello to " + recordMetadata); + producer.close(); + } +} diff --git a/kop/java/src/main/resources/hydra/docker-compose.yml b/kop/java/src/main/resources/hydra/docker-compose.yml new file mode 100644 index 0000000..6241f18 --- /dev/null +++ b/kop/java/src/main/resources/hydra/docker-compose.yml @@ -0,0 +1,35 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '3' + +services: + + hydra: + image: oryd/hydra:v1.11.7 + container_name: hydra + ports: + - 4444:4444 + - 4445:4445 + - 9020:9020 + command: serve all --dangerous-force-http + environment: + - DSN=memory + - URLS_SELF_ISSUER=http://127.0.0.1:4444/ + - URLS_CONSENT=http://127.0.0.1:9020/consent + - URLS_LOGIN=http://127.0.0.1:9020/login + - SERVE_PUBLIC_PORT=4444 + - SERVE_ADMIN_PORT=4445 + - STRATEGIES_ACCESS_TOKEN=jwt + - OIDC_SUBJECT_IDENTIFIERS_SUPPORTED_TYPES=public diff --git a/kop/java/src/main/resources/hydra/keys/private_key.json b/kop/java/src/main/resources/hydra/keys/private_key.json new file mode 100644 index 0000000..02d37b0 --- /dev/null +++ b/kop/java/src/main/resources/hydra/keys/private_key.json @@ -0,0 +1,12 @@ +{ + "alg": "RS256", + "d": "eojGh8GMfF-g3eloMhHCRsqn01T1YRjbiyLNOfZO6xelUo8hlEtGyZR9oMVNml4k-cVxtO-3PuTstMbhU95Z3XcbhxMCQNZcWxKhVHK436A9wpGoyY1p7EKO1qXkNmSpBD_sxMbsS7qz3YpAUp9WmdsxhuuOnzp6TB3PQW8jIOwODZpblYbO8M8PYloH2nq85CDlzBvO_0YaSNE_7CK6Uevt4edUQyXfjzGCs-vbJd5Hrbb_p1B4z5BJzMBxTOS36H1B_w4boycEoCJabEiaGQojQSqpxBbwHYmJGGu-ycaruRahdMKeosQyV3_tTPJqm2OFGRrqNvR3k1zIHkU_1PUbyFQouhp36n87uPUDGqeTJ-fzK7XkcDy27XFwoDmCDdsBHiLl4Z-608jc8DePmtPd4DrYr4nxQxLzSO8LWZOmvxnkLPXd4qO-_XSBx7kQQpbdHYFXysZ_A_YVFtJPD8_2qXux-o_hbJxDUiNLUr6SPiIVTgkYbzShLyh1uyVWu-rXUE1lQl9JEVAiNOq4S_ZjRRyG_GEJeozIAfeX_rKn1bMOPdDSifc70tcY9ZDmJmYsMp0obNtk7x4VMbxS--wiwQiTASaE5tt1DTGzp0BZR-eo3WwXXCE-1Wd1-rUSjNt08kSksha8rBQgV6bqUIsMtZ6NrBTLY2pI_qGPjUE", + "e": "AQAB", + "kid": "private:7e9cbc3a-20f4-44aa-831c-75471946a7dc", + "kty": "RSA", + "n": "4g8rgGslfLNGdfh94KbfsMPjgX17nnEHnCLhrlVyA-jxSThiQyQVQCkZfav9k4cLCiKdoqxKtLV0RA3hWXGHE0qUNUJWVN3vz3NOI7ccEHBJHzbDk24NYxsW7M6zNfBfTc6ZrJr5XENy7emscODn8HJ2Qf1UkMUeze5EirJ2lsB9Zzo1GIw9ZU65W9HWWcgS5sL9eHlDRbVLmgph7jRzkQJGm2hOeyiE-ufUOWkBQH49BhKaNGfjZ8BOJ1WRsbIIVtwhS7m-HSIKmglboG-onNd5LYAmngbkCuhwjJajBQayxkeBeumvRQACC1-mKC5KaW40JmVRKFFHDcf892t6GX6c7PaVWPqvf2l6nYRbYT9nl4fQK1aUTiCqrPf2-WjEH1JIEwTfFZKTwpTtlr3ejGJMT7wH2L4uFbpguKawTo4lYHWN3IsryDfUVvNbb7l8KMqiuDIy-5R6WezajsCYI_GzvLGCYO1EnRTDFdEmipfbNT2_D91OPKNGmZLVUkVVlL0z-1iQtwfRamn2oRNHzMYMAplGikxrQld_IPUIbKjgtLWPDnfskoWvuCIDQdRzMpxAXa3O_cq5uQRpu2o8xZ8RYWixxrIGc1_8m-QQLy7DwcmVd0dGU29S-fnfOzWr43KWlyWfGsBLFxUkltjY6gx6oB6tsQVC3Cy5Eku8Fdc", + "p": "7Lb4GzmiAzIpZZbhSt-mQsequ4lgiKAbSnJ7nHmmaRoa8MSS5N-5hxPu_PMDgJSBnC4m8b9LArDTBzS3_b_FN_AC8v1ZXKraiKN0s7QCQuokQwia4MXCvZg9taFqlQEd_c0w6hY6RYZFqAXF6baNb6smE2VJ3RTqTbPY8Gv0NN0KHOUONfZpsDlXUMNSMI8hlz6t9fEqNSrp_P_dgPjv6li1ycyG5sNT8YjuSuy9PuFG7go4RI5K_2j4uzRGVOmidQ4G1kPPUMvUHPnlvqiH7ofD8cMnnOsvHrUwBVXV7zgMK9aG952uz2Sa27AoSG79PWMtVr5_1wcO5ddEzSe_GQ", + "q": "9Hn3KdBbBNmg6QIWU_k9SbhYEHwJFsQo6ZioDvi26-5WnNgYaRH6JEHa44aD_LE_0Rhkvf9DBr9DiMgB6VV6q_ZVLMGGMZFE11VPh7HU-_JMMkfJN1SEhNUtu9eWyP4suuxjHjD8EQO__efXZO76zvcnB9m6tnhjy37zacqveXgK3CB-lX2BzQEyexh2xgf6NAZorFruohc-KeD2YGqH4XMYPOAgjRgaGgEvV8RNdNOA_qUedFQkIz_XWoB93TmYmxr7ezIkDGjKcJ0Cmb0kpGY_gD5tm7xec6WcQAkMaasV7cCvV9bE19wd1_4TXYW-4UWBvy0tTNmmNMbQu7tKbw", + "use": "sig", + "x5c": null +} \ No newline at end of file diff --git a/kop/java/src/main/resources/hydra/keys/public_key.json b/kop/java/src/main/resources/hydra/keys/public_key.json new file mode 100644 index 0000000..4dafb50 --- /dev/null +++ b/kop/java/src/main/resources/hydra/keys/public_key.json @@ -0,0 +1,9 @@ +{ + "alg": "RS256", + "e": "AQAB", + "kid": "public:7e9cbc3a-20f4-44aa-831c-75471946a7dc", + "kty": "RSA", + "n": "4g8rgGslfLNGdfh94KbfsMPjgX17nnEHnCLhrlVyA-jxSThiQyQVQCkZfav9k4cLCiKdoqxKtLV0RA3hWXGHE0qUNUJWVN3vz3NOI7ccEHBJHzbDk24NYxsW7M6zNfBfTc6ZrJr5XENy7emscODn8HJ2Qf1UkMUeze5EirJ2lsB9Zzo1GIw9ZU65W9HWWcgS5sL9eHlDRbVLmgph7jRzkQJGm2hOeyiE-ufUOWkBQH49BhKaNGfjZ8BOJ1WRsbIIVtwhS7m-HSIKmglboG-onNd5LYAmngbkCuhwjJajBQayxkeBeumvRQACC1-mKC5KaW40JmVRKFFHDcf892t6GX6c7PaVWPqvf2l6nYRbYT9nl4fQK1aUTiCqrPf2-WjEH1JIEwTfFZKTwpTtlr3ejGJMT7wH2L4uFbpguKawTo4lYHWN3IsryDfUVvNbb7l8KMqiuDIy-5R6WezajsCYI_GzvLGCYO1EnRTDFdEmipfbNT2_D91OPKNGmZLVUkVVlL0z-1iQtwfRamn2oRNHzMYMAplGikxrQld_IPUIbKjgtLWPDnfskoWvuCIDQdRzMpxAXa3O_cq5uQRpu2o8xZ8RYWixxrIGc1_8m-QQLy7DwcmVd0dGU29S-fnfOzWr43KWlyWfGsBLFxUkltjY6gx6oB6tsQVC3Cy5Eku8Fdc", + "use": "sig", + "x5c": null +} \ No newline at end of file diff --git a/kop/java/src/main/resources/init_hydra_oauth_server.sh b/kop/java/src/main/resources/init_hydra_oauth_server.sh new file mode 100755 index 0000000..c0550b6 --- /dev/null +++ b/kop/java/src/main/resources/init_hydra_oauth_server.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +SRC_DIR=$(git rev-parse --show-toplevel) +cd "$SRC_DIR" + +wait_for_url() { + URL=$1 + MSG=$2 + + if [[ $URL == https* ]]; then + CMD="curl -k -sL -o /dev/null -w %{http_code} $URL" + else + CMD="curl -sL -o /dev/null -w %{http_code} $URL" + fi + + until [ "200" == "$($CMD)" ] + do + echo "$MSG ($URL)" + sleep 2 + done +} + +# Wait until the hydra server started +wait_for_url "http://localhost:4445/clients" "Waiting for Hydra admin REST to start" + +# Delete hydra default access token +docker run --rm \ + --network hydra_default \ + oryd/hydra:v1.11.7 \ + --endpoint=http://hydra:4445 \ + keys delete hydra.jwt.access-token + +# Replace hydra access token +docker run --rm \ + --network hydra_default \ + -v "$(pwd)"/kop/java/src/main/resources/hydra/keys:/tmp/keys \ + oryd/hydra:v1.11.7 \ + --endpoint=http://hydra:4445 \ + keys import hydra.jwt.access-token /tmp/keys/private_key.json /tmp/keys/public_key.json + +# Create a new client, the client id is simple_client_id and secret is simple_client_secret +# Correspond pulsar role is simple_client_id +docker run --rm \ + --network hydra_default \ + oryd/hydra:v1.11.7 \ + clients create \ + --endpoint http://hydra:4445 \ + --id simple_client_id \ + --secret simple_client_secret \ + --grant-types client_credentials \ + --response-types token,code \ + --token-endpoint-auth-method client_secret_post \ + --audience http://example.com/api/v2/ + +# Try to generate a token using the new client +docker run --rm \ + --network hydra_default \ + oryd/hydra:v1.11.7 \ + token client \ + --client-id simple_client_id \ + --client-secret simple_client_secret \ + --endpoint http://hydra:4444 + diff --git a/kop/java/src/main/resources/oauth.properties b/kop/java/src/main/resources/oauth.properties new file mode 100644 index 0000000..d9a21a3 --- /dev/null +++ b/kop/java/src/main/resources/oauth.properties @@ -0,0 +1,20 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +bootstrap.servers=localhost:9092 +topic=persistent://public/default/my-topic +group=my-group +issuerUrl=http://localhost:4444 +credentialsUrl=/path/to/credentials.json +audience=http://example.com/api/v2/ \ No newline at end of file diff --git a/kop/java/src/main/resources/oauth_credentials/credentials.json b/kop/java/src/main/resources/oauth_credentials/credentials.json new file mode 100644 index 0000000..2847694 --- /dev/null +++ b/kop/java/src/main/resources/oauth_credentials/credentials.json @@ -0,0 +1,4 @@ +{ + "client_id":"test_role", + "client_secret":"test_secret" +} \ No newline at end of file diff --git a/kop/java/src/main/resources/oauth_credentials/simple_credentials_file.json b/kop/java/src/main/resources/oauth_credentials/simple_credentials_file.json new file mode 100644 index 0000000..62af531 --- /dev/null +++ b/kop/java/src/main/resources/oauth_credentials/simple_credentials_file.json @@ -0,0 +1,4 @@ +{ + "client_id":"simple_client_id", + "client_secret":"simple_client_secret" +} From 66e779d6657dfbc25fc55322f29567de8222adfb Mon Sep 17 00:00:00 2001 From: Demogorgon314 Date: Tue, 19 Apr 2022 08:53:12 +0800 Subject: [PATCH 2/8] Fix stop hydra server command --- kop/java/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kop/java/README.md b/kop/java/README.md index 47eade8..c8d1202 100644 --- a/kop/java/README.md +++ b/kop/java/README.md @@ -187,7 +187,7 @@ docker run --rm \ 10. Stop the hydra server ```shell -docker-compose -f $(git rev-parse --show-toplevel)/kop/java/src/main/resources/hydra/docker-compose.yml down -d +docker-compose -f $(git rev-parse --show-toplevel)/kop/java/src/main/resources/hydra/docker-compose.yml down ``` 11. Stop the Pulsar server \ No newline at end of file From a8bfa36cf14409ca0de8e3002b54c60d8ad70016 Mon Sep 17 00:00:00 2001 From: Demogorgon314 Date: Tue, 19 Apr 2022 21:59:22 +0800 Subject: [PATCH 3/8] Fix json format --- .../resources/oauth_credentials/simple_credentials_file.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kop/java/src/main/resources/oauth_credentials/simple_credentials_file.json b/kop/java/src/main/resources/oauth_credentials/simple_credentials_file.json index 62af531..9979090 100644 --- a/kop/java/src/main/resources/oauth_credentials/simple_credentials_file.json +++ b/kop/java/src/main/resources/oauth_credentials/simple_credentials_file.json @@ -1,4 +1,4 @@ { "client_id":"simple_client_id", "client_secret":"simple_client_secret" -} +} \ No newline at end of file From cc3ecfd4199a907824beef522493ed0ca6018ea7 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Wed, 20 Apr 2022 09:29:33 +0800 Subject: [PATCH 4/8] Update kop/java/README.md Co-authored-by: Huanli Meng <48120384+Huanli-Meng@users.noreply.github.com> --- kop/java/README.md | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/kop/java/README.md b/kop/java/README.md index c8d1202..c0c9dd2 100644 --- a/kop/java/README.md +++ b/kop/java/README.md @@ -69,22 +69,24 @@ See [KoP Security](https://github.com/streamnative/kop/blob/master/docs/security ## Example: OAuth2 authentication -See [KoP Security](https://github.com/streamnative/kop/blob/master/docs/security.md#oauthbearer) for how to configure KoP with oauth authentication. This example takes a topic named `my-topic` under `public/default` namespace as reference. -1. Start the hydra OAuth2 Server +See [KoP Security](https://github.com/streamnative/kop/blob/master/docs/security.md#oauthbearer) for how to configure KoP with OAuth authentication. This example takes a topic named `my-topic` under `public/default` namespace as reference. +1. Start the Hydra OAuth2 server. -Start hydra server +Start the Hydra OAuth2 server. ```shell docker-compose -f $(git rev-parse --show-toplevel)/kop/java/src/main/resources/hydra/docker-compose.yml up -d ``` -Init the hydra server +Initialize the Hydra OAuth2 server. ```shell ./$(git rev-parse --show-toplevel)/kop/java/src/main/resources/init_hydra_oauth_server.sh ``` 2. Configure the oauth in Pulsar -In this example it will use the follow value: -> Need replace the privateKey value to your local credentials_file.json path. +This example will use the follow values: +> **Note** +> +> You need to replace the `privateKey` value with your local path to your `credentials_file.json` file. ```properties # Enable the authentication authenticationEnabled=true @@ -105,7 +107,7 @@ authorizationEnabled=true authorizationProvider=org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider ``` -3. Create a new oauth client +3. Create a new OAuth2 client. ```shell docker run --rm \ @@ -121,7 +123,7 @@ docker run --rm \ --audience http://example.com/api/v2/ ``` -4. Create the credentials file json calls `credentials.json` +4. Create a credentials file json named `credentials.json` ```properties { @@ -139,11 +141,11 @@ docker run --rm \ --actions produce,consume ``` -> **NOTE** +> **Note** > > The `conf/client.conf` should be configured. For details, see [Configure CLI Tools](http://pulsar.apache.org/docs/en/security-jwt/#cli-tools). -6. Configure the oauth in [oauth.properties](src/main/resources/oauth.properties). +6. Configure OAuth2 authentication parameters in [oauth.properties](src/main/resources/oauth.properties). ```properties bootstrap.servers=localhost:9092 @@ -184,10 +186,10 @@ docker run --rm \ Receive record: hello from persistent://public/default/my-topic-0@0 ``` -10. Stop the hydra server +10. Stop the Hydra OAuth2 server. ```shell docker-compose -f $(git rev-parse --show-toplevel)/kop/java/src/main/resources/hydra/docker-compose.yml down ``` -11. Stop the Pulsar server \ No newline at end of file +11. Stop the Pulsar server. \ No newline at end of file From a27356117ed198ceb94af5a0f7051926d5c09d17 Mon Sep 17 00:00:00 2001 From: Demogorgon314 Date: Wed, 20 Apr 2022 13:26:19 +0800 Subject: [PATCH 5/8] Change oauth-client version --- kop/java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kop/java/pom.xml b/kop/java/pom.xml index 9bf5e2c..c9b4a3d 100644 --- a/kop/java/pom.xml +++ b/kop/java/pom.xml @@ -45,7 +45,7 @@ io.streamnative.pulsar.handlers oauth-client - 2.9.2.8 + 2.8.3.1 From 009b452c36c8a25c7361356294f0609c58a2fbc8 Mon Sep 17 00:00:00 2001 From: Demogorgon314 Date: Thu, 21 Apr 2022 13:35:11 +0800 Subject: [PATCH 6/8] Use online oauth server in example --- kop/java/README.md | 134 +++++++----------- kop/java/src/main/resources/credentials.json | 6 + .../resources/hydra/keys/private_key.json | 12 -- .../main/resources/hydra/keys/public_key.json | 9 -- .../main/resources/init_hydra_oauth_server.sh | 78 ---------- ...pose.yml => kop-handler-oauth2.properties} | 22 +-- kop/java/src/main/resources/oauth.properties | 4 +- .../oauth_credentials/credentials.json | 4 - .../simple_credentials_file.json | 4 - 9 files changed, 61 insertions(+), 212 deletions(-) create mode 100644 kop/java/src/main/resources/credentials.json delete mode 100644 kop/java/src/main/resources/hydra/keys/private_key.json delete mode 100644 kop/java/src/main/resources/hydra/keys/public_key.json delete mode 100755 kop/java/src/main/resources/init_hydra_oauth_server.sh rename kop/java/src/main/resources/{hydra/docker-compose.yml => kop-handler-oauth2.properties} (50%) delete mode 100644 kop/java/src/main/resources/oauth_credentials/credentials.json delete mode 100644 kop/java/src/main/resources/oauth_credentials/simple_credentials_file.json diff --git a/kop/java/README.md b/kop/java/README.md index c0c9dd2..855d215 100644 --- a/kop/java/README.md +++ b/kop/java/README.md @@ -70,99 +70,77 @@ See [KoP Security](https://github.com/streamnative/kop/blob/master/docs/security ## Example: OAuth2 authentication See [KoP Security](https://github.com/streamnative/kop/blob/master/docs/security.md#oauthbearer) for how to configure KoP with OAuth authentication. This example takes a topic named `my-topic` under `public/default` namespace as reference. -1. Start the Hydra OAuth2 server. - -Start the Hydra OAuth2 server. -```shell -docker-compose -f $(git rev-parse --show-toplevel)/kop/java/src/main/resources/hydra/docker-compose.yml up -d -``` -Initialize the Hydra OAuth2 server. -```shell -./$(git rev-parse --show-toplevel)/kop/java/src/main/resources/init_hydra_oauth_server.sh -``` - -2. Configure the oauth in Pulsar - -This example will use the follow values: +1. Configure the pulsar broker, this example will use the follow values: > **Note** > -> You need to replace the `privateKey` value with your local path to your `credentials_file.json` file. -```properties -# Enable the authentication -authenticationEnabled=true -authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken -superUserRoles=simple_client_id -brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 -brokerClientAuthenticationParameters={"type":"client_credentials","privateKey":"file:///path/to/simple_credentials_file.json","issuerUrl":"http://localhost:4444","audience":"http://example.com/api/v2/"} -tokenPublicKey=data:;base64,MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA4g8rgGslfLNGdfh94KbfsMPjgX17nnEHnCLhrlVyA+jxSThiQyQVQCkZfav9k4cLCiKdoqxKtLV0RA3hWXGHE0qUNUJWVN3vz3NOI7ccEHBJHzbDk24NYxsW7M6zNfBfTc6ZrJr5XENy7emscODn8HJ2Qf1UkMUeze5EirJ2lsB9Zzo1GIw9ZU65W9HWWcgS5sL9eHlDRbVLmgph7jRzkQJGm2hOeyiE+ufUOWkBQH49BhKaNGfjZ8BOJ1WRsbIIVtwhS7m+HSIKmglboG+onNd5LYAmngbkCuhwjJajBQayxkeBeumvRQACC1+mKC5KaW40JmVRKFFHDcf892t6GX6c7PaVWPqvf2l6nYRbYT9nl4fQK1aUTiCqrPf2+WjEH1JIEwTfFZKTwpTtlr3ejGJMT7wH2L4uFbpguKawTo4lYHWN3IsryDfUVvNbb7l8KMqiuDIy+5R6WezajsCYI/GzvLGCYO1EnRTDFdEmipfbNT2/D91OPKNGmZLVUkVVlL0z+1iQtwfRamn2oRNHzMYMAplGikxrQld/IPUIbKjgtLWPDnfskoWvuCIDQdRzMpxAXa3O/cq5uQRpu2o8xZ8RYWixxrIGc1/8m+QQLy7DwcmVd0dGU29S+fnfOzWr43KWlyWfGsBLFxUkltjY6gx6oB6tsQVC3Cy5Eku8FdcCAwEAAQ== - -# Use the KoP's built-in handler -kopOauth2AuthenticateCallbackHandler=io.streamnative.pulsar.handlers.kop.security.oauth.OauthValidatorCallbackHandler - -# Java property configuration file of OauthValidatorCallbackHandler -kopOauth2ConfigFile=conf/kop-handler.properties - -# Enable the authorization for test -authorizationEnabled=true -authorizationProvider=org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider -``` - -3. Create a new OAuth2 client. - -```shell -docker run --rm \ - --network hydra_default \ - oryd/hydra:v1.11.7 \ - clients create \ - --endpoint http://hydra:4445 \ - --id test_role \ - --secret test_secret \ - --grant-types client_credentials \ - --response-types token,code \ - --token-endpoint-auth-method client_secret_post \ - --audience http://example.com/api/v2/ -``` - -4. Create a credentials file json named `credentials.json` - -```properties -{ - "client_id":"test_role", - "client_secret":"test_secret" -} - -``` - -5. Grant produce and consume permissions to the specific role. +> Need to change the `credentials.json` and `kop-handler-oauth2.properties` paths to your local path. The example file can be found in `src/main/resources/`. +> + ```properties + # Enable KoP + messagingProtocols=kafka + protocolHandlerDirectory=./protocols + allowAutoTopicCreationType=partitioned + + # Use `kafkaListeners` here for KoP 2.8.0 because `listeners` is marked as deprecated from KoP 2.8.0 + kafkaListeners=PLAINTEXT://127.0.0.1:9092 + # This config is not required unless you want to expose another address to the Kafka client. + # If it’s not configured, it will be the same with `kafkaListeners` config by default + kafkaAdvertisedListeners=PLAINTEXT://127.0.0.1:9092 + brokerEntryMetadataInterceptors=org.apache.pulsar.common.intercept.AppendIndexMetadataInterceptor + + brokerDeleteInactiveTopicsEnabled=false + + # Enable the authentication + authenticationEnabled=true + authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken + superUserRoles=Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x@clients + brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 + brokerClientAuthenticationParameters={"type":"client_credentials","privateKey":"/path/to/credentials.json","issuerUrl":"https://dev-kt-aa9ne.us.auth0.com","audience":"https://dev-kt-aa9ne.us.auth0.com/api/v2/"} + tokenPublicKey=data:;base64,MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2tZd/4gJda3U2Pc3tpgRAN7JPGWx/Gn17v/0IiZlNNRbP/Mmf0Vc6G1qsnaRaWNWOR+t6/a6ekFHJMikQ1N2X6yfz4UjMc8/G2FDPRmWjA+GURzARjVhxc/BBEYGoD0Kwvbq/u9CZm2QjlKrYaLfg3AeB09j0btNrDJ8rBsNzU6AuzChRvXj9IdcE/A/4N/UQ+S9cJ4UXP6NJbToLwajQ5km+CnxdGE6nfB7LWHvOFHjn9C2Rb9e37CFlmeKmIVFkagFM0gbmGOb6bnGI8Bp/VNGV0APef4YaBvBTqwoZ1Z4aDHy5eRxXfAMdtBkBupmBXqL6bpd15XRYUbu/7ck9QIDAQAB + + # Use the KoP's built-in handler + kopOauth2AuthenticateCallbackHandler=io.streamnative.pulsar.handlers.kop.security.oauth.OauthValidatorCallbackHandler + + # Java property configuration file of OauthValidatorCallbackHandler + kopOauth2ConfigFile=/path/to/kop-handler-oauth2.properties + + saslAllowedMechanisms=OAUTHBEARER + ``` - ```bash - bin/pulsar-admin namespaces grant-permission public/default \ - --role test_role \ - --actions produce,consume + +2. Configure the credentials in [credentials.json](src/main/resources/credentials.json). + + ```json + { + "client_id":"Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x", + "client_secret":"rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb", + "audience":"https://dev-kt-aa9ne.us.auth0.com/api/v2/", + "grant_type":"client_credentials" + } ``` -> **Note** +> **NOTE** > > The `conf/client.conf` should be configured. For details, see [Configure CLI Tools](http://pulsar.apache.org/docs/en/security-jwt/#cli-tools). -6. Configure OAuth2 authentication parameters in [oauth.properties](src/main/resources/oauth.properties). +2. Configure the oauth in [oauth.properties](src/main/resources/oauth.properties). ```properties bootstrap.servers=localhost:9092 topic=persistent://public/default/my-topic group=my-group - issuerUrl=http://localhost:4444 - credentialsUrl=file:///path/to/credentials.json - audience=http://example.com/api/v2/ + issuerUrl=https://dev-kt-aa9ne.us.auth0.com + credentialsUrl=/path/to/credentials.json + audience=https://dev-kt-aa9ne.us.auth0.com/api/v2/ ``` -7. Compile the project. +3. Compile the project. ``` mvn clean compile ``` -8. Run a Kafka producer to produce a `hello` message. +4. Run a Kafka producer to produce a `hello` message. ```bash mvn exec:java -Dexec.mainClass=io.streamnative.examples.kafka.OAuthProducer @@ -174,7 +152,7 @@ docker run --rm \ Send hello to persistent://public/default/my-topic-0@0 ``` -9. Run a Kafka consumer to consume some messages. +5. Run a Kafka consumer to consume some messages. ```bash mvn exec:java -Dexec.mainClass=io.streamnative.examples.kafka.OAuthConsumer @@ -185,11 +163,3 @@ docker run --rm \ ``` Receive record: hello from persistent://public/default/my-topic-0@0 ``` - -10. Stop the Hydra OAuth2 server. - -```shell -docker-compose -f $(git rev-parse --show-toplevel)/kop/java/src/main/resources/hydra/docker-compose.yml down -``` - -11. Stop the Pulsar server. \ No newline at end of file diff --git a/kop/java/src/main/resources/credentials.json b/kop/java/src/main/resources/credentials.json new file mode 100644 index 0000000..ce61f69 --- /dev/null +++ b/kop/java/src/main/resources/credentials.json @@ -0,0 +1,6 @@ +{ + "client_id":"Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x", + "client_secret":"rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb", + "audience":"https://dev-kt-aa9ne.us.auth0.com/api/v2/", + "grant_type":"client_credentials" +} \ No newline at end of file diff --git a/kop/java/src/main/resources/hydra/keys/private_key.json b/kop/java/src/main/resources/hydra/keys/private_key.json deleted file mode 100644 index 02d37b0..0000000 --- a/kop/java/src/main/resources/hydra/keys/private_key.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "alg": "RS256", - "d": "eojGh8GMfF-g3eloMhHCRsqn01T1YRjbiyLNOfZO6xelUo8hlEtGyZR9oMVNml4k-cVxtO-3PuTstMbhU95Z3XcbhxMCQNZcWxKhVHK436A9wpGoyY1p7EKO1qXkNmSpBD_sxMbsS7qz3YpAUp9WmdsxhuuOnzp6TB3PQW8jIOwODZpblYbO8M8PYloH2nq85CDlzBvO_0YaSNE_7CK6Uevt4edUQyXfjzGCs-vbJd5Hrbb_p1B4z5BJzMBxTOS36H1B_w4boycEoCJabEiaGQojQSqpxBbwHYmJGGu-ycaruRahdMKeosQyV3_tTPJqm2OFGRrqNvR3k1zIHkU_1PUbyFQouhp36n87uPUDGqeTJ-fzK7XkcDy27XFwoDmCDdsBHiLl4Z-608jc8DePmtPd4DrYr4nxQxLzSO8LWZOmvxnkLPXd4qO-_XSBx7kQQpbdHYFXysZ_A_YVFtJPD8_2qXux-o_hbJxDUiNLUr6SPiIVTgkYbzShLyh1uyVWu-rXUE1lQl9JEVAiNOq4S_ZjRRyG_GEJeozIAfeX_rKn1bMOPdDSifc70tcY9ZDmJmYsMp0obNtk7x4VMbxS--wiwQiTASaE5tt1DTGzp0BZR-eo3WwXXCE-1Wd1-rUSjNt08kSksha8rBQgV6bqUIsMtZ6NrBTLY2pI_qGPjUE", - "e": "AQAB", - "kid": "private:7e9cbc3a-20f4-44aa-831c-75471946a7dc", - "kty": "RSA", - "n": "4g8rgGslfLNGdfh94KbfsMPjgX17nnEHnCLhrlVyA-jxSThiQyQVQCkZfav9k4cLCiKdoqxKtLV0RA3hWXGHE0qUNUJWVN3vz3NOI7ccEHBJHzbDk24NYxsW7M6zNfBfTc6ZrJr5XENy7emscODn8HJ2Qf1UkMUeze5EirJ2lsB9Zzo1GIw9ZU65W9HWWcgS5sL9eHlDRbVLmgph7jRzkQJGm2hOeyiE-ufUOWkBQH49BhKaNGfjZ8BOJ1WRsbIIVtwhS7m-HSIKmglboG-onNd5LYAmngbkCuhwjJajBQayxkeBeumvRQACC1-mKC5KaW40JmVRKFFHDcf892t6GX6c7PaVWPqvf2l6nYRbYT9nl4fQK1aUTiCqrPf2-WjEH1JIEwTfFZKTwpTtlr3ejGJMT7wH2L4uFbpguKawTo4lYHWN3IsryDfUVvNbb7l8KMqiuDIy-5R6WezajsCYI_GzvLGCYO1EnRTDFdEmipfbNT2_D91OPKNGmZLVUkVVlL0z-1iQtwfRamn2oRNHzMYMAplGikxrQld_IPUIbKjgtLWPDnfskoWvuCIDQdRzMpxAXa3O_cq5uQRpu2o8xZ8RYWixxrIGc1_8m-QQLy7DwcmVd0dGU29S-fnfOzWr43KWlyWfGsBLFxUkltjY6gx6oB6tsQVC3Cy5Eku8Fdc", - "p": "7Lb4GzmiAzIpZZbhSt-mQsequ4lgiKAbSnJ7nHmmaRoa8MSS5N-5hxPu_PMDgJSBnC4m8b9LArDTBzS3_b_FN_AC8v1ZXKraiKN0s7QCQuokQwia4MXCvZg9taFqlQEd_c0w6hY6RYZFqAXF6baNb6smE2VJ3RTqTbPY8Gv0NN0KHOUONfZpsDlXUMNSMI8hlz6t9fEqNSrp_P_dgPjv6li1ycyG5sNT8YjuSuy9PuFG7go4RI5K_2j4uzRGVOmidQ4G1kPPUMvUHPnlvqiH7ofD8cMnnOsvHrUwBVXV7zgMK9aG952uz2Sa27AoSG79PWMtVr5_1wcO5ddEzSe_GQ", - "q": "9Hn3KdBbBNmg6QIWU_k9SbhYEHwJFsQo6ZioDvi26-5WnNgYaRH6JEHa44aD_LE_0Rhkvf9DBr9DiMgB6VV6q_ZVLMGGMZFE11VPh7HU-_JMMkfJN1SEhNUtu9eWyP4suuxjHjD8EQO__efXZO76zvcnB9m6tnhjy37zacqveXgK3CB-lX2BzQEyexh2xgf6NAZorFruohc-KeD2YGqH4XMYPOAgjRgaGgEvV8RNdNOA_qUedFQkIz_XWoB93TmYmxr7ezIkDGjKcJ0Cmb0kpGY_gD5tm7xec6WcQAkMaasV7cCvV9bE19wd1_4TXYW-4UWBvy0tTNmmNMbQu7tKbw", - "use": "sig", - "x5c": null -} \ No newline at end of file diff --git a/kop/java/src/main/resources/hydra/keys/public_key.json b/kop/java/src/main/resources/hydra/keys/public_key.json deleted file mode 100644 index 4dafb50..0000000 --- a/kop/java/src/main/resources/hydra/keys/public_key.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "alg": "RS256", - "e": "AQAB", - "kid": "public:7e9cbc3a-20f4-44aa-831c-75471946a7dc", - "kty": "RSA", - "n": "4g8rgGslfLNGdfh94KbfsMPjgX17nnEHnCLhrlVyA-jxSThiQyQVQCkZfav9k4cLCiKdoqxKtLV0RA3hWXGHE0qUNUJWVN3vz3NOI7ccEHBJHzbDk24NYxsW7M6zNfBfTc6ZrJr5XENy7emscODn8HJ2Qf1UkMUeze5EirJ2lsB9Zzo1GIw9ZU65W9HWWcgS5sL9eHlDRbVLmgph7jRzkQJGm2hOeyiE-ufUOWkBQH49BhKaNGfjZ8BOJ1WRsbIIVtwhS7m-HSIKmglboG-onNd5LYAmngbkCuhwjJajBQayxkeBeumvRQACC1-mKC5KaW40JmVRKFFHDcf892t6GX6c7PaVWPqvf2l6nYRbYT9nl4fQK1aUTiCqrPf2-WjEH1JIEwTfFZKTwpTtlr3ejGJMT7wH2L4uFbpguKawTo4lYHWN3IsryDfUVvNbb7l8KMqiuDIy-5R6WezajsCYI_GzvLGCYO1EnRTDFdEmipfbNT2_D91OPKNGmZLVUkVVlL0z-1iQtwfRamn2oRNHzMYMAplGikxrQld_IPUIbKjgtLWPDnfskoWvuCIDQdRzMpxAXa3O_cq5uQRpu2o8xZ8RYWixxrIGc1_8m-QQLy7DwcmVd0dGU29S-fnfOzWr43KWlyWfGsBLFxUkltjY6gx6oB6tsQVC3Cy5Eku8Fdc", - "use": "sig", - "x5c": null -} \ No newline at end of file diff --git a/kop/java/src/main/resources/init_hydra_oauth_server.sh b/kop/java/src/main/resources/init_hydra_oauth_server.sh deleted file mode 100755 index c0550b6..0000000 --- a/kop/java/src/main/resources/init_hydra_oauth_server.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -SRC_DIR=$(git rev-parse --show-toplevel) -cd "$SRC_DIR" - -wait_for_url() { - URL=$1 - MSG=$2 - - if [[ $URL == https* ]]; then - CMD="curl -k -sL -o /dev/null -w %{http_code} $URL" - else - CMD="curl -sL -o /dev/null -w %{http_code} $URL" - fi - - until [ "200" == "$($CMD)" ] - do - echo "$MSG ($URL)" - sleep 2 - done -} - -# Wait until the hydra server started -wait_for_url "http://localhost:4445/clients" "Waiting for Hydra admin REST to start" - -# Delete hydra default access token -docker run --rm \ - --network hydra_default \ - oryd/hydra:v1.11.7 \ - --endpoint=http://hydra:4445 \ - keys delete hydra.jwt.access-token - -# Replace hydra access token -docker run --rm \ - --network hydra_default \ - -v "$(pwd)"/kop/java/src/main/resources/hydra/keys:/tmp/keys \ - oryd/hydra:v1.11.7 \ - --endpoint=http://hydra:4445 \ - keys import hydra.jwt.access-token /tmp/keys/private_key.json /tmp/keys/public_key.json - -# Create a new client, the client id is simple_client_id and secret is simple_client_secret -# Correspond pulsar role is simple_client_id -docker run --rm \ - --network hydra_default \ - oryd/hydra:v1.11.7 \ - clients create \ - --endpoint http://hydra:4445 \ - --id simple_client_id \ - --secret simple_client_secret \ - --grant-types client_credentials \ - --response-types token,code \ - --token-endpoint-auth-method client_secret_post \ - --audience http://example.com/api/v2/ - -# Try to generate a token using the new client -docker run --rm \ - --network hydra_default \ - oryd/hydra:v1.11.7 \ - token client \ - --client-id simple_client_id \ - --client-secret simple_client_secret \ - --endpoint http://hydra:4444 - diff --git a/kop/java/src/main/resources/hydra/docker-compose.yml b/kop/java/src/main/resources/kop-handler-oauth2.properties similarity index 50% rename from kop/java/src/main/resources/hydra/docker-compose.yml rename to kop/java/src/main/resources/kop-handler-oauth2.properties index 6241f18..d110e59 100644 --- a/kop/java/src/main/resources/hydra/docker-compose.yml +++ b/kop/java/src/main/resources/kop-handler-oauth2.properties @@ -12,24 +12,4 @@ # limitations under the License. # -version: '3' - -services: - - hydra: - image: oryd/hydra:v1.11.7 - container_name: hydra - ports: - - 4444:4444 - - 4445:4445 - - 9020:9020 - command: serve all --dangerous-force-http - environment: - - DSN=memory - - URLS_SELF_ISSUER=http://127.0.0.1:4444/ - - URLS_CONSENT=http://127.0.0.1:9020/consent - - URLS_LOGIN=http://127.0.0.1:9020/login - - SERVE_PUBLIC_PORT=4444 - - SERVE_ADMIN_PORT=4445 - - STRATEGIES_ACCESS_TOKEN=jwt - - OIDC_SUBJECT_IDENTIFIERS_SUPPORTED_TYPES=public +oauth.validate.method=token diff --git a/kop/java/src/main/resources/oauth.properties b/kop/java/src/main/resources/oauth.properties index d9a21a3..c4aa39f 100644 --- a/kop/java/src/main/resources/oauth.properties +++ b/kop/java/src/main/resources/oauth.properties @@ -15,6 +15,6 @@ bootstrap.servers=localhost:9092 topic=persistent://public/default/my-topic group=my-group -issuerUrl=http://localhost:4444 +issuerUrl=https://dev-kt-aa9ne.us.auth0.com credentialsUrl=/path/to/credentials.json -audience=http://example.com/api/v2/ \ No newline at end of file +audience=https://dev-kt-aa9ne.us.auth0.com/api/v2/ \ No newline at end of file diff --git a/kop/java/src/main/resources/oauth_credentials/credentials.json b/kop/java/src/main/resources/oauth_credentials/credentials.json deleted file mode 100644 index 2847694..0000000 --- a/kop/java/src/main/resources/oauth_credentials/credentials.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "client_id":"test_role", - "client_secret":"test_secret" -} \ No newline at end of file diff --git a/kop/java/src/main/resources/oauth_credentials/simple_credentials_file.json b/kop/java/src/main/resources/oauth_credentials/simple_credentials_file.json deleted file mode 100644 index 9979090..0000000 --- a/kop/java/src/main/resources/oauth_credentials/simple_credentials_file.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "client_id":"simple_client_id", - "client_secret":"simple_client_secret" -} \ No newline at end of file From a9d84176b34f48071d643b9b7acde00b326ae4a9 Mon Sep 17 00:00:00 2001 From: Demogorgon314 Date: Thu, 21 Apr 2022 13:38:04 +0800 Subject: [PATCH 7/8] Use an online OAuth server as in the example --- kop/java/README.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/kop/java/README.md b/kop/java/README.md index 855d215..18ba2ab 100644 --- a/kop/java/README.md +++ b/kop/java/README.md @@ -70,7 +70,9 @@ See [KoP Security](https://github.com/streamnative/kop/blob/master/docs/security ## Example: OAuth2 authentication See [KoP Security](https://github.com/streamnative/kop/blob/master/docs/security.md#oauthbearer) for how to configure KoP with OAuth authentication. This example takes a topic named `my-topic` under `public/default` namespace as reference. + 1. Configure the pulsar broker, this example will use the follow values: + > **Note** > > Need to change the `credentials.json` and `kop-handler-oauth2.properties` paths to your local path. The example file can be found in `src/main/resources/`. @@ -119,11 +121,7 @@ See [KoP Security](https://github.com/streamnative/kop/blob/master/docs/security } ``` -> **NOTE** -> -> The `conf/client.conf` should be configured. For details, see [Configure CLI Tools](http://pulsar.apache.org/docs/en/security-jwt/#cli-tools). - -2. Configure the oauth in [oauth.properties](src/main/resources/oauth.properties). +3.Configure the oauth in [oauth.properties](src/main/resources/oauth.properties). ```properties bootstrap.servers=localhost:9092 @@ -134,13 +132,13 @@ See [KoP Security](https://github.com/streamnative/kop/blob/master/docs/security audience=https://dev-kt-aa9ne.us.auth0.com/api/v2/ ``` -3. Compile the project. +4.Compile the project. ``` mvn clean compile ``` -4. Run a Kafka producer to produce a `hello` message. +5.Run a Kafka producer to produce a `hello` message. ```bash mvn exec:java -Dexec.mainClass=io.streamnative.examples.kafka.OAuthProducer @@ -152,7 +150,7 @@ See [KoP Security](https://github.com/streamnative/kop/blob/master/docs/security Send hello to persistent://public/default/my-topic-0@0 ``` -5. Run a Kafka consumer to consume some messages. +6. Run a Kafka consumer to consume some messages. ```bash mvn exec:java -Dexec.mainClass=io.streamnative.examples.kafka.OAuthConsumer From bc725594dc4c4358d3fdf4ba266b3faf7eac227d Mon Sep 17 00:00:00 2001 From: Demogorgon314 Date: Thu, 21 Apr 2022 13:40:43 +0800 Subject: [PATCH 8/8] Fix format --- kop/java/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/kop/java/README.md b/kop/java/README.md index 18ba2ab..b968996 100644 --- a/kop/java/README.md +++ b/kop/java/README.md @@ -73,10 +73,10 @@ See [KoP Security](https://github.com/streamnative/kop/blob/master/docs/security 1. Configure the pulsar broker, this example will use the follow values: -> **Note** -> -> Need to change the `credentials.json` and `kop-handler-oauth2.properties` paths to your local path. The example file can be found in `src/main/resources/`. -> + > **Note** + > + > Need to change the `credentials.json` and `kop-handler-oauth2.properties` paths to your local path. The example file can be found in `src/main/resources/`. + > ```properties # Enable KoP messagingProtocols=kafka @@ -121,7 +121,7 @@ See [KoP Security](https://github.com/streamnative/kop/blob/master/docs/security } ``` -3.Configure the oauth in [oauth.properties](src/main/resources/oauth.properties). +3. Configure the oauth in [oauth.properties](src/main/resources/oauth.properties). ```properties bootstrap.servers=localhost:9092 @@ -132,13 +132,13 @@ See [KoP Security](https://github.com/streamnative/kop/blob/master/docs/security audience=https://dev-kt-aa9ne.us.auth0.com/api/v2/ ``` -4.Compile the project. +4. Compile the project. ``` mvn clean compile ``` -5.Run a Kafka producer to produce a `hello` message. +5. Run a Kafka producer to produce a `hello` message. ```bash mvn exec:java -Dexec.mainClass=io.streamnative.examples.kafka.OAuthProducer