From d3640aee431d71a9a9520c67a2fb67ebaa0e83e8 Mon Sep 17 00:00:00 2001 From: saketa Date: Mon, 26 Aug 2024 23:57:17 -0700 Subject: [PATCH 1/9] HDDS-11371. Added xcompat tests to check for regressions in secure cluster --- .../dist/src/main/compose/xcompat/.env | 2 + .../dist/src/main/compose/xcompat/krb5.conf | 41 +++++++ .../main/compose/xcompat/secure-clients.yaml | 73 +++++++++++ .../main/compose/xcompat/secure-docker-config | 114 ++++++++++++++++++ .../compose/xcompat/secure-new-cluster.yaml | 96 +++++++++++++++ .../compose/xcompat/secure-old-cluster.yaml | 96 +++++++++++++++ .../src/main/compose/xcompat/test-secure.sh | 83 +++++++++++++ 7 files changed, 505 insertions(+) create mode 100644 hadoop-ozone/dist/src/main/compose/xcompat/krb5.conf create mode 100644 hadoop-ozone/dist/src/main/compose/xcompat/secure-clients.yaml create mode 100644 hadoop-ozone/dist/src/main/compose/xcompat/secure-docker-config create mode 100644 hadoop-ozone/dist/src/main/compose/xcompat/secure-new-cluster.yaml create mode 100644 hadoop-ozone/dist/src/main/compose/xcompat/secure-old-cluster.yaml create mode 100755 hadoop-ozone/dist/src/main/compose/xcompat/test-secure.sh diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/.env b/hadoop-ozone/dist/src/main/compose/xcompat/.env index 140975d4bd0e..a673b7f46550 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/.env +++ b/hadoop-ozone/dist/src/main/compose/xcompat/.env @@ -17,3 +17,5 @@ HDDS_VERSION=${hdds.version} OZONE_RUNNER_VERSION=${docker.ozone-runner.version} OZONE_RUNNER_IMAGE=apache/ozone-runner +HADOOP_VERSION=${hadoop.version} +OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image} diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/krb5.conf b/hadoop-ozone/dist/src/main/compose/xcompat/krb5.conf new file mode 100644 index 000000000000..eefc5b9c6858 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/xcompat/krb5.conf @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[logging] +default = FILE:/var/log/krb5libs.log +kdc = FILE:/var/log/krb5kdc.log +admin_server = FILE:/var/log/kadmind.log + +[libdefaults] + dns_canonicalize_hostname = false + dns_lookup_realm = false + ticket_lifetime = 24h + renew_lifetime = 7d + forwardable = true + rdns = false + default_realm = EXAMPLE.COM + +[realms] + EXAMPLE.COM = { + kdc = kdc + admin_server = kdc + max_renewable_life = 7d + } + +[domain_realm] + .example.com = EXAMPLE.COM + example.com = EXAMPLE.COM + diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/secure-clients.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/secure-clients.yaml new file mode 100644 index 000000000000..be8592a986cd --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/xcompat/secure-clients.yaml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + old_client_1_0_0: + image: apache/ozone:1.0.0 + env_file: + - secure-docker-config + volumes: + - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf + command: ["sleep","1000000"] + old_client_1_1_0: + image: apache/ozone:1.1.0 + env_file: + - secure-docker-config + volumes: + - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf + command: ["sleep","1000000"] + old_client_1_2_1: + image: apache/ozone:1.2.1 + env_file: + - secure-docker-config + volumes: + - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf + command: ["sleep","1000000"] + old_client_1_3_0: + image: apache/ozone:1.3.0 + env_file: + - secure-docker-config + volumes: + - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf + command: ["sleep","1000000"] + old_client_1_4_0: + image: apache/ozone:1.4.0 + env_file: + - secure-docker-config + volumes: + - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf + command: ["sleep","1000000"] + new_client: + image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} + env_file: + - secure-docker-config + volumes: + - ../..:/opt/hadoop + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf + environment: + OZONE_OPTS: + command: ["sleep","1000000"] diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/secure-docker-config b/hadoop-ozone/dist/src/main/compose/xcompat/secure-docker-config new file mode 100644 index 000000000000..1804d3c8f4ba --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/xcompat/secure-docker-config @@ -0,0 +1,114 @@ +CORE-SITE.XML_fs.defaultFS=ofs://om +CORE-SITE.XML_fs.trash.interval=1 +CORE-SITE.XML_fs.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzoneFileSystem + +OZONE-SITE.XML_hdds.datanode.dir=/data/hdds +OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB +OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 +OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata +OZONE-SITE.XML_ozone.om.address=om +OZONE-SITE.XML_ozone.om.http-address=om:9874 +OZONE-SITE.XML_ozone.scm.http-address=scm:9876 +OZONE-SITE.XML_ozone.recon.address=recon:9891 +OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon +OZONE-SITE.XML_ozone.server.default.replication=3 +OZONE-SITE.XML_ozone.scm.block.client.address=scm +OZONE-SITE.XML_ozone.scm.client.address=scm +OZONE-SITE.XML_ozone.scm.container.size=1GB +OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.names=scm +OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 +OZONE-SITE.XML_recon.om.snapshot.task.interval.delay=1m +OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s +OZONE-SITE.XML_ozone.default.bucket.layout=LEGACY +OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http + +OZONE-SITE.XML_hdds.block.token.enabled=true +OZONE-SITE.XML_hdds.container.token.enabled=true +OZONE-SITE.XML_hdds.grpc.tls.enabled=true + +OZONE-SITE.XML_ozone.security.enabled=true +OZONE-SITE.XML_ozone.acl.enabled=true +OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer +OZONE-SITE.XML_ozone.administrators="testuser,recon,om" +OZONE-SITE.XML_ozone.s3.administrators="testuser,recon,om" +OZONE-SITE.XML_ozone.recon.administrators="testuser2" +OZONE-SITE.XML_ozone.s3.administrators="testuser,s3g" + +HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019 +HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012 +CORE-SITE.XML_dfs.data.transfer.protection=authentication +CORE-SITE.XML_hadoop.security.authentication=kerberos +CORE-SITE.XML_hadoop.security.auth_to_local="DEFAULT" +CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms + +OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM +OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab +OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM +OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab +OZONE-SITE.XML_ozone.recon.kerberos.keytab.file=/etc/security/keytabs/recon.keytab +OZONE-SITE.XML_ozone.recon.kerberos.principal=recon/recon@EXAMPLE.COM + +OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab +OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/s3g@EXAMPLE.COM + +OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file=/etc/security/keytabs/httpfs.keytab +OZONE-SITE.XML_ozone.httpfs.kerberos.principal=httpfs/httpfs@EXAMPLE.COM + +HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/dn@EXAMPLE.COM +HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab +HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM +HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab + +OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true +OZONE-SITE.XML_ozone.s3g.secret.http.enabled=true +OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.AuthenticationFilterInitializer + +OZONE-SITE.XML_ozone.om.http.auth.type=kerberos +OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos +OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos +OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos +OZONE-SITE.XML_ozone.s3g.secret.http.auth.type=kerberos +OZONE-SITE.XML_ozone.httpfs.http.auth.type=kerberos +OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos + +OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM +OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/scm.keytab +OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/om@EXAMPLE.COM +OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/om.keytab +OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/dn@EXAMPLE.COM +OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/dn.keytab +OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/s3g.keytab +OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM +OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.keytab=/etc/security/keytabs/httpfs.keytab +OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.principal=HTTP/httpfs@EXAMPLE.COM +OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal=* +OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab + +CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed=false +CORE-SITE.XML_hadoop.http.authentication.signature.secret.file=/etc/security/http_secret +CORE-SITE.XML_hadoop.http.authentication.type=kerberos +CORE-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM +CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab + +CORE-SITE.XML_hadoop.security.authorization=true +HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=* +HADOOP-POLICY.XML_ozone.security.reconfigure.protocol.acl=* + +KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users=* +KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups=* +KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts=* + +OZONE_DATANODE_SECURE_USER=root +JSVC_HOME=/usr/bin + +OZONE_CONF_DIR=/etc/hadoop +OZONE_LOG_DIR=/var/log/hadoop + +no_proxy=om,scm,recon,s3g,kdc,localhost,127.0.0.1 + diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/secure-new-cluster.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/secure-new-cluster.yaml new file mode 100644 index 000000000000..3783d9a2c69d --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/xcompat/secure-new-cluster.yaml @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# reusable fragments (see https://docs.docker.com/compose/compose-file/#extension-fields) +x-new-config: + &new-config + image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} + dns_search: . + env_file: + - secure-docker-config + volumes: + - ../..:/opt/hadoop + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf + +services: + kdc: + image: ${OZONE_TESTKRB5_IMAGE} + hostname: kdc + dns_search: . + volumes: + - ../..:/opt/hadoop + - ../_keytabs:/etc/security/keytabs + command: [ "krb5kdc","-n" ] + kms: + image: apache/hadoop:${HADOOP_VERSION} + hostname: kms + dns_search: . + ports: + - 9600:9600 + env_file: + - ./secure-docker-config + environment: + HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop + volumes: + - ../../libexec/transformation.py:/opt/transformation.py + command: [ "hadoop", "kms" ] + datanode: + <<: *new-config + hostname: dn + ports: + - 19864 + - 9882 + environment: + OZONE_OPTS: + command: ["ozone","datanode"] + om: + <<: *new-config + hostname: om + environment: + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + OZONE_OPTS: -Dcom.sun.net.ssl.checkRevocation=false + ports: + - 9874:9874 + - 9862:9862 + command: ["ozone","om"] + recon: + <<: *new-config + hostname: recon + ports: + - 9888:9888 + environment: + OZONE_OPTS: + command: ["ozone","recon"] + s3g: + <<: *new-config + hostname: s3g + environment: + OZONE_OPTS: + ports: + - 9878:9878 + command: ["ozone","s3g"] + scm: + <<: *new-config + hostname: scm + ports: + - 9876:9876 + - 9860:9860 + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" + OZONE_OPTS: + command: ["ozone","scm"] diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/secure-old-cluster.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/secure-old-cluster.yaml new file mode 100644 index 000000000000..eca9e57cd7f7 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/xcompat/secure-old-cluster.yaml @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# reusable fragments (see https://docs.docker.com/compose/compose-file/#extension-fields) +x-old-config: + &old-config + image: apache/ozone:${OZONE_VERSION} + dns_search: . + env_file: + - secure-docker-config + volumes: + - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf + +services: + kdc: + image: ${OZONE_TESTKRB5_IMAGE} + hostname: kdc + dns_search: . + volumes: + - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + command: [ "krb5kdc","-n" ] + kms: + image: apache/hadoop:${HADOOP_VERSION} + hostname: kms + dns_search: . + ports: + - 9600:9600 + env_file: + - ./secure-docker-config + environment: + HADOOP_CONF_DIR: /opt/ozone/etc/hadoop + volumes: + - ../../libexec/transformation.py:/opt/transformation.py + command: [ "hadoop", "kms" ] + datanode: + <<: *old-config + hostname: dn + ports: + - 19864 + - 9882 + environment: + HADOOP_OPTS: + command: ["ozone","datanode"] + om: + <<: *old-config + hostname: om + environment: + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + HADOOP_OPTS: + ports: + - 9874:9874 + - 9862:9862 + command: ["ozone","om"] + recon: + <<: *old-config + hostname: recon + ports: + - 9888:9888 + environment: + HADOOP_OPTS: + command: ["ozone","recon"] + s3g: + <<: *old-config + hostname: s3g + environment: + HADOOP_OPTS: + ports: + - 9878:9878 + command: ["ozone","s3g"] + scm: + <<: *old-config + hostname: scm + ports: + - 9876:9876 + - 9860:9860 + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" + HADOOP_OPTS: + command: ["ozone","scm"] diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/test-secure.sh b/hadoop-ozone/dist/src/main/compose/xcompat/test-secure.sh new file mode 100755 index 000000000000..49e1e1500c86 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/xcompat/test-secure.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#suite:compat + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR +basename=$(basename ${COMPOSE_DIR}) + +current_version="${ozone.version}" +old_versions="1.0.0 1.1.0 1.2.1 1.3.0 1.4.0" + +# shellcheck source=hadoop-ozone/dist/src/main/compose/testlib.sh +source "${COMPOSE_DIR}/../testlib.sh" + +export SECURITY_ENABLED=true +: ${OZONE_BUCKET_KEY_NAME:=key1} + +old_client() { + OZONE_DIR=/opt/ozone + container=${client} + "$@" +} + +new_client() { + OZONE_DIR=/opt/hadoop + container=new_client + client_version=${current_version} + "$@" +} + + +test_bucket_encryption() { + + execute_command_in_container ${container} kinit -k -t /etc/security/keytabs/testuser.keytab testuser/scm@EXAMPLE.COM + + execute_robot_test ${container} -N "xcompat-cluster-${cluster_version}-client-${client_version}" -v SUFFIX:${client_version} security/bucket-encryption.robot +} + +test_encryption_cross_compatibility() { + + echo "Running client connection to multiple KMS backward compatibility tests." + + echo "Starting secure cluster with COMPOSE_FILE=${COMPOSE_FILE}" + OZONE_KEEP_RESULTS=true start_docker_env + + execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME} + + new_client test_bucket_encryption + + for client_version in "$@"; do + client="old_client_${client_version//./_}" + old_client test_bucket_encryption + done + + KEEP_RUNNING=false stop_docker_env +} + +create_results_dir + +encryption_old_versions="1.2.1 1.3.0 1.4.0" +# current cluster with various clients +COMPOSE_FILE=secure-new-cluster.yaml:secure-clients.yaml cluster_version=${current_version} test_encryption_cross_compatibility ${encryption_old_versions} + +# old cluster with clients: same version and current version +for cluster_version in ${encryption_old_versions}; do + export OZONE_VERSION=${cluster_version} + COMPOSE_FILE=secure-old-cluster.yaml:secure-clients.yaml test_encryption_cross_compatibility ${cluster_version} +done \ No newline at end of file From d2b6c369d2651b10e90239391efbbf0f90cfb472 Mon Sep 17 00:00:00 2001 From: saketa Date: Tue, 27 Aug 2024 00:20:42 -0700 Subject: [PATCH 2/9] HDDS-11371. Added license for xcompat/secure-docker-config --- .../main/compose/xcompat/secure-docker-config | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/secure-docker-config b/hadoop-ozone/dist/src/main/compose/xcompat/secure-docker-config index 1804d3c8f4ba..7e103b5f96ea 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/secure-docker-config +++ b/hadoop-ozone/dist/src/main/compose/xcompat/secure-docker-config @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + CORE-SITE.XML_fs.defaultFS=ofs://om CORE-SITE.XML_fs.trash.interval=1 CORE-SITE.XML_fs.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzoneFileSystem From fabcae667bdfeafb10512da3de04eeb2333d01f7 Mon Sep 17 00:00:00 2001 From: saketa Date: Tue, 27 Aug 2024 02:27:25 -0700 Subject: [PATCH 3/9] HDDS-11371. Fixed secure old cluster setup in xcompat tests. --- .../dist/src/main/compose/xcompat/secure-docker-config | 3 +++ .../dist/src/main/compose/xcompat/secure-old-cluster.yaml | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/secure-docker-config b/hadoop-ozone/dist/src/main/compose/xcompat/secure-docker-config index 7e103b5f96ea..a461c92aa608 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/secure-docker-config +++ b/hadoop-ozone/dist/src/main/compose/xcompat/secure-docker-config @@ -34,7 +34,9 @@ OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.names=scm +OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 +OZONE-SITE.XML_ozone.datanode.pipeline.limit=1 OZONE-SITE.XML_recon.om.snapshot.task.interval.delay=1m OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s OZONE-SITE.XML_ozone.default.bucket.layout=LEGACY @@ -73,6 +75,7 @@ OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file=/etc/security/keytabs/httpfs.ke OZONE-SITE.XML_ozone.httpfs.kerberos.principal=httpfs/httpfs@EXAMPLE.COM HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/dn@EXAMPLE.COM +HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/secure-old-cluster.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/secure-old-cluster.yaml index eca9e57cd7f7..3237fa44deb2 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/secure-old-cluster.yaml +++ b/hadoop-ozone/dist/src/main/compose/xcompat/secure-old-cluster.yaml @@ -44,7 +44,7 @@ services: env_file: - ./secure-docker-config environment: - HADOOP_CONF_DIR: /opt/ozone/etc/hadoop + HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop volumes: - ../../libexec/transformation.py:/opt/transformation.py command: [ "hadoop", "kms" ] @@ -88,7 +88,6 @@ services: hostname: scm ports: - 9876:9876 - - 9860:9860 environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" From 8181a171424279ceba312f960a0796b680ea76eb Mon Sep 17 00:00:00 2001 From: saketa Date: Tue, 27 Aug 2024 03:19:36 -0700 Subject: [PATCH 4/9] HDDS-11371. Handled cases where OM does not have getServerDeaults() implemented. --- .../hadoop/ozone/client/rpc/RpcClient.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 15babfde69db..c823638ecadd 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -2581,19 +2581,28 @@ public KeyProvider call() throws Exception { @Override public OzoneFsServerDefaults getServerDefaults() throws IOException { long now = Time.monotonicNow(); - if ((serverDefaults == null) || - (now - serverDefaultsLastUpdate > serverDefaultsValidityPeriod)) { - serverDefaults = ozoneManagerClient.getServerDefaults(); - serverDefaultsLastUpdate = now; + try { + if ((serverDefaults == null) || + (now - serverDefaultsLastUpdate > serverDefaultsValidityPeriod)) { + serverDefaults = ozoneManagerClient.getServerDefaults(); + serverDefaultsLastUpdate = now; + } + assert serverDefaults != null; + return serverDefaults; + } catch (Exception e) { + LOG.error("Could not get server defaults from OM", e); + return null; } - assert serverDefaults != null; - return serverDefaults; } @Override public URI getKeyProviderUri() throws IOException { - return OzoneKMSUtil.getKeyProviderUri(ugi, - null, getServerDefaults().getKeyProviderUri(), conf); + OzoneFsServerDefaults omServerDefaults = getServerDefaults(); + if (omServerDefaults != null) { + return OzoneKMSUtil.getKeyProviderUri(ugi, + null, omServerDefaults.getKeyProviderUri(), conf); + } + return OzoneKMSUtil.getKeyProviderUri(ugi, null, null, conf); } @Override From 85daf08139a39f8834f77657025c66ef81eb0904 Mon Sep 17 00:00:00 2001 From: saketa Date: Tue, 27 Aug 2024 13:32:43 -0700 Subject: [PATCH 5/9] HDDS-11371. Fixed acceptance test errors. Added Om version for getServerDefault() API. --- .../hadoop/ozone/OzoneManagerVersion.java | 3 ++ .../hadoop/ozone/client/rpc/RpcClient.java | 28 +++++++++---------- .../compose/xcompat/secure-old-cluster.yaml | 1 + 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java index eec2ceeb5e8d..4a3ddd1ee118 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java @@ -45,6 +45,9 @@ public enum OzoneManagerVersion implements ComponentVersion { ATOMIC_REWRITE_KEY(6, "OzoneManager version that supports rewriting key as atomic operation"), HBASE_SUPPORT(7, "OzoneManager version that supports HBase integration"), + SERVER_DEFAULTS(8, "OzoneManager version that supports get server" + + " defaults API."), + FUTURE_VERSION(-1, "Used internally in the client when the server side is " + " newer and an unknown server version has arrived to the client."); diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index c823638ecadd..d3a325b53918 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -2581,26 +2581,24 @@ public KeyProvider call() throws Exception { @Override public OzoneFsServerDefaults getServerDefaults() throws IOException { long now = Time.monotonicNow(); - try { - if ((serverDefaults == null) || - (now - serverDefaultsLastUpdate > serverDefaultsValidityPeriod)) { - serverDefaults = ozoneManagerClient.getServerDefaults(); - serverDefaultsLastUpdate = now; - } - assert serverDefaults != null; - return serverDefaults; - } catch (Exception e) { - LOG.error("Could not get server defaults from OM", e); - return null; + if ((serverDefaults == null) || + (now - serverDefaultsLastUpdate > serverDefaultsValidityPeriod)) { + serverDefaults = ozoneManagerClient.getServerDefaults(); + serverDefaultsLastUpdate = now; } + assert serverDefaults != null; + return serverDefaults; } @Override public URI getKeyProviderUri() throws IOException { - OzoneFsServerDefaults omServerDefaults = getServerDefaults(); - if (omServerDefaults != null) { - return OzoneKMSUtil.getKeyProviderUri(ugi, - null, omServerDefaults.getKeyProviderUri(), conf); + if (omVersion.compareTo(OzoneManagerVersion.SERVER_DEFAULTS) >= 0) { + try { + return OzoneKMSUtil.getKeyProviderUri(ugi, + null, getServerDefaults().getKeyProviderUri(), conf); + } catch (Exception e) { + LOG.warn("Could not get key provider URI from OM.", e); + } } return OzoneKMSUtil.getKeyProviderUri(ugi, null, null, conf); } diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/secure-old-cluster.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/secure-old-cluster.yaml index 3237fa44deb2..cf3bded3bda4 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/secure-old-cluster.yaml +++ b/hadoop-ozone/dist/src/main/compose/xcompat/secure-old-cluster.yaml @@ -62,6 +62,7 @@ services: hostname: om environment: ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + OZONE_OPTS: -Dcom.sun.net.ssl.checkRevocation=false HADOOP_OPTS: ports: - 9874:9874 From c611fc3a35857ddf9d58b8be3232150f3b426af6 Mon Sep 17 00:00:00 2001 From: saketa Date: Tue, 27 Aug 2024 19:50:42 -0700 Subject: [PATCH 6/9] HDDS-11371. Added newline to test-secure.sh --- hadoop-ozone/dist/src/main/compose/xcompat/test-secure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/test-secure.sh b/hadoop-ozone/dist/src/main/compose/xcompat/test-secure.sh index 49e1e1500c86..0010c07f66b6 100755 --- a/hadoop-ozone/dist/src/main/compose/xcompat/test-secure.sh +++ b/hadoop-ozone/dist/src/main/compose/xcompat/test-secure.sh @@ -80,4 +80,4 @@ COMPOSE_FILE=secure-new-cluster.yaml:secure-clients.yaml cluster_version=${curre for cluster_version in ${encryption_old_versions}; do export OZONE_VERSION=${cluster_version} COMPOSE_FILE=secure-old-cluster.yaml:secure-clients.yaml test_encryption_cross_compatibility ${cluster_version} -done \ No newline at end of file +done From eb9c9f4769800b8135f4f090c274a073caaf1715 Mon Sep 17 00:00:00 2001 From: saketa Date: Thu, 5 Sep 2024 21:22:04 -0700 Subject: [PATCH 7/9] HDDS-11371. Added error handling in getServerDefaults() and fixed acceptance tests. --- .../hadoop/ozone/client/rpc/RpcClient.java | 24 ++-- .../src/main/compose/xcompat/clients.yaml | 12 ++ .../src/main/compose/xcompat/docker-config | 94 ++++++++++++- .../src/main/compose/xcompat/new-cluster.yaml | 33 ++++- .../src/main/compose/xcompat/old-cluster.yaml | 31 ++++ .../main/compose/xcompat/secure-clients.yaml | 73 ---------- .../main/compose/xcompat/secure-docker-config | 133 ------------------ .../compose/xcompat/secure-new-cluster.yaml | 96 ------------- .../compose/xcompat/secure-old-cluster.yaml | 96 ------------- .../src/main/compose/xcompat/test-secure.sh | 83 ----------- .../dist/src/main/compose/xcompat/test.sh | 29 +++- 11 files changed, 207 insertions(+), 497 deletions(-) delete mode 100644 hadoop-ozone/dist/src/main/compose/xcompat/secure-clients.yaml delete mode 100644 hadoop-ozone/dist/src/main/compose/xcompat/secure-docker-config delete mode 100644 hadoop-ozone/dist/src/main/compose/xcompat/secure-new-cluster.yaml delete mode 100644 hadoop-ozone/dist/src/main/compose/xcompat/secure-old-cluster.yaml delete mode 100755 hadoop-ozone/dist/src/main/compose/xcompat/test-secure.sh diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index d3a325b53918..5a5937b01f7e 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -2580,27 +2580,27 @@ public KeyProvider call() throws Exception { @Override public OzoneFsServerDefaults getServerDefaults() throws IOException { + if (omVersion.compareTo(OzoneManagerVersion.SERVER_DEFAULTS) < 0) { + return null; + } long now = Time.monotonicNow(); if ((serverDefaults == null) || (now - serverDefaultsLastUpdate > serverDefaultsValidityPeriod)) { - serverDefaults = ozoneManagerClient.getServerDefaults(); - serverDefaultsLastUpdate = now; + try { + serverDefaults = ozoneManagerClient.getServerDefaults(); + serverDefaultsLastUpdate = now; + } catch (Exception e) { + LOG.warn("Could not get server defaults from OM.", e); + } } - assert serverDefaults != null; return serverDefaults; } @Override public URI getKeyProviderUri() throws IOException { - if (omVersion.compareTo(OzoneManagerVersion.SERVER_DEFAULTS) >= 0) { - try { - return OzoneKMSUtil.getKeyProviderUri(ugi, - null, getServerDefaults().getKeyProviderUri(), conf); - } catch (Exception e) { - LOG.warn("Could not get key provider URI from OM.", e); - } - } - return OzoneKMSUtil.getKeyProviderUri(ugi, null, null, conf); + String keyProviderUri = (getServerDefaults() != null) ? + serverDefaults.getKeyProviderUri() : null; + return OzoneKMSUtil.getKeyProviderUri(ugi, null, keyProviderUri, conf); } @Override diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml index 0bf0f619bd7c..bfb346f0747b 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml +++ b/hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml @@ -21,6 +21,8 @@ services: - docker-config volumes: - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf command: ["sleep","1000000"] old_client_1_1_0: image: apache/ozone:1.1.0 @@ -28,6 +30,8 @@ services: - docker-config volumes: - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf command: ["sleep","1000000"] old_client_1_2_1: image: apache/ozone:1.2.1 @@ -35,6 +39,8 @@ services: - docker-config volumes: - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf command: ["sleep","1000000"] old_client_1_3_0: image: apache/ozone:1.3.0 @@ -42,6 +48,8 @@ services: - docker-config volumes: - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf command: ["sleep","1000000"] old_client_1_4_0: image: apache/ozone:1.4.0 @@ -49,6 +57,8 @@ services: - docker-config volumes: - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf command: ["sleep","1000000"] new_client: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} @@ -56,6 +66,8 @@ services: - docker-config volumes: - ../..:/opt/hadoop + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf environment: OZONE_OPTS: command: ["sleep","1000000"] diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/docker-config b/hadoop-ozone/dist/src/main/compose/xcompat/docker-config index 85099f902d39..1a61aaf4f7e9 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/docker-config +++ b/hadoop-ozone/dist/src/main/compose/xcompat/docker-config @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +CORE-SITE.XML_fs.defaultFS=ofs://om +CORE-SITE.XML_fs.trash.interval=1 CORE-SITE.XML_fs.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzoneFileSystem OZONE-SITE.XML_hdds.datanode.dir=/data/hdds @@ -22,6 +24,7 @@ OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 +OZONE-SITE.XML_ozone.scm.http-address=scm:9876 OZONE-SITE.XML_ozone.recon.address=recon:9891 OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon OZONE-SITE.XML_ozone.server.default.replication=3 @@ -31,9 +34,98 @@ OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.names=scm +OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 +OZONE-SITE.XML_ozone.datanode.pipeline.limit=1 OZONE-SITE.XML_recon.om.snapshot.task.interval.delay=1m OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s OZONE-SITE.XML_ozone.default.bucket.layout=LEGACY OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http -no_proxy=om,recon,scm,s3g,kdc,localhost,127.0.0.1 + +OZONE-SITE.XML_hdds.block.token.enabled=true +OZONE-SITE.XML_hdds.container.token.enabled=true +OZONE-SITE.XML_hdds.grpc.tls.enabled=true + +OZONE-SITE.XML_ozone.security.enabled=true +OZONE-SITE.XML_ozone.acl.enabled=true +OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer +OZONE-SITE.XML_ozone.administrators="testuser,recon,om" +OZONE-SITE.XML_ozone.s3.administrators="testuser,recon,om" +OZONE-SITE.XML_ozone.recon.administrators="testuser2" +OZONE-SITE.XML_ozone.s3.administrators="testuser,s3g" + +HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019 +HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012 +CORE-SITE.XML_dfs.data.transfer.protection=authentication +CORE-SITE.XML_hadoop.security.authentication=kerberos +CORE-SITE.XML_hadoop.security.auth_to_local="DEFAULT" +CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms + +OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM +OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab +OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM +OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab +OZONE-SITE.XML_ozone.recon.kerberos.keytab.file=/etc/security/keytabs/recon.keytab +OZONE-SITE.XML_ozone.recon.kerberos.principal=recon/recon@EXAMPLE.COM + +OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab +OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/s3g@EXAMPLE.COM + +OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file=/etc/security/keytabs/httpfs.keytab +OZONE-SITE.XML_ozone.httpfs.kerberos.principal=httpfs/httpfs@EXAMPLE.COM + +HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/dn@EXAMPLE.COM +HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab +HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab +HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM +HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab + +OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true +OZONE-SITE.XML_ozone.s3g.secret.http.enabled=true +OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.AuthenticationFilterInitializer + +OZONE-SITE.XML_ozone.om.http.auth.type=kerberos +OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos +OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos +OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos +OZONE-SITE.XML_ozone.s3g.secret.http.auth.type=kerberos +OZONE-SITE.XML_ozone.httpfs.http.auth.type=kerberos +OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos + +OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM +OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/scm.keytab +OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/om@EXAMPLE.COM +OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/om.keytab +OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/dn@EXAMPLE.COM +OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/dn.keytab +OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/s3g.keytab +OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM +OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.keytab=/etc/security/keytabs/httpfs.keytab +OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.principal=HTTP/httpfs@EXAMPLE.COM +OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal=* +OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab + +CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed=false +CORE-SITE.XML_hadoop.http.authentication.signature.secret.file=/etc/security/http_secret +CORE-SITE.XML_hadoop.http.authentication.type=kerberos +CORE-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM +CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab + +CORE-SITE.XML_hadoop.security.authorization=true +HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=* +HADOOP-POLICY.XML_ozone.security.reconfigure.protocol.acl=* + +KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users=* +KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups=* +KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts=* + +OZONE_DATANODE_SECURE_USER=root +JSVC_HOME=/usr/bin + +OZONE_LOG_DIR=/var/log/hadoop + +no_proxy=om,scm,recon,s3g,kdc,localhost,127.0.0.1 diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml index 6e3ff6cfbc9c..32059140ce91 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml +++ b/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml @@ -18,14 +18,39 @@ x-new-config: &new-config image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} + dns_search: . env_file: - docker-config volumes: - ../..:/opt/hadoop + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf services: + kdc: + image: ${OZONE_TESTKRB5_IMAGE} + hostname: kdc + dns_search: . + volumes: + - ../..:/opt/hadoop + - ../_keytabs:/etc/security/keytabs + command: [ "krb5kdc","-n" ] + kms: + image: apache/hadoop:${HADOOP_VERSION} + hostname: kms + dns_search: . + ports: + - 9600:9600 + env_file: + - ./docker-config + environment: + HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop + volumes: + - ../../libexec/transformation.py:/opt/transformation.py + command: [ "hadoop", "kms" ] datanode: <<: *new-config + hostname: dn ports: - 19864 - 9882 @@ -34,15 +59,17 @@ services: command: ["ozone","datanode"] om: <<: *new-config + hostname: om environment: ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION - OZONE_OPTS: + OZONE_OPTS: -Dcom.sun.net.ssl.checkRevocation=false ports: - 9874:9874 - 9862:9862 command: ["ozone","om"] recon: <<: *new-config + hostname: recon ports: - 9888:9888 environment: @@ -50,6 +77,7 @@ services: command: ["ozone","recon"] s3g: <<: *new-config + hostname: s3g environment: OZONE_OPTS: ports: @@ -57,9 +85,12 @@ services: command: ["ozone","s3g"] scm: <<: *new-config + hostname: scm ports: - 9876:9876 + - 9860:9860 environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" OZONE_OPTS: command: ["ozone","scm"] diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml index c603bb51df32..941cbc14df1c 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml +++ b/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml @@ -18,14 +18,39 @@ x-old-config: &old-config image: apache/ozone:${OZONE_VERSION} + dns_search: . env_file: - docker-config volumes: - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf services: + kdc: + image: ${OZONE_TESTKRB5_IMAGE} + hostname: kdc + dns_search: . + volumes: + - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + command: [ "krb5kdc","-n" ] + kms: + image: apache/hadoop:${HADOOP_VERSION} + hostname: kms + dns_search: . + ports: + - 9600:9600 + env_file: + - ./docker-config + environment: + HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop + volumes: + - ../../libexec/transformation.py:/opt/transformation.py + command: [ "hadoop", "kms" ] datanode: <<: *old-config + hostname: dn ports: - 19864 - 9882 @@ -34,8 +59,10 @@ services: command: ["ozone","datanode"] om: <<: *old-config + hostname: om environment: ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + OZONE_OPTS: -Dcom.sun.net.ssl.checkRevocation=false HADOOP_OPTS: ports: - 9874:9874 @@ -43,6 +70,7 @@ services: command: ["ozone","om"] recon: <<: *old-config + hostname: recon ports: - 9888:9888 environment: @@ -50,6 +78,7 @@ services: command: ["ozone","recon"] s3g: <<: *old-config + hostname: s3g environment: HADOOP_OPTS: ports: @@ -57,9 +86,11 @@ services: command: ["ozone","s3g"] scm: <<: *old-config + hostname: scm ports: - 9876:9876 environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" HADOOP_OPTS: command: ["ozone","scm"] diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/secure-clients.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/secure-clients.yaml deleted file mode 100644 index be8592a986cd..000000000000 --- a/hadoop-ozone/dist/src/main/compose/xcompat/secure-clients.yaml +++ /dev/null @@ -1,73 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -services: - old_client_1_0_0: - image: apache/ozone:1.0.0 - env_file: - - secure-docker-config - volumes: - - ../..:/opt/ozone - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf - command: ["sleep","1000000"] - old_client_1_1_0: - image: apache/ozone:1.1.0 - env_file: - - secure-docker-config - volumes: - - ../..:/opt/ozone - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf - command: ["sleep","1000000"] - old_client_1_2_1: - image: apache/ozone:1.2.1 - env_file: - - secure-docker-config - volumes: - - ../..:/opt/ozone - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf - command: ["sleep","1000000"] - old_client_1_3_0: - image: apache/ozone:1.3.0 - env_file: - - secure-docker-config - volumes: - - ../..:/opt/ozone - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf - command: ["sleep","1000000"] - old_client_1_4_0: - image: apache/ozone:1.4.0 - env_file: - - secure-docker-config - volumes: - - ../..:/opt/ozone - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf - command: ["sleep","1000000"] - new_client: - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} - env_file: - - secure-docker-config - volumes: - - ../..:/opt/hadoop - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf - environment: - OZONE_OPTS: - command: ["sleep","1000000"] diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/secure-docker-config b/hadoop-ozone/dist/src/main/compose/xcompat/secure-docker-config deleted file mode 100644 index a461c92aa608..000000000000 --- a/hadoop-ozone/dist/src/main/compose/xcompat/secure-docker-config +++ /dev/null @@ -1,133 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -CORE-SITE.XML_fs.defaultFS=ofs://om -CORE-SITE.XML_fs.trash.interval=1 -CORE-SITE.XML_fs.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzoneFileSystem - -OZONE-SITE.XML_hdds.datanode.dir=/data/hdds -OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB -OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 -OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata -OZONE-SITE.XML_ozone.om.address=om -OZONE-SITE.XML_ozone.om.http-address=om:9874 -OZONE-SITE.XML_ozone.scm.http-address=scm:9876 -OZONE-SITE.XML_ozone.recon.address=recon:9891 -OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon -OZONE-SITE.XML_ozone.server.default.replication=3 -OZONE-SITE.XML_ozone.scm.block.client.address=scm -OZONE-SITE.XML_ozone.scm.client.address=scm -OZONE-SITE.XML_ozone.scm.container.size=1GB -OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data -OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s -OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 -OZONE-SITE.XML_ozone.datanode.pipeline.limit=1 -OZONE-SITE.XML_recon.om.snapshot.task.interval.delay=1m -OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s -OZONE-SITE.XML_ozone.default.bucket.layout=LEGACY -OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http - -OZONE-SITE.XML_hdds.block.token.enabled=true -OZONE-SITE.XML_hdds.container.token.enabled=true -OZONE-SITE.XML_hdds.grpc.tls.enabled=true - -OZONE-SITE.XML_ozone.security.enabled=true -OZONE-SITE.XML_ozone.acl.enabled=true -OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer -OZONE-SITE.XML_ozone.administrators="testuser,recon,om" -OZONE-SITE.XML_ozone.s3.administrators="testuser,recon,om" -OZONE-SITE.XML_ozone.recon.administrators="testuser2" -OZONE-SITE.XML_ozone.s3.administrators="testuser,s3g" - -HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019 -HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012 -CORE-SITE.XML_dfs.data.transfer.protection=authentication -CORE-SITE.XML_hadoop.security.authentication=kerberos -CORE-SITE.XML_hadoop.security.auth_to_local="DEFAULT" -CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms - -OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM -OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab -OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM -OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab -OZONE-SITE.XML_ozone.recon.kerberos.keytab.file=/etc/security/keytabs/recon.keytab -OZONE-SITE.XML_ozone.recon.kerberos.principal=recon/recon@EXAMPLE.COM - -OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab -OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/s3g@EXAMPLE.COM - -OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file=/etc/security/keytabs/httpfs.keytab -OZONE-SITE.XML_ozone.httpfs.kerberos.principal=httpfs/httpfs@EXAMPLE.COM - -HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/dn@EXAMPLE.COM -HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab -HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab -HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM -HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab - -OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true -OZONE-SITE.XML_ozone.s3g.secret.http.enabled=true -OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.AuthenticationFilterInitializer - -OZONE-SITE.XML_ozone.om.http.auth.type=kerberos -OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos -OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos -OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos -OZONE-SITE.XML_ozone.s3g.secret.http.auth.type=kerberos -OZONE-SITE.XML_ozone.httpfs.http.auth.type=kerberos -OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos - -OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM -OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/scm.keytab -OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/om@EXAMPLE.COM -OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/om.keytab -OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/dn@EXAMPLE.COM -OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/dn.keytab -OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/s3g.keytab -OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM -OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.keytab=/etc/security/keytabs/httpfs.keytab -OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.principal=HTTP/httpfs@EXAMPLE.COM -OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal=* -OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab - -CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed=false -CORE-SITE.XML_hadoop.http.authentication.signature.secret.file=/etc/security/http_secret -CORE-SITE.XML_hadoop.http.authentication.type=kerberos -CORE-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM -CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab - -CORE-SITE.XML_hadoop.security.authorization=true -HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=* -HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=* -HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl=* -HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl=* -HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=* -HADOOP-POLICY.XML_ozone.security.reconfigure.protocol.acl=* - -KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users=* -KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups=* -KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts=* - -OZONE_DATANODE_SECURE_USER=root -JSVC_HOME=/usr/bin - -OZONE_CONF_DIR=/etc/hadoop -OZONE_LOG_DIR=/var/log/hadoop - -no_proxy=om,scm,recon,s3g,kdc,localhost,127.0.0.1 - diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/secure-new-cluster.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/secure-new-cluster.yaml deleted file mode 100644 index 3783d9a2c69d..000000000000 --- a/hadoop-ozone/dist/src/main/compose/xcompat/secure-new-cluster.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# reusable fragments (see https://docs.docker.com/compose/compose-file/#extension-fields) -x-new-config: - &new-config - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} - dns_search: . - env_file: - - secure-docker-config - volumes: - - ../..:/opt/hadoop - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf - -services: - kdc: - image: ${OZONE_TESTKRB5_IMAGE} - hostname: kdc - dns_search: . - volumes: - - ../..:/opt/hadoop - - ../_keytabs:/etc/security/keytabs - command: [ "krb5kdc","-n" ] - kms: - image: apache/hadoop:${HADOOP_VERSION} - hostname: kms - dns_search: . - ports: - - 9600:9600 - env_file: - - ./secure-docker-config - environment: - HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop - volumes: - - ../../libexec/transformation.py:/opt/transformation.py - command: [ "hadoop", "kms" ] - datanode: - <<: *new-config - hostname: dn - ports: - - 19864 - - 9882 - environment: - OZONE_OPTS: - command: ["ozone","datanode"] - om: - <<: *new-config - hostname: om - environment: - ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION - OZONE_OPTS: -Dcom.sun.net.ssl.checkRevocation=false - ports: - - 9874:9874 - - 9862:9862 - command: ["ozone","om"] - recon: - <<: *new-config - hostname: recon - ports: - - 9888:9888 - environment: - OZONE_OPTS: - command: ["ozone","recon"] - s3g: - <<: *new-config - hostname: s3g - environment: - OZONE_OPTS: - ports: - - 9878:9878 - command: ["ozone","s3g"] - scm: - <<: *new-config - hostname: scm - ports: - - 9876:9876 - - 9860:9860 - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" - OZONE_OPTS: - command: ["ozone","scm"] diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/secure-old-cluster.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/secure-old-cluster.yaml deleted file mode 100644 index cf3bded3bda4..000000000000 --- a/hadoop-ozone/dist/src/main/compose/xcompat/secure-old-cluster.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# reusable fragments (see https://docs.docker.com/compose/compose-file/#extension-fields) -x-old-config: - &old-config - image: apache/ozone:${OZONE_VERSION} - dns_search: . - env_file: - - secure-docker-config - volumes: - - ../..:/opt/ozone - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf - -services: - kdc: - image: ${OZONE_TESTKRB5_IMAGE} - hostname: kdc - dns_search: . - volumes: - - ../..:/opt/ozone - - ../_keytabs:/etc/security/keytabs - command: [ "krb5kdc","-n" ] - kms: - image: apache/hadoop:${HADOOP_VERSION} - hostname: kms - dns_search: . - ports: - - 9600:9600 - env_file: - - ./secure-docker-config - environment: - HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop - volumes: - - ../../libexec/transformation.py:/opt/transformation.py - command: [ "hadoop", "kms" ] - datanode: - <<: *old-config - hostname: dn - ports: - - 19864 - - 9882 - environment: - HADOOP_OPTS: - command: ["ozone","datanode"] - om: - <<: *old-config - hostname: om - environment: - ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION - OZONE_OPTS: -Dcom.sun.net.ssl.checkRevocation=false - HADOOP_OPTS: - ports: - - 9874:9874 - - 9862:9862 - command: ["ozone","om"] - recon: - <<: *old-config - hostname: recon - ports: - - 9888:9888 - environment: - HADOOP_OPTS: - command: ["ozone","recon"] - s3g: - <<: *old-config - hostname: s3g - environment: - HADOOP_OPTS: - ports: - - 9878:9878 - command: ["ozone","s3g"] - scm: - <<: *old-config - hostname: scm - ports: - - 9876:9876 - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" - HADOOP_OPTS: - command: ["ozone","scm"] diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/test-secure.sh b/hadoop-ozone/dist/src/main/compose/xcompat/test-secure.sh deleted file mode 100755 index 0010c07f66b6..000000000000 --- a/hadoop-ozone/dist/src/main/compose/xcompat/test-secure.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#suite:compat - -COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -export COMPOSE_DIR -basename=$(basename ${COMPOSE_DIR}) - -current_version="${ozone.version}" -old_versions="1.0.0 1.1.0 1.2.1 1.3.0 1.4.0" - -# shellcheck source=hadoop-ozone/dist/src/main/compose/testlib.sh -source "${COMPOSE_DIR}/../testlib.sh" - -export SECURITY_ENABLED=true -: ${OZONE_BUCKET_KEY_NAME:=key1} - -old_client() { - OZONE_DIR=/opt/ozone - container=${client} - "$@" -} - -new_client() { - OZONE_DIR=/opt/hadoop - container=new_client - client_version=${current_version} - "$@" -} - - -test_bucket_encryption() { - - execute_command_in_container ${container} kinit -k -t /etc/security/keytabs/testuser.keytab testuser/scm@EXAMPLE.COM - - execute_robot_test ${container} -N "xcompat-cluster-${cluster_version}-client-${client_version}" -v SUFFIX:${client_version} security/bucket-encryption.robot -} - -test_encryption_cross_compatibility() { - - echo "Running client connection to multiple KMS backward compatibility tests." - - echo "Starting secure cluster with COMPOSE_FILE=${COMPOSE_FILE}" - OZONE_KEEP_RESULTS=true start_docker_env - - execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME} - - new_client test_bucket_encryption - - for client_version in "$@"; do - client="old_client_${client_version//./_}" - old_client test_bucket_encryption - done - - KEEP_RUNNING=false stop_docker_env -} - -create_results_dir - -encryption_old_versions="1.2.1 1.3.0 1.4.0" -# current cluster with various clients -COMPOSE_FILE=secure-new-cluster.yaml:secure-clients.yaml cluster_version=${current_version} test_encryption_cross_compatibility ${encryption_old_versions} - -# old cluster with clients: same version and current version -for cluster_version in ${encryption_old_versions}; do - export OZONE_VERSION=${cluster_version} - COMPOSE_FILE=secure-old-cluster.yaml:secure-clients.yaml test_encryption_cross_compatibility ${cluster_version} -done diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/test.sh b/hadoop-ozone/dist/src/main/compose/xcompat/test.sh index 695d8bf06abc..8774cf2f6322 100755 --- a/hadoop-ozone/dist/src/main/compose/xcompat/test.sh +++ b/hadoop-ozone/dist/src/main/compose/xcompat/test.sh @@ -22,11 +22,15 @@ export COMPOSE_DIR basename=$(basename ${COMPOSE_DIR}) current_version="${ozone.version}" -old_versions="1.0.0 1.1.0 1.2.1 1.3.0 1.4.0" # container is needed for each version in clients.yaml +# TODO: debug acceptance test failures for client versions 1.0.0 on secure clusters +old_versions="1.1.0 1.2.1 1.3.0 1.4.0" # container is needed for each version in clients.yaml # shellcheck source=hadoop-ozone/dist/src/main/compose/testlib.sh source "${COMPOSE_DIR}/../testlib.sh" +export SECURITY_ENABLED=true +: ${OZONE_BUCKET_KEY_NAME:=key1} + old_client() { OZONE_DIR=/opt/ozone container=${client} @@ -40,24 +44,40 @@ new_client() { "$@" } +_kinit() { + execute_command_in_container ${container} kinit -k -t /etc/security/keytabs/testuser.keytab testuser/scm@EXAMPLE.COM +} + _init() { + _kinit execute_command_in_container ${container} ozone freon ockg -n1 -t1 -p warmup } _write() { + _kinit execute_robot_test ${container} -N "xcompat-cluster-${cluster_version}-client-${client_version}-write" -v SUFFIX:${client_version} compatibility/write.robot } _read() { + _kinit local data_version="$1" execute_robot_test ${container} -N "xcompat-cluster-${cluster_version}-client-${client_version}-read-${data_version}" -v SUFFIX:${data_version} compatibility/read.robot } +test_bucket_encryption() { + _kinit + execute_robot_test ${container} -N "xcompat-cluster-${cluster_version}-client-${client_version}" -v SUFFIX:${client_version} security/bucket-encryption.robot +} + test_cross_compatibility() { echo "Starting cluster with COMPOSE_FILE=${COMPOSE_FILE}" OZONE_KEEP_RESULTS=true start_docker_env + execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME} + new_client test_bucket_encryption + + container=scm _kinit execute_command_in_container scm ozone freon ockg -n1 -t1 -p warmup new_client _write new_client _read ${current_version} @@ -65,6 +85,8 @@ test_cross_compatibility() { for client_version in "$@"; do client="old_client_${client_version//./_}" + old_client test_bucket_encryption + old_client _write old_client _read ${client_version} @@ -79,7 +101,8 @@ test_ec_cross_compatibility() { echo "Running Erasure Coded storage backward compatibility tests." # local cluster_versions_with_ec="1.3.0 1.4.0 ${current_version}" local cluster_versions_with_ec="${current_version}" # until HDDS-11334 - local non_ec_client_versions="1.0.0 1.1.0 1.2.1" + # TODO: debug acceptance test failures for client versions 1.0.0 on secure clusters + local non_ec_client_versions="1.1.0 1.2.1" for cluster_version in ${cluster_versions_with_ec}; do export COMPOSE_FILE=new-cluster.yaml:clients.yaml cluster_version=${cluster_version} @@ -102,12 +125,14 @@ test_ec_cross_compatibility() { local prefix=$(LC_CTYPE=C tr -dc '[:alnum:]' < /dev/urandom | head -c 5 | tr '[:upper:]' '[:lower:]') OZONE_DIR=/opt/hadoop + new_client _kinit execute_robot_test new_client --include setup-ec-data -N "xcompat-cluster-${cluster_version}-setup-data" -v prefix:"${prefix}" ec/backward-compat.robot OZONE_DIR=/opt/ozone for client_version in ${non_ec_client_versions}; do client="old_client_${client_version//./_}" unset OUTPUT_PATH + container="${client}" _kinit execute_robot_test "${client}" --include test-ec-compat -N "xcompat-cluster-${cluster_version}-client-${client_version}-read-${cluster_version}" -v prefix:"${prefix}" ec/backward-compat.robot done From 0810ff54ca12749769bb2bb43bf4dd280c2bf44f Mon Sep 17 00:00:00 2001 From: saketa Date: Mon, 9 Sep 2024 21:46:40 -0700 Subject: [PATCH 8/9] HDDS-11371. Reusing OM service info to get server defaults instead of adding new server API. --- .../hadoop/ozone/OzoneManagerVersion.java | 3 -- .../hadoop/ozone/client/rpc/RpcClient.java | 13 ++++-- .../java/org/apache/hadoop/ozone/OmUtils.java | 1 - .../hadoop/ozone/om/helpers/ServiceInfo.java | 46 ++++++++++++++++++- .../om/protocol/OzoneManagerProtocol.java | 9 ---- ...ManagerProtocolClientSideTranslatorPB.java | 19 -------- .../src/main/proto/OmClientProtocol.proto | 11 +---- .../apache/hadoop/ozone/om/OzoneManager.java | 23 ++++------ .../OzoneManagerRequestHandler.java | 7 --- 9 files changed, 62 insertions(+), 70 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java index 4a3ddd1ee118..eec2ceeb5e8d 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java @@ -45,9 +45,6 @@ public enum OzoneManagerVersion implements ComponentVersion { ATOMIC_REWRITE_KEY(6, "OzoneManager version that supports rewriting key as atomic operation"), HBASE_SUPPORT(7, "OzoneManager version that supports HBase integration"), - SERVER_DEFAULTS(8, "OzoneManager version that supports get server" - + " defaults API."), - FUTURE_VERSION(-1, "Used internally in the client when the server side is " + " newer and an unknown server version has arrived to the client."); diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 5a5937b01f7e..d0e91ecfcff7 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -2580,15 +2580,18 @@ public KeyProvider call() throws Exception { @Override public OzoneFsServerDefaults getServerDefaults() throws IOException { - if (omVersion.compareTo(OzoneManagerVersion.SERVER_DEFAULTS) < 0) { - return null; - } long now = Time.monotonicNow(); if ((serverDefaults == null) || (now - serverDefaultsLastUpdate > serverDefaultsValidityPeriod)) { try { - serverDefaults = ozoneManagerClient.getServerDefaults(); - serverDefaultsLastUpdate = now; + for (ServiceInfo si : ozoneManagerClient.getServiceInfo() + .getServiceInfoList()) { + if (si.getServerDefaults() == null) { + continue; + } + serverDefaults = si.getServerDefaults(); + serverDefaultsLastUpdate = now; + } } catch (Exception e) { LOG.warn("Could not get server defaults from OM.", e); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 8fa8921cc9a9..59177f0e9854 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -278,7 +278,6 @@ public static boolean isReadOnly( case SetSafeMode: case PrintCompactionLogDag: case GetSnapshotInfo: - case GetServerDefaults: case GetQuotaRepairStatus: case StartQuotaRepair: return true; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfo.java index c8bdbf43c429..5dbe3487e19c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfo.java @@ -25,6 +25,7 @@ import java.util.Map; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType; +import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo; @@ -59,6 +60,7 @@ public final class ServiceInfo { private Map ports; private OMRoleInfo omRoleInfo; + private OzoneFsServerDefaults serverDefaults; /** * Default constructor for JSON deserialization. @@ -76,6 +78,24 @@ private ServiceInfo(NodeType nodeType, List portList, OzoneManagerVersion omVersion, OMRoleInfo omRole) { + this(nodeType, hostname, portList, omVersion, omRole, null); + } + + /** + * Constructs the ServiceInfo for the {@code nodeType}. + * @param nodeType type of node/service + * @param hostname hostname of the service + * @param portList list of ports the service listens to + * @param omVersion Om Version + * @param omRole OM role Ino + * @param keyProviderUri KMS provider URI + */ + private ServiceInfo(NodeType nodeType, + String hostname, + List portList, + OzoneManagerVersion omVersion, + OMRoleInfo omRole, + OzoneFsServerDefaults serverDefaults) { Preconditions.checkNotNull(nodeType); Preconditions.checkNotNull(hostname); this.nodeType = nodeType; @@ -86,6 +106,7 @@ private ServiceInfo(NodeType nodeType, ports.put(port.getType(), port.getValue()); } this.omRoleInfo = omRole; + this.serverDefaults = serverDefaults; } /** @@ -143,6 +164,15 @@ public OMRoleInfo getOmRoleInfo() { return omRoleInfo; } + /** + * Returns the Ozone Server default configuration. + * @return OmRoleInfo + */ + @JsonIgnore + public OzoneFsServerDefaults getServerDefaults() { + return serverDefaults; + } + /** * Converts {@link ServiceInfo} to OzoneManagerProtocolProtos.ServiceInfo. * @@ -170,6 +200,9 @@ public OzoneManagerProtocolProtos.ServiceInfo getProtobuf() { if (nodeType == NodeType.OM && omRoleInfo != null) { builder.setOmRole(omRoleInfo); } + if (serverDefaults != null) { + builder.setServerDefaults(serverDefaults.getProtobuf()); + } return builder.build(); } @@ -185,7 +218,9 @@ public static ServiceInfo getFromProtobuf( serviceInfo.getHostname(), serviceInfo.getServicePortsList(), OzoneManagerVersion.fromProtoValue(serviceInfo.getOMVersion()), - serviceInfo.hasOmRole() ? serviceInfo.getOmRole() : null); + serviceInfo.hasOmRole() ? serviceInfo.getOmRole() : null, + serviceInfo.hasServerDefaults() ? OzoneFsServerDefaults.getFromProtobuf( + serviceInfo.getServerDefaults()) : null); } /** @@ -206,6 +241,7 @@ public static class Builder { private List portList = new ArrayList<>(); private OMRoleInfo omRoleInfo; private OzoneManagerVersion omVersion; + private OzoneFsServerDefaults serverDefaults; /** * Gets the Om Client Protocol Version. @@ -259,6 +295,11 @@ public Builder setOmRoleInfo(OMRoleInfo omRole) { return this; } + public Builder setServerDefaults(OzoneFsServerDefaults defaults) { + serverDefaults = defaults; + return this; + } + /** * Builds and returns {@link ServiceInfo} with the set values. * @return {@link ServiceInfo} @@ -268,7 +309,8 @@ public ServiceInfo build() { host, portList, omVersion, - omRoleInfo); + omRoleInfo, + serverDefaults); } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 37481b00ea28..79ddbbf8dad5 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -28,7 +28,6 @@ import org.apache.hadoop.fs.SafeModeAction; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.om.IOmMetadataReader; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -1146,14 +1145,6 @@ void setTimes(OmKeyArgs keyArgs, long mtime, long atime) boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException; - /** - * Get server default configurations. - * - * @return OzoneFsServerDefaults some default configurations from server. - * @throws IOException - */ - OzoneFsServerDefaults getServerDefaults() throws IOException; - /** * Get status of last triggered quota repair in OM. * @return String diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index bdd1428b16dc..276959fe2d00 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -41,7 +41,6 @@ import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.ErrorInfo; @@ -193,8 +192,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Secret; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SafeMode; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServerDefaultsRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServerDefaultsResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetAclRequest; @@ -2563,22 +2560,6 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) return setSafeModeResponse.getResponse(); } - @Override - public OzoneFsServerDefaults getServerDefaults() - throws IOException { - ServerDefaultsRequest serverDefaultsRequest = - ServerDefaultsRequest.newBuilder().build(); - - OMRequest omRequest = createOMRequest(Type.GetServerDefaults) - .setServerDefaultsRequest(serverDefaultsRequest).build(); - - ServerDefaultsResponse serverDefaultsResponse = - handleError(submitRequest(omRequest)).getServerDefaultsResponse(); - - return OzoneFsServerDefaults.getFromProtobuf( - serverDefaultsResponse.getServerDefaults()); - } - @Override public String getQuotaRepairStatus() throws IOException { OzoneManagerProtocolProtos.GetQuotaRepairStatusRequest quotaRepairStatusRequest = diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index e79797993c13..d1d65d5d768e 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -150,7 +150,6 @@ enum Type { RenameSnapshot = 131; ListOpenFiles = 132; QuotaRepair = 133; - GetServerDefaults = 134; GetQuotaRepairStatus = 135; StartQuotaRepair = 136; } @@ -292,7 +291,6 @@ message OMRequest { optional RenameSnapshotRequest RenameSnapshotRequest = 129; optional ListOpenFilesRequest ListOpenFilesRequest = 130; optional QuotaRepairRequest QuotaRepairRequest = 131; - optional ServerDefaultsRequest ServerDefaultsRequest = 132; optional GetQuotaRepairStatusRequest GetQuotaRepairStatusRequest = 133; optional StartQuotaRepairRequest StartQuotaRepairRequest = 134; } @@ -422,7 +420,6 @@ message OMResponse { optional RenameSnapshotResponse RenameSnapshotResponse = 132; optional ListOpenFilesResponse ListOpenFilesResponse = 133; optional QuotaRepairResponse QuotaRepairResponse = 134; - optional ServerDefaultsResponse ServerDefaultsResponse = 135; optional GetQuotaRepairStatusResponse GetQuotaRepairStatusResponse = 136; optional StartQuotaRepairResponse StartQuotaRepairResponse = 137; } @@ -1629,6 +1626,7 @@ message ServiceInfo { repeated ServicePort servicePorts = 3; optional OMRoleInfo omRole = 4; optional int32 OMVersion = 5 [default = 0]; + optional FsServerDefaultsProto serverDefaults = 6; } message MultipartInfoInitiateRequest { @@ -2221,17 +2219,10 @@ message BucketQuotaCount { message QuotaRepairResponse { } -message ServerDefaultsRequest { -} - message FsServerDefaultsProto { optional string keyProviderUri = 1; } -message ServerDefaultsResponse { - required FsServerDefaultsProto serverDefaults = 1; -} - message GetQuotaRepairStatusRequest { } message GetQuotaRepairStatusResponse { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index b6903ca9e91f..ca6f58333b12 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -437,7 +437,6 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl private List ratisReporterList = null; private KeyProviderCryptoExtension kmsProvider; - private OzoneFsServerDefaults serverDefaults; private final OMLayoutVersionManager versionManager; private final ReplicationConfigValidator replicationConfigValidator; @@ -655,14 +654,6 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) kmsProvider = null; LOG.error("Fail to create Key Provider"); } - Configuration hadoopConfig = - LegacyHadoopConfigurationSource.asHadoopConfiguration(configuration); - URI keyProviderUri = KMSUtil.getKeyProviderUri( - hadoopConfig, - CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH); - String keyProviderUriStr = - (keyProviderUri != null) ? keyProviderUri.toString() : null; - serverDefaults = new OzoneFsServerDefaults(keyProviderUriStr); if (secConfig.isSecurityEnabled()) { omComponent = OM_DAEMON + "-" + omId; HddsProtos.OzoneManagerDetailsProto omInfo = @@ -3138,6 +3129,15 @@ public List getServiceList() throws IOException { .setType(ServicePort.Type.RPC) .setValue(omRpcAddress.getPort()) .build()); + Configuration hadoopConfig = + LegacyHadoopConfigurationSource.asHadoopConfiguration(configuration); + URI keyProviderUri = KMSUtil.getKeyProviderUri( + hadoopConfig, + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH); + String keyProviderUriStr = + (keyProviderUri != null) ? keyProviderUri.toString() : null; + omServiceInfoBuilder.setServerDefaults( + new OzoneFsServerDefaults(keyProviderUriStr)); if (httpServer != null && httpServer.getHttpAddress() != null) { omServiceInfoBuilder.addServicePort(ServicePort.newBuilder() @@ -4747,11 +4747,6 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) } } - @Override - public OzoneFsServerDefaults getServerDefaults() { - return serverDefaults; - } - @Override public String getQuotaRepairStatus() throws IOException { checkAdminUserPrivilege("quota repair status"); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 576fac48c736..5682b040e859 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -130,7 +130,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetSafeModeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetSafeModeResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3VolumeContextResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServerDefaultsResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotDiffRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotDiffResponse; @@ -376,12 +375,6 @@ public OMResponse handleReadRequest(OMRequest request) { getSnapshotInfo(request.getSnapshotInfoRequest()); responseBuilder.setSnapshotInfoResponse(snapshotInfoResponse); break; - case GetServerDefaults: - responseBuilder.setServerDefaultsResponse( - ServerDefaultsResponse.newBuilder() - .setServerDefaults(impl.getServerDefaults().getProtobuf()) - .build()); - break; case GetQuotaRepairStatus: OzoneManagerProtocolProtos.GetQuotaRepairStatusResponse quotaRepairStatusRsp = getQuotaRepairStatus(request.getGetQuotaRepairStatusRequest()); From cecaaadd08c2cb99e9b26445ad28f099f6dd02d8 Mon Sep 17 00:00:00 2001 From: saketa Date: Fri, 13 Sep 2024 10:36:56 -0700 Subject: [PATCH 9/9] HDDS-11371. Exiting loop after first non-null loop in RpcClient.getServerDefaults() --- .../org/apache/hadoop/ozone/client/rpc/RpcClient.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index dd38ba284cda..fe9866401765 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -2593,11 +2593,11 @@ public OzoneFsServerDefaults getServerDefaults() throws IOException { try { for (ServiceInfo si : ozoneManagerClient.getServiceInfo() .getServiceInfoList()) { - if (si.getServerDefaults() == null) { - continue; + if (si.getServerDefaults() != null) { + serverDefaults = si.getServerDefaults(); + serverDefaultsLastUpdate = now; + break; } - serverDefaults = si.getServerDefaults(); - serverDefaultsLastUpdate = now; } } catch (Exception e) { LOG.warn("Could not get server defaults from OM.", e);