From 994a83ab36062dc155f8d3bf834797b46f9e01e6 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 3 Dec 2024 08:34:57 +0100 Subject: [PATCH 1/5] add helper to regenerate examples --- .../dev-support/k8s/regenerate-examples.sh | 27 +++++++++++++++++++ .../dist/src/main/k8s/examples/README.md | 22 +++++++++++++++ 2 files changed, 49 insertions(+) create mode 100755 hadoop-ozone/dev-support/k8s/regenerate-examples.sh create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/README.md diff --git a/hadoop-ozone/dev-support/k8s/regenerate-examples.sh b/hadoop-ozone/dev-support/k8s/regenerate-examples.sh new file mode 100755 index 000000000000..ffedaa3b725d --- /dev/null +++ b/hadoop-ozone/dev-support/k8s/regenerate-examples.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -u -o pipefail + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../../.." || exit 1 + +source "hadoop-ozone/dev-support/checks/_lib.sh" + +install_flekszible + +hadoop-ozone/dist/src/main/k8s/examples/regenerate-all.sh diff --git a/hadoop-ozone/dist/src/main/k8s/examples/README.md b/hadoop-ozone/dist/src/main/k8s/examples/README.md new file mode 100644 index 000000000000..574efc6704a6 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/README.md @@ -0,0 +1,22 @@ + + +# Ozone examples for Kubernetes + +This directory contains example resources for running Ozone in Kubernetes. + +Note that the files are generated based on the [definitions](../definitions) using [Flekszible](https://github.com/elek/flekszible). If you would like to modify them permanently, edit the definition and regenerate the examples by running `regenerate-all.sh` after installing Flekszible. As a developer, you can run `hadoop-ozone/dev-support/k8s/regenerate-examples.sh`, which will also install Flekszible. From 8f6fc4d6da0b001979b6a765fbc60303cf2d1471 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 3 Dec 2024 08:49:02 +0100 Subject: [PATCH 2/5] HDDS-11845. Extract k8s definitions for HttpFS and Recon from getting-started example --- .../main/k8s/definitions/ozone/config.yaml | 8 ++- .../definitions/ozone/httpfs-ss-service.yaml | 27 ++++++++++ .../main/k8s/definitions/ozone/httpfs-ss.yaml | 44 ++++++++++++++++ .../definitions/ozone/recon-ss-service.yaml | 27 ++++++++++ .../main/k8s/definitions/ozone/recon-ss.yaml | 52 +++++++++++++++++++ 5 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone/httpfs-ss-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone/httpfs-ss.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone/recon-ss-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone/recon-ss.yaml diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml index f451a893eba7..22be047bf3ce 100644 --- a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml @@ -13,12 +13,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + apiVersion: v1 kind: ConfigMap metadata: name: config data: - + HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config + CORE-SITE.XML_fs.defaultFS: ofs://om/ + CORE-SITE.XML_fs.trash.interval: "1" + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: "*" + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: "*" OZONE-SITE.XML_hdds.datanode.dir: "/data/storage" OZONE-SITE.XML_ozone.scm.datanode.id.dir: "/data/metadata" OZONE-SITE.XML_ozone.metadata.dirs: "/data/metadata" @@ -26,6 +31,7 @@ data: OZONE-SITE.XML_ozone.om.address: "om-0.om" OZONE-SITE.XML_ozone.scm.client.address: "scm-0.scm" OZONE-SITE.XML_ozone.scm.names: "scm-0.scm" + OZONE-SITE.XML_ozone.recon.address: "recon-0.recon" OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/httpfs-ss-service.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/httpfs-ss-service.yaml new file mode 100644 index 000000000000..0b8f11752464 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/httpfs-ss-service.yaml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Service +metadata: + name: httpfs +spec: + ports: + - port: 14000 + name: rest + clusterIP: None + selector: + app: ozone + component: httpfs diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/httpfs-ss.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/httpfs-ss.yaml new file mode 100644 index 000000000000..1084ca0373c5 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/httpfs-ss.yaml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: httpfs + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: httpfs + serviceName: httpfs + replicas: 1 + template: + metadata: + labels: + app: ozone + component: httpfs + spec: + containers: + - name: httpfs + image: "@docker.image@" + args: ["ozone","httpfs"] + livenessProbe: + httpGet: + path: /webhdfs/v1/?op=LISTSTATUS&user.name=hadoop + port: 14000 + initialDelaySeconds: 30 diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/recon-ss-service.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/recon-ss-service.yaml new file mode 100644 index 000000000000..a7994a6b91a8 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/recon-ss-service.yaml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Service +metadata: + name: recon +spec: + ports: + - port: 9888 + name: ui + clusterIP: None + selector: + app: ozone + component: recon diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/recon-ss.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/recon-ss.yaml new file mode 100644 index 000000000000..f06dc54c6648 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/recon-ss.yaml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: recon + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: recon + serviceName: recon + replicas: 1 + template: + metadata: + labels: + app: ozone + component: recon + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9888" + prometheus.io/path: "/prom" + spec: + securityContext: + fsGroup: 1000 + containers: + - name: recon + image: "@docker.image@" + args: ["ozone","recon"] + env: + - name: WAITFOR + value: scm-0.scm:9876 + livenessProbe: + tcpSocket: + port: 9891 + initialDelaySeconds: 30 + volumes: [] From d5c96085877f49a7719b292d00b124d0e0d0d14e Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 3 Dec 2024 08:49:40 +0100 Subject: [PATCH 3/5] regenerate examples --- .../getting-started/config-configmap.yaml | 4 +- .../getting-started/kustomization.yaml | 12 ++-- .../examples/minikube/config-configmap.yaml | 6 ++ .../minikube/httpfs-public-service.yaml | 28 ++++++++ .../k8s/examples/minikube/httpfs-service.yaml | 28 ++++++++ .../examples/minikube/httpfs-statefulset.yaml | 55 +++++++++++++++ .../k8s/examples/minikube/kustomization.yaml | 6 ++ .../minikube/recon-public-service.yaml | 28 ++++++++ .../k8s/examples/minikube/recon-service.yaml | 28 ++++++++ .../examples/minikube/recon-statefulset.yaml | 63 +++++++++++++++++ .../examples/ozone-dev/config-configmap.yaml | 6 ++ .../ozone-dev/httpfs-public-service.yaml | 28 ++++++++ .../examples/ozone-dev/httpfs-service.yaml | 28 ++++++++ .../ozone-dev/httpfs-statefulset.yaml | 62 ++++++++++++++++ .../k8s/examples/ozone-dev/kustomization.yaml | 6 ++ .../ozone-dev/recon-public-service.yaml | 28 ++++++++ .../k8s/examples/ozone-dev/recon-service.yaml | 28 ++++++++ .../examples/ozone-dev/recon-statefulset.yaml | 69 ++++++++++++++++++ .../examples/ozone-ha/config-configmap.yaml | 6 ++ .../k8s/examples/ozone-ha/httpfs-service.yaml | 28 ++++++++ .../examples/ozone-ha/httpfs-statefulset.yaml | 61 ++++++++++++++++ .../k8s/examples/ozone-ha/kustomization.yaml | 4 ++ .../k8s/examples/ozone-ha/recon-service.yaml | 28 ++++++++ .../examples/ozone-ha/recon-statefulset.yaml | 70 +++++++++++++++++++ .../k8s/examples/ozone/config-configmap.yaml | 6 ++ .../k8s/examples/ozone/httpfs-service.yaml | 28 ++++++++ .../examples/ozone/httpfs-statefulset.yaml | 61 ++++++++++++++++ .../k8s/examples/ozone/kustomization.yaml | 4 ++ .../k8s/examples/ozone/recon-service.yaml | 28 ++++++++ .../k8s/examples/ozone/recon-statefulset.yaml | 70 +++++++++++++++++++ 30 files changed, 899 insertions(+), 8 deletions(-) create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-public-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-statefulset.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-public-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-statefulset.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-public-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-statefulset.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-public-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-statefulset.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/httpfs-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/httpfs-statefulset.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/recon-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/recon-statefulset.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/httpfs-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/httpfs-statefulset.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/recon-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/recon-statefulset.yaml diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml index bb0608dac82d..e32d18240766 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml @@ -22,8 +22,8 @@ data: HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config CORE-SITE.XML_fs.defaultFS: ofs://om/ CORE-SITE.XML_fs.trash.interval: "1" - HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: "*" - HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: "*" + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*' OZONE-SITE.XML_hdds.datanode.dir: /data/storage OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/kustomization.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/kustomization.yaml index 4f60be17872b..427c654d543f 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/kustomization.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/kustomization.yaml @@ -18,19 +18,19 @@ resources: - config-configmap.yaml - datanode-service.yaml - datanode-statefulset.yaml +- httpfs-service.yaml +- httpfs-statefulset.yaml - om-service.yaml - om-statefulset.yaml +- recon-service.yaml +- recon-statefulset.yaml - s3g-service.yaml - s3g-statefulset.yaml - scm-service.yaml - scm-statefulset.yaml -- httpfs-service.yaml -- httpfs-statefulset.yaml -- recon-service.yaml -- recon-statefulset.yaml - datanode-public-service.yaml +- httpfs-public-service.yaml - om-public-service.yaml +- recon-public-service.yaml - s3g-public-service.yaml - scm-public-service.yaml -- httpfs-public-service.yaml -- recon-public-service.yaml diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml index 2a0cf869a59c..e32d18240766 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml @@ -19,6 +19,11 @@ kind: ConfigMap metadata: name: config data: + HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config + CORE-SITE.XML_fs.defaultFS: ofs://om/ + CORE-SITE.XML_fs.trash.interval: "1" + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*' OZONE-SITE.XML_hdds.datanode.dir: /data/storage OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata @@ -26,6 +31,7 @@ data: OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm OZONE-SITE.XML_ozone.scm.names: scm-0.scm + OZONE-SITE.XML_ozone.recon.address: recon-0.recon OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-public-service.yaml new file mode 100644 index 000000000000..d8586250553a --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-public-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: httpfs-public +spec: + ports: + - port: 14000 + name: rest + selector: + app: ozone + component: httpfs + type: NodePort diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-service.yaml new file mode 100644 index 000000000000..0ab49c2d72e4 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: httpfs +spec: + ports: + - port: 14000 + name: rest + clusterIP: None + selector: + app: ozone + component: httpfs diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-statefulset.yaml new file mode 100644 index 000000000000..7bca21585c13 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-statefulset.yaml @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: httpfs + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: httpfs + serviceName: httpfs + replicas: 1 + template: + metadata: + labels: + app: ozone + component: httpfs + spec: + containers: + - name: httpfs + image: '@docker.image@' + args: + - ozone + - httpfs + livenessProbe: + httpGet: + path: /webhdfs/v1/?op=LISTSTATUS&user.name=hadoop + port: 14000 + initialDelaySeconds: 30 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/kustomization.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/kustomization.yaml index 3059b9c801e2..427c654d543f 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/minikube/kustomization.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/kustomization.yaml @@ -18,13 +18,19 @@ resources: - config-configmap.yaml - datanode-service.yaml - datanode-statefulset.yaml +- httpfs-service.yaml +- httpfs-statefulset.yaml - om-service.yaml - om-statefulset.yaml +- recon-service.yaml +- recon-statefulset.yaml - s3g-service.yaml - s3g-statefulset.yaml - scm-service.yaml - scm-statefulset.yaml - datanode-public-service.yaml +- httpfs-public-service.yaml - om-public-service.yaml +- recon-public-service.yaml - s3g-public-service.yaml - scm-public-service.yaml diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-public-service.yaml new file mode 100644 index 000000000000..c737a02f446f --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-public-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: recon-public +spec: + ports: + - port: 9888 + name: ui + selector: + app: ozone + component: recon + type: NodePort diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-service.yaml new file mode 100644 index 000000000000..9c52d393d55d --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: recon +spec: + ports: + - port: 9888 + name: ui + clusterIP: None + selector: + app: ozone + component: recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-statefulset.yaml new file mode 100644 index 000000000000..8b9ee191d035 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-statefulset.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: recon + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: recon + serviceName: recon + replicas: 1 + template: + metadata: + labels: + app: ozone + component: recon + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9888" + prometheus.io/path: /prom + spec: + securityContext: + fsGroup: 1000 + containers: + - name: recon + image: '@docker.image@' + args: + - ozone + - recon + env: + - name: WAITFOR + value: scm-0.scm:9876 + livenessProbe: + tcpSocket: + port: 9891 + initialDelaySeconds: 30 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml index 1d85fe91f5c9..0c02a5447c74 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml @@ -19,6 +19,11 @@ kind: ConfigMap metadata: name: config data: + HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config + CORE-SITE.XML_fs.defaultFS: ofs://om/ + CORE-SITE.XML_fs.trash.interval: "1" + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*' OZONE-SITE.XML_hdds.datanode.dir: /data/storage OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata @@ -26,6 +31,7 @@ data: OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm OZONE-SITE.XML_ozone.scm.names: scm-0.scm + OZONE-SITE.XML_ozone.recon.address: recon-0.recon OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-public-service.yaml new file mode 100644 index 000000000000..d8586250553a --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-public-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: httpfs-public +spec: + ports: + - port: 14000 + name: rest + selector: + app: ozone + component: httpfs + type: NodePort diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-service.yaml new file mode 100644 index 000000000000..0ab49c2d72e4 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: httpfs +spec: + ports: + - port: 14000 + name: rest + clusterIP: None + selector: + app: ozone + component: httpfs diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-statefulset.yaml new file mode 100644 index 000000000000..59abe8547f64 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-statefulset.yaml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: httpfs + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: httpfs + serviceName: httpfs + replicas: 1 + template: + metadata: + labels: + app: ozone + component: httpfs + spec: + containers: + - name: httpfs + image: '@docker.image@' + args: + - ozone + - httpfs + livenessProbe: + httpGet: + path: /webhdfs/v1/?op=LISTSTATUS&user.name=hadoop + port: 14000 + initialDelaySeconds: 30 + envFrom: + - configMapRef: + name: config + env: + - name: JAEGER_SAMPLER_TYPE + value: probabilistic + - name: JAEGER_SAMPLER_PARAM + value: "0.01" + - name: JAEGER_AGENT_HOST + value: jaeger-0.jaeger + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/kustomization.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/kustomization.yaml index cf0cbe152d2e..f654f6f510fe 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/kustomization.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/kustomization.yaml @@ -18,8 +18,12 @@ resources: - config-configmap.yaml - datanode-service.yaml - datanode-statefulset.yaml +- httpfs-service.yaml +- httpfs-statefulset.yaml - om-service.yaml - om-statefulset.yaml +- recon-service.yaml +- recon-statefulset.yaml - s3g-service.yaml - s3g-statefulset.yaml - scm-service.yaml @@ -33,7 +37,9 @@ resources: - jaeger-service.yaml - jaeger-statefulset.yaml - datanode-public-service.yaml +- httpfs-public-service.yaml - om-public-service.yaml +- recon-public-service.yaml - s3g-public-service.yaml - scm-public-service.yaml - jaeger-public-service.yaml diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-public-service.yaml new file mode 100644 index 000000000000..c737a02f446f --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-public-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: recon-public +spec: + ports: + - port: 9888 + name: ui + selector: + app: ozone + component: recon + type: NodePort diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-service.yaml new file mode 100644 index 000000000000..9c52d393d55d --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: recon +spec: + ports: + - port: 9888 + name: ui + clusterIP: None + selector: + app: ozone + component: recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-statefulset.yaml new file mode 100644 index 000000000000..6466c29595cf --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-statefulset.yaml @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: recon + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: recon + serviceName: recon + replicas: 1 + template: + metadata: + labels: + app: ozone + component: recon + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9888" + prometheus.io/path: /prom + spec: + securityContext: + fsGroup: 1000 + containers: + - name: recon + image: '@docker.image@' + args: + - ozone + - recon + env: + - name: WAITFOR + value: scm-0.scm:9876 + - name: JAEGER_SAMPLER_TYPE + value: probabilistic + - name: JAEGER_SAMPLER_PARAM + value: "0.01" + - name: JAEGER_AGENT_HOST + value: jaeger-0.jaeger + livenessProbe: + tcpSocket: + port: 9891 + initialDelaySeconds: 30 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml index 193a0618ab4f..4f3e37f383a6 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml @@ -19,10 +19,16 @@ kind: ConfigMap metadata: name: config data: + HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config + CORE-SITE.XML_fs.defaultFS: ofs://om/ + CORE-SITE.XML_fs.trash.interval: "1" + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*' OZONE-SITE.XML_hdds.datanode.dir: /data/storage OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata OZONE-SITE.XML_ozone.om.address: om-0.om + OZONE-SITE.XML_ozone.recon.address: recon-0.recon OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/httpfs-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/httpfs-service.yaml new file mode 100644 index 000000000000..0ab49c2d72e4 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/httpfs-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: httpfs +spec: + ports: + - port: 14000 + name: rest + clusterIP: None + selector: + app: ozone + component: httpfs diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/httpfs-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/httpfs-statefulset.yaml new file mode 100644 index 000000000000..2c076ae8fcd4 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/httpfs-statefulset.yaml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: httpfs + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: httpfs + serviceName: httpfs + replicas: 1 + template: + metadata: + labels: + app: ozone + component: httpfs + spec: + containers: + - name: httpfs + image: '@docker.image@' + args: + - ozone + - httpfs + livenessProbe: + httpGet: + path: /webhdfs/v1/?op=LISTSTATUS&user.name=hadoop + port: 14000 + initialDelaySeconds: 30 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/kustomization.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/kustomization.yaml index 6b3d553113e6..49b6b6e2606b 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/kustomization.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/kustomization.yaml @@ -18,8 +18,12 @@ resources: - config-configmap.yaml - datanode-service.yaml - datanode-statefulset.yaml +- httpfs-service.yaml +- httpfs-statefulset.yaml - om-service.yaml - om-statefulset.yaml +- recon-service.yaml +- recon-statefulset.yaml - s3g-service.yaml - s3g-statefulset.yaml - scm-service.yaml diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/recon-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/recon-service.yaml new file mode 100644 index 000000000000..9c52d393d55d --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/recon-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: recon +spec: + ports: + - port: 9888 + name: ui + clusterIP: None + selector: + app: ozone + component: recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/recon-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/recon-statefulset.yaml new file mode 100644 index 000000000000..445c2e222d75 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/recon-statefulset.yaml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: recon + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: recon + serviceName: recon + replicas: 1 + template: + metadata: + labels: + app: ozone + component: recon + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9888" + prometheus.io/path: /prom + spec: + securityContext: + fsGroup: 1000 + containers: + - name: recon + image: '@docker.image@' + args: + - ozone + - recon + env: + - name: WAITFOR + value: scm-0.scm:9876 + livenessProbe: + tcpSocket: + port: 9891 + initialDelaySeconds: 30 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumes: [] + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml index 859307310507..c1f688afd642 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml @@ -19,6 +19,11 @@ kind: ConfigMap metadata: name: config data: + HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config + CORE-SITE.XML_fs.defaultFS: ofs://om/ + CORE-SITE.XML_fs.trash.interval: "1" + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*' OZONE-SITE.XML_hdds.datanode.dir: /data/storage OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata @@ -26,6 +31,7 @@ data: OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm OZONE-SITE.XML_ozone.scm.names: scm-0.scm + OZONE-SITE.XML_ozone.recon.address: recon-0.recon OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/httpfs-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/httpfs-service.yaml new file mode 100644 index 000000000000..0ab49c2d72e4 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/httpfs-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: httpfs +spec: + ports: + - port: 14000 + name: rest + clusterIP: None + selector: + app: ozone + component: httpfs diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/httpfs-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/httpfs-statefulset.yaml new file mode 100644 index 000000000000..2c076ae8fcd4 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/httpfs-statefulset.yaml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: httpfs + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: httpfs + serviceName: httpfs + replicas: 1 + template: + metadata: + labels: + app: ozone + component: httpfs + spec: + containers: + - name: httpfs + image: '@docker.image@' + args: + - ozone + - httpfs + livenessProbe: + httpGet: + path: /webhdfs/v1/?op=LISTSTATUS&user.name=hadoop + port: 14000 + initialDelaySeconds: 30 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/kustomization.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/kustomization.yaml index 6b3d553113e6..49b6b6e2606b 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/kustomization.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/kustomization.yaml @@ -18,8 +18,12 @@ resources: - config-configmap.yaml - datanode-service.yaml - datanode-statefulset.yaml +- httpfs-service.yaml +- httpfs-statefulset.yaml - om-service.yaml - om-statefulset.yaml +- recon-service.yaml +- recon-statefulset.yaml - s3g-service.yaml - s3g-statefulset.yaml - scm-service.yaml diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/recon-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/recon-service.yaml new file mode 100644 index 000000000000..9c52d393d55d --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/recon-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: recon +spec: + ports: + - port: 9888 + name: ui + clusterIP: None + selector: + app: ozone + component: recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/recon-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/recon-statefulset.yaml new file mode 100644 index 000000000000..445c2e222d75 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/recon-statefulset.yaml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: recon + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: recon + serviceName: recon + replicas: 1 + template: + metadata: + labels: + app: ozone + component: recon + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9888" + prometheus.io/path: /prom + spec: + securityContext: + fsGroup: 1000 + containers: + - name: recon + image: '@docker.image@' + args: + - ozone + - recon + env: + - name: WAITFOR + value: scm-0.scm:9876 + livenessProbe: + tcpSocket: + port: 9891 + initialDelaySeconds: 30 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumes: [] + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi From e07c4e8e438f06c4dbe2e5def2e59b5320e4be10 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 3 Dec 2024 08:50:37 +0100 Subject: [PATCH 4/5] sort config properties by name --- .../src/main/k8s/definitions/ozone/config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml index 22be047bf3ce..8239325dd972 100644 --- a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml @@ -19,22 +19,22 @@ kind: ConfigMap metadata: name: config data: - HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config CORE-SITE.XML_fs.defaultFS: ofs://om/ CORE-SITE.XML_fs.trash.interval: "1" - HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: "*" + HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: "*" + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: "*" + OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" OZONE-SITE.XML_hdds.datanode.dir: "/data/storage" - OZONE-SITE.XML_ozone.scm.datanode.id.dir: "/data/metadata" + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" + OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_ozone.metadata.dirs: "/data/metadata" - OZONE-SITE.XML_ozone.scm.block.client.address: "scm-0.scm" OZONE-SITE.XML_ozone.om.address: "om-0.om" + OZONE-SITE.XML_ozone.recon.address: "recon-0.recon" + OZONE-SITE.XML_ozone.scm.block.client.address: "scm-0.scm" OZONE-SITE.XML_ozone.scm.client.address: "scm-0.scm" + OZONE-SITE.XML_ozone.scm.datanode.id.dir: "/data/metadata" OZONE-SITE.XML_ozone.scm.names: "scm-0.scm" - OZONE-SITE.XML_ozone.recon.address: "recon-0.recon" - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" - OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" - OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" LOG4J.PROPERTIES_log4j.rootLogger: "INFO, stdout" LOG4J.PROPERTIES_log4j.appender.stdout: "org.apache.log4j.ConsoleAppender" LOG4J.PROPERTIES_log4j.appender.stdout.layout: "org.apache.log4j.PatternLayout" From 78009e9d10bc338f812c57cf9c9e994dde5bc8d9 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 3 Dec 2024 08:51:11 +0100 Subject: [PATCH 5/5] regenerate examples --- .../getting-started/config-configmap.yaml | 16 ++++++++-------- .../k8s/examples/minikube/config-configmap.yaml | 16 ++++++++-------- .../k8s/examples/ozone-dev/config-configmap.yaml | 16 ++++++++-------- .../k8s/examples/ozone-ha/config-configmap.yaml | 12 ++++++------ .../k8s/examples/ozone/config-configmap.yaml | 16 ++++++++-------- 5 files changed, 38 insertions(+), 38 deletions(-) diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml index e32d18240766..4e503b7bd05c 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml @@ -19,22 +19,22 @@ kind: ConfigMap metadata: name: config data: - HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config CORE-SITE.XML_fs.defaultFS: ofs://om/ CORE-SITE.XML_fs.trash.interval: "1" - HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*' + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" OZONE-SITE.XML_hdds.datanode.dir: /data/storage - OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" + OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata - OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm OZONE-SITE.XML_ozone.om.address: om-0.om + OZONE-SITE.XML_ozone.recon.address: recon-0.recon + OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm + OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata OZONE-SITE.XML_ozone.scm.names: scm-0.scm - OZONE-SITE.XML_ozone.recon.address: recon-0.recon - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" - OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" - OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml index e32d18240766..4e503b7bd05c 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml @@ -19,22 +19,22 @@ kind: ConfigMap metadata: name: config data: - HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config CORE-SITE.XML_fs.defaultFS: ofs://om/ CORE-SITE.XML_fs.trash.interval: "1" - HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*' + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" OZONE-SITE.XML_hdds.datanode.dir: /data/storage - OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" + OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata - OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm OZONE-SITE.XML_ozone.om.address: om-0.om + OZONE-SITE.XML_ozone.recon.address: recon-0.recon + OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm + OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata OZONE-SITE.XML_ozone.scm.names: scm-0.scm - OZONE-SITE.XML_ozone.recon.address: recon-0.recon - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" - OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" - OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml index 0c02a5447c74..7a966957364d 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml @@ -19,22 +19,22 @@ kind: ConfigMap metadata: name: config data: - HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config CORE-SITE.XML_fs.defaultFS: ofs://om/ CORE-SITE.XML_fs.trash.interval: "1" - HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*' + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" OZONE-SITE.XML_hdds.datanode.dir: /data/storage - OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" + OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata - OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm OZONE-SITE.XML_ozone.om.address: om-0.om + OZONE-SITE.XML_ozone.recon.address: recon-0.recon + OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm + OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata OZONE-SITE.XML_ozone.scm.names: scm-0.scm - OZONE-SITE.XML_ozone.recon.address: recon-0.recon - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" - OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" - OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml index 4f3e37f383a6..a9315eb2103c 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml @@ -19,19 +19,19 @@ kind: ConfigMap metadata: name: config data: - HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config CORE-SITE.XML_fs.defaultFS: ofs://om/ CORE-SITE.XML_fs.trash.interval: "1" - HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*' + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" OZONE-SITE.XML_hdds.datanode.dir: /data/storage - OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" + OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.recon.address: recon-0.recon - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" - OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" - OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" + OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml index c1f688afd642..0825afc37b66 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml @@ -19,22 +19,22 @@ kind: ConfigMap metadata: name: config data: - HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config CORE-SITE.XML_fs.defaultFS: ofs://om/ CORE-SITE.XML_fs.trash.interval: "1" - HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*' + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" OZONE-SITE.XML_hdds.datanode.dir: /data/storage - OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" + OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata - OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm OZONE-SITE.XML_ozone.om.address: om-0.om + OZONE-SITE.XML_ozone.recon.address: recon-0.recon + OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm + OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata OZONE-SITE.XML_ozone.scm.names: scm-0.scm - OZONE-SITE.XML_ozone.recon.address: recon-0.recon - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" - OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" - OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout