From 07158b34c8d5b9b320fd8873fb4ed0a032c420ea Mon Sep 17 00:00:00 2001 From: HalimKim Date: Tue, 26 Nov 2024 11:26:23 +0900 Subject: [PATCH 1/3] HDDS-11806. Add HttpFS in the k8s example files --- .../getting-started/config-configmap.yaml | 5 ++ .../httpfs-public-service.yaml | 28 ++++++++++ .../getting-started/httpfs-service.yaml | 28 ++++++++++ .../getting-started/httpfs-statefulset.yaml | 55 +++++++++++++++++++ .../getting-started/kustomization.yaml | 3 + 5 files changed, 119 insertions(+) create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/getting-started/httpfs-public-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/getting-started/httpfs-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/getting-started/httpfs-statefulset.yaml diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml index b3acc6f1d221..d13b5a737c32 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml @@ -19,6 +19,11 @@ kind: ConfigMap metadata: name: config data: + HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config + CORE-SITE.XML_fs.defaultFS: ofs://om/ + CORE-SITE.XML_fs.trash.interval: "1" + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: "*" + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: "*" OZONE-SITE.XML_hdds.datanode.dir: /data/storage OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/httpfs-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/httpfs-public-service.yaml new file mode 100644 index 000000000000..d8586250553a --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/httpfs-public-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: httpfs-public +spec: + ports: + - port: 14000 + name: rest + selector: + app: ozone + component: httpfs + type: NodePort diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/httpfs-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/httpfs-service.yaml new file mode 100644 index 000000000000..0ab49c2d72e4 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/httpfs-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: httpfs +spec: + ports: + - port: 14000 + name: rest + clusterIP: None + selector: + app: ozone + component: httpfs diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/httpfs-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/httpfs-statefulset.yaml new file mode 100644 index 000000000000..7bca21585c13 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/httpfs-statefulset.yaml @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: httpfs + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: httpfs + serviceName: httpfs + replicas: 1 + template: + metadata: + labels: + app: ozone + component: httpfs + spec: + containers: + - name: httpfs + image: '@docker.image@' + args: + - ozone + - httpfs + livenessProbe: + httpGet: + path: /webhdfs/v1/?op=LISTSTATUS&user.name=hadoop + port: 14000 + initialDelaySeconds: 30 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/kustomization.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/kustomization.yaml index 3059b9c801e2..e062536c1012 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/kustomization.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/kustomization.yaml @@ -24,7 +24,10 @@ resources: - s3g-statefulset.yaml - scm-service.yaml - scm-statefulset.yaml +- httpfs-service.yaml +- httpfs-statefulset.yaml - datanode-public-service.yaml - om-public-service.yaml - s3g-public-service.yaml - scm-public-service.yaml +- httpfs-public-service.yaml From 5a910b90d1cc5e82778fbfb28aa65b264075cc26 Mon Sep 17 00:00:00 2001 From: HalimKim Date: Tue, 26 Nov 2024 22:19:25 +0900 Subject: [PATCH 2/3] HDDS-11806. Add Recon in the k8s example files --- .../getting-started/config-configmap.yaml | 1 + .../getting-started/kustomization.yaml | 3 + .../getting-started/recon-public-service.yaml | 28 +++++++++ .../getting-started/recon-service.yaml | 28 +++++++++ .../getting-started/recon-statefulset.yaml | 63 +++++++++++++++++++ 5 files changed, 123 insertions(+) create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/getting-started/recon-public-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/getting-started/recon-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/getting-started/recon-statefulset.yaml diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml index d13b5a737c32..ce7dafaf0c1a 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml @@ -31,6 +31,7 @@ data: OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm OZONE-SITE.XML_ozone.scm.names: scm-0.scm + OZONE-SITE.XML_ozone.recon.address: recon-0.recon OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/kustomization.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/kustomization.yaml index e062536c1012..4f60be17872b 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/kustomization.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/kustomization.yaml @@ -26,8 +26,11 @@ resources: - scm-statefulset.yaml - httpfs-service.yaml - httpfs-statefulset.yaml +- recon-service.yaml +- recon-statefulset.yaml - datanode-public-service.yaml - om-public-service.yaml - s3g-public-service.yaml - scm-public-service.yaml - httpfs-public-service.yaml +- recon-public-service.yaml diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/recon-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/recon-public-service.yaml new file mode 100644 index 000000000000..c737a02f446f --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/recon-public-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: recon-public +spec: + ports: + - port: 9888 + name: ui + selector: + app: ozone + component: recon + type: NodePort diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/recon-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/recon-service.yaml new file mode 100644 index 000000000000..9c52d393d55d --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/recon-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: recon +spec: + ports: + - port: 9888 + name: ui + clusterIP: None + selector: + app: ozone + component: recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/recon-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/recon-statefulset.yaml new file mode 100644 index 000000000000..0ded5dc148ef --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/recon-statefulset.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: recon + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: recon + serviceName: recon + replicas: 1 + template: + metadata: + labels: + app: ozone + component: recon + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9888" + prometheus.io/path: /prom + spec: + securityContext: + fsGroup: 1000 + containers: + - name: recon + image: 'apache/ozone:1.4.0' + args: + - ozone + - recon + env: + - name: WAITFOR + value: scm-0.scm:9876 + livenessProbe: + tcpSocket: + port: 9891 + initialDelaySeconds: 30 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + emptyDir: {} From 7ec6a82f03e8ed641c64b0a8e47faa66a0d8e2af Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Tue, 3 Dec 2024 08:08:25 +0100 Subject: [PATCH 3/3] Replace hard-coded image --- .../main/k8s/examples/getting-started/recon-statefulset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/recon-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/recon-statefulset.yaml index 0ded5dc148ef..8b9ee191d035 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/recon-statefulset.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/recon-statefulset.yaml @@ -41,7 +41,7 @@ spec: fsGroup: 1000 containers: - name: recon - image: 'apache/ozone:1.4.0' + image: '@docker.image@' args: - ozone - recon