[DO NOT MERGE] Backport some fixes from master to ozone-1.4

hadoop-ozone/dist/src/main/compose/compose_v2_compatibility.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# use Docker Compose v2 if available
+if docker compose version > /dev/null 2>&1; then
+  echo "Using Docker Compose v2"
+  docker-compose() {
+    docker compose --progress quiet "$@"
+  }
+fi

hadoop-ozone/dist/src/main/compose/ozone/prometheus.yml

@@ -14,8 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 global:
-  scrape_interval:     15s # By default, scrape targets every 15 seconds.
-
+  scrape_interval: 15s # By default, scrape targets every 15 seconds.
 scrape_configs:
   - job_name: ozone
     metrics_path: /prom
@@ -32,7 +31,9 @@ scrape_configs:
         - "s3g:9878"
        labels:
          component: s3g
-     - targets:
+     - targets: # During compose bring up the number of datanodes can be specific, adding 10 nodes to account for that.
+        # Duplicate datanodes are specified here to account for compose v1 vs v2 differences.
+        # compose v1
         - "ozone_datanode_1:9882"
         - "ozone_datanode_2:9882"
         - "ozone_datanode_3:9882"
@@ -43,5 +44,20 @@ scrape_configs:
         - "ozone_datanode_8:9882"
         - "ozone_datanode_9:9882"
         - "ozone_datanode_10:9882"
+        # compose v2
+        - "ozone-datanode-1:9882"
+        - "ozone-datanode-2:9882"
+        - "ozone-datanode-3:9882"
+        - "ozone-datanode-4:9882"
+        - "ozone-datanode-5:9882"
+        - "ozone-datanode-6:9882"
+        - "ozone-datanode-7:9882"
+        - "ozone-datanode-8:9882"
+        - "ozone-datanode-9:9882"
+        - "ozone-datanode-10:9882"
+       labels:
+         component: datanode
+     - targets:
+         - "recon:9888"
        labels:
-         component: datanode
+         component: recon

hadoop-ozone/dist/src/main/compose/ozone/run.sh

@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+source ../compose_v2_compatibility.sh
+
 declare -i OZONE_DATANODES OZONE_REPLICATION_FACTOR OZONE_SAFEMODE_MIN_DATANODES
 
 ORIG_DATANODES="${OZONE_DATANODES:-}"

hadoop-ozone/dist/src/main/compose/ozonescripts/README.md

@@ -12,7 +12,7 @@
   limitations under the License. See accompanying LICENSE file.
 -->
 
-# start-ozone environment
+# ozonescripts environment
 
 This is an example environment to use/test `./sbin/start-ozone.sh` and `./sbin/stop-ozone.sh` scripts.
 

hadoop-ozone/dist/src/main/compose/ozonescripts/ps.sh

@@ -14,4 +14,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-docker-compose ps -q | xargs -n1 -I CONTAINER docker exec CONTAINER ps xa
+
+source ../compose_v2_compatibility.sh
+
+docker-compose ps -q | xargs -n1 -I CONTAINER docker exec CONTAINER ps xa

hadoop-ozone/dist/src/main/compose/ozonescripts/start.sh

@@ -14,6 +14,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+source ../compose_v2_compatibility.sh
+
 set -x
 docker-compose ps | grep datanode | awk '{print $1}' | xargs -n1  docker inspect --format '{{ .Config.Hostname }}' > ../../etc/hadoop/workers
 docker-compose ps | grep ozonescripts | awk '{print $1}' | xargs -I CONTAINER -n1 docker exec CONTAINER cp /opt/hadoop/etc/hadoop/workers /etc/hadoop/workers

hadoop-ozone/dist/src/main/compose/ozonescripts/stop.sh

@@ -14,4 +14,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+source ../compose_v2_compatibility.sh
+
 docker-compose exec -T scm /opt/hadoop/sbin/stop-ozone.sh

hadoop-ozone/dist/src/main/compose/testlib.sh

@@ -28,6 +28,8 @@ if [[ -n "${OM_SERVICE_ID}" ]] && [[ "${OM_SERVICE_ID}" != "om" ]]; then
   OM_HA_PARAM="--om-service-id=${OM_SERVICE_ID}"
 fi
 
+source ${_testlib_dir}/compose_v2_compatibility.sh
+
 : ${SCM:=scm}
 
 ## @description create results directory, purging any prior data
@@ -142,11 +144,32 @@ start_docker_env(){
 
   trap stop_docker_env EXIT HUP INT TERM
 
-  docker-compose --ansi never up -d --scale datanode="${datanode_count}"
+  opts=""
+  if has_scalable_datanode; then
+    opts="--scale datanode=${datanode_count}"
+  fi
+
+  docker-compose --ansi never up -d $opts
+
   wait_for_safemode_exit
   wait_for_om_leader
 }
 
+has_scalable_datanode() {
+  local files="${COMPOSE_FILE:-docker-compose.yaml}"
+  local oifs=${IFS}
+  local rc=1
+  IFS=:
+  for f in ${files}; do
+    if [[ -e "${f}" ]] && grep -q '^\s*datanode:' ${f}; then
+      rc=0
+      break
+    fi
+  done
+  IFS=${oifs}
+  return ${rc}
+}
+
 ## @description  Execute robot tests in a specific container.
 ## @param        Name of the container in the docker-compose file
 ## @param        robot test file or directory relative to the smoketest dir
@@ -189,7 +212,7 @@ execute_robot_test(){
       "$SMOKETEST_DIR_INSIDE/$TEST"
   local -i rc=$?
 
-  FULL_CONTAINER_NAME=$(docker-compose ps | grep "_${CONTAINER}_" | head -n 1 | awk '{print $1}')
+  FULL_CONTAINER_NAME=$(docker-compose ps | grep "[-_]${CONTAINER}[-_]" | head -n 1 | awk '{print $1}')
   docker cp "$FULL_CONTAINER_NAME:$OUTPUT_PATH" "$RESULT_DIR/"
 
   if [[ ${rc} -gt 0 ]] && [[ ${rc} -le 250 ]]; then
@@ -228,7 +251,7 @@ create_stack_dumps() {
 ## @description Copy any 'out' files for daemon processes to the result dir
 copy_daemon_logs() {
   local c f
-  for c in $(docker-compose ps | grep "^${COMPOSE_ENV_NAME}_" | awk '{print $1}'); do
+  for c in $(docker-compose ps | grep "^${COMPOSE_ENV_NAME}[_-]" | awk '{print $1}'); do
     for f in $(docker exec "${c}" ls -1 /var/log/hadoop 2> /dev/null | grep -F -e '.out' -e audit); do
       docker cp "${c}:/var/log/hadoop/${f}" "$RESULT_DIR/"
     done

hadoop-ozone/dist/src/main/smoketest/recon/recon-api.robot

@@ -71,17 +71,17 @@ Check if Recon picks up OM data
 Check if Recon picks up DN heartbeats
     ${result} =         Execute                             curl --negotiate -u : -LSs ${API_ENDPOINT_URL}/datanodes
                         Should contain      ${result}       datanodes
-                        Should contain      ${result}       datanode_1
-                        Should contain      ${result}       datanode_2
-                        Should contain      ${result}       datanode_3
+                        Should Match Regexp    ${result}    datanode[-_]1
+                        Should Match Regexp    ${result}    datanode[-_]2
+                        Should Match Regexp    ${result}    datanode[-_]3
 
     ${result} =         Execute                             curl --negotiate -u : -LSs ${API_ENDPOINT_URL}/pipelines
                         Should contain      ${result}       pipelines
                         Should contain      ${result}       RATIS
                         Should contain      ${result}       OPEN
-                        Should contain      ${result}       datanode_1
-                        Should contain      ${result}       datanode_2
-                        Should contain      ${result}       datanode_3
+                        Should Match Regexp    ${result}    datanode[-_]1
+                        Should Match Regexp    ${result}    datanode[-_]2
+                        Should Match Regexp    ${result}    datanode[-_]3
 
     ${result} =         Execute                             curl --negotiate -u : -LSs ${API_ENDPOINT_URL}/clusterState
                         Should contain      ${result}       \"totalDatanodes\"

hadoop-ozone/dist/src/main/smoketest/topology/cli.robot

@@ -22,18 +22,29 @@ Test Timeout        5 minutes
 
 *** Variables ***
 
+*** Keywords ***
+Validate PrintTopology Output
+    [Arguments]           ${output}
+    Should Contain        ${output}         State = HEALTHY
+    Should Contain        ${output}         IN_SERVICE
+    Should Match Regexp   ${output}         .*ozone.*datanode[-_]\\d+.*IN_SERVICE.*rack.*
+
 
 *** Test Cases ***
 Run printTopology
     ${output} =         Execute          ozone admin printTopology
-                        Should contain   ${output}         10.5.0.7(ozone-topology_datanode_4_1.ozone-topology_net)    IN_SERVICE    /rack2
+                        Validate PrintTopology Output    ${output}
+
 Run printTopology -o
     ${output} =         Execute          ozone admin printTopology -o
-                        Should contain   ${output}         Location: /rack2
-                        Should contain   ${output}         10.5.0.7(ozone-topology_datanode_4_1.ozone-topology_net) IN_SERVICE
+                        Should Match Regexp   ${output}         Location: /.*rack.*
+                        Should Match Regexp   ${output}         .*ozone.*datanode[-_]\\d+.*IN_SERVICE.*
+
 Run printTopology --operational-state IN_SERVICE
     ${output} =         Execute          ozone admin printTopology --operational-state IN_SERVICE
-                        Should contain   ${output}         10.5.0.7(ozone-topology_datanode_4_1.ozone-topology_net)    IN_SERVICE    /rack2
+                        Validate PrintTopology Output    ${output}
+
 Run printTopology --node-state HEALTHY
     ${output} =         Execute          ozone admin printTopology --node-state HEALTHY
-                        Should contain   ${output}         10.5.0.7(ozone-topology_datanode_4_1.ozone-topology_net)    IN_SERVICE    /rack2
+                        Validate PrintTopology Output    ${output}
+

hadoop-ozone/recon/pom.xml

@@ -24,7 +24,7 @@
   <modelVersion>4.0.0</modelVersion>
   <artifactId>ozone-recon</artifactId>
   <properties>
-    <pnpm.version>7.33.6</pnpm.version>
+    <pnpm.version>8.15.7</pnpm.version>
     <allow.junit4>false</allow.junit4>
   </properties>
   <build>

hadoop-ozone/dist/src/main/smoketest/env-compose.robot → hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/.eslintignore

@@ -4,29 +4,23 @@
 # The ASF licenses this file to You under the Apache License, Version 2.0
 # (the "License"); you may not use this file except in compliance with
 # the License.  You may obtain a copy of the License at
-#
+
 #     http://www.apache.org/licenses/LICENSE-2.0
-#
+
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-*** Settings ***
-Documentation       High level utilities to execute commands and tests in docker-compose based environments.
-Resource            commonlib.robot
-Test Timeout        5 minutes
+# Ignore node modules and dist/build folders
+./node_modules
+./build
+./dist
 
-*** Keywords ***
+./api
 
-Run tests on host
-    [arguments]        ${host}       ${robotfile}
-    ${result} =        Execute       docker-compose exec ${host} robot smoketest/${robotfile}
 
-Execute on host
-    [arguments]                     ${host}     ${command}
-    ${rc}                           ${output} =                 Run And Return Rc And Output           docker-compose exec ${host} ${command}
-    Log                             ${output}
-    Should Be Equal As Integers     ${rc}                       0
-    [return]                        ${output}
+# Vite related configs
+./vite.config.ts
+./vite-env.d.ts

hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/.eslintrc.json

@@ -0,0 +1,109 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+* http://www.apache.org/licenses/LICENSE-2.0
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+{
+  "extends": [
+    "eslint:recommended",
+    "plugin:react/recommended",
+    "plugin:react/jsx-runtime",
+    "plugin:@typescript-eslint/recommended",
+    "plugin:import/typescript",
+    "prettier"
+  ],
+  "plugins": ["react", "@typescript-eslint", "prettier"],
+  "rules": {
+    "camelcase": "off",
+    "space-infix-ops": "warn",
+    "quotes": [
+      "warn",
+      "single",
+      {
+        "avoidEscape": true,
+        "allowTemplateLiterals": true
+      }
+    ],
+    "no-unused-vars": [
+      "warn",
+      {
+        "argsIgnorePattern": "^_\\w*",
+        "varsIgnorePattern": "^_\\w*"
+      }
+    ],
+    "object-curly-spacing": [
+      "warn",
+      "always"
+    ],
+    "object-property-newline": "warn",
+    "no-return-assign": "off",
+    "indent": [
+      "warn",
+      2,
+      {
+        "SwitchCase": 1
+      }
+    ],
+    "constructor-super": "warn",
+    "import/no-unassigned-import": "off",
+    "import/no-unused-modules": [
+      1,
+      {
+        "unusedExports": true
+      }
+    ],
+    "import/no-extraneous-dependencies": [
+      "error",
+      {
+        "devDependencies": true,
+        "optionalDependencies": true,
+        "peerDependencies": true
+      }
+    ],
+    "react/state-in-constructor": "off",
+    "react/require-default-props": "off",
+    "react/default-props-match-prop-types": "off",
+    "react/no-array-index-key": "off",
+    "promise/prefer-await-to-then": "warn",
+    "@typescript-eslint/explicit-function-return-type": "off",
+    "@typescript-eslint/prefer-readonly-parameter-types": "off",
+    "@typescript-eslint/no-unused-vars": [
+      "warn", {
+        "argsIgnorePattern": "^_\\w*",
+        "varsIgnorePattern": "^_\\w*"
+      }
+    ],
+    "@typescript-eslint/no-non-null-assertion": "off",
+    "@typescript-eslint/interface-name-prefix": ["warn", { "prefixWithI": "always" }]
+  },
+  "settings": {
+    "import/parsers": {
+      "@typescript-eslint/parser": [
+        ".ts",
+        ".tsx"
+      ]
+    },
+    "import/resolver": {
+      "typescript": {}
+    },
+    "react": {
+      "version": "16.8.6",
+      "pragma": "React",
+      "fragment": "Fragment"
+    }
+  },
+  "env": {
+      "browser": true,
+      "node": true
+  }
+}