Update the scripts and dockers in madengine package

src/madengine/scripts/common/pre_scripts/rocEnvTool/env_tags.json

@@ -0,0 +1,32 @@
+{
+    "env_tags": [
+        "os_information",
+        "cpu_information",
+        "gpu_information",
+        "rocm_smi_gpudeviceid",
+        "memory_information",
+        "rocm_information",
+        "rocm_repo_setup",
+        "rocm_packages_installed",
+        "rocm_env_variables",
+        "rocm_smi",
+        "ifwi_version",
+        "rocm_smi_showhw",
+        "rocm_smi_pcie",
+        "rocm_smi_pids",
+        "rocm_smi_topology",
+        "rocm_smi_showserial",
+        "rocm_smi_showperflevel",
+        "rocm_smi_showrasinfo",
+        "rocm_smi_showxgmierr",
+        "rocm_smi_clocks",
+        "rocm_smi_showcompute_partition",
+        "rocm_smi_nodesbwi",
+        "rocm_info",
+        "cuda_information",
+        "cuda_env_variables",
+        "cuda_packages_installed",
+        "pip_list",
+        "numa_balancing"
+    ]
+}

src/madengine/scripts/common/tools.json

@@ -0,0 +1,113 @@
+{
+  "tools": {
+    "rpd": {
+      "pre_scripts": [
+        {
+          "path": "scripts/common/pre_scripts/trace.sh",
+          "args": "rpd"
+        }
+      ],
+      "cmd": "runTracer.sh",
+      "env_vars": {},
+      "post_scripts": [
+        {
+          "path": "scripts/common/post_scripts/trace.sh",
+          "args": "rpd"
+        }
+      ]
+    },
+    "rocprof": {
+      "pre_scripts": [],
+      "cmd": "rocprof",
+      "env_vars": {},
+      "post_scripts": [
+        {
+          "path": "scripts/common/post_scripts/trace.sh",
+          "args": "rocprof"
+        }
+      ]
+    },
+    "rocblas_trace": {
+      "env_vars": {"ROCBLAS_TRACE": "1"},
+      "cmd": "python3 ../scripts/common/tools/get_library_trace.py"
+    },
+    "hipblaslt_trace": {
+      "env_vars": {"HIPBLASLT_TRACE": "1"},
+      "cmd": "python3 ../scripts/common/tools/get_library_trace.py"
+    },    
+    "miopen_trace": {
+      "env_vars": {"MIOPEN_TRACE": "1"},
+      "cmd": "python3 ../scripts/common/tools/get_library_trace.py"
+    },
+    "tensile_trace": {
+      "env_vars": {"TENSILE_TRACE": "1"},
+      "cmd": "python3 ../scripts/common/tools/get_library_trace.py"
+    },
+    "rccl_trace": {
+      "env_vars": {"RCCL_TRACE": "1"},
+      "cmd": "python3 ../scripts/common/tools/get_library_trace.py"
+    },
+    "test_tools_A": {
+      "pre_scripts": [
+        {
+          "path": "scripts/common/test_echo.sh",
+          "args": "pre_script A"
+        }
+      ],
+      "cmd": "echo cmd_A && ",
+      "env_vars": {},
+      "post_scripts": [
+        {
+          "path": "scripts/common/test_echo.sh",
+          "args": "post_script A"
+        }
+      ]
+    },
+    "test_tools_B": {
+      "pre_scripts": [
+        {
+          "path": "scripts/common/test_echo.sh",
+          "args": "pre_script B"
+        }
+      ],
+      "cmd": "echo cmd_B && ",
+      "env_vars": {},
+      "post_scripts": [
+        {
+          "path": "scripts/common/test_echo.sh",
+          "args": "post_script B"
+        }
+      ]
+    },
+    "gpu_info_power_profiler": {
+      "pre_scripts": [
+        {
+          "path": "scripts/common/pre_scripts/gpu_info_pre.sh"
+        }
+      ],
+      "cmd": "python3 ../scripts/common/tools/gpu_info_profiler.py",
+      "env_vars": {"DEVICE":"all", "SAMPLING_RATE":"0.1", "MODE":"power", "DUAL-GCD":"false"},
+      "post_scripts": [
+        {
+          "path": "scripts/common/post_scripts/gpu_info_post.sh",
+          "args": "gpu_info_power_profiler"
+        }
+      ]
+    },
+    "gpu_info_vram_profiler": {
+      "pre_scripts": [
+        {
+          "path": "scripts/common/pre_scripts/gpu_info_pre.sh"
+        }
+      ],
+      "cmd": "python3 ../scripts/common/tools/gpu_info_profiler.py",
+      "env_vars": {"DEVICE":"all", "SAMPLING_RATE":"0.1", "MODE":"vram", "DUAL-GCD":"false"},
+      "post_scripts": [
+        {
+          "path": "scripts/common/post_scripts/gpu_info_post.sh",
+          "args": "gpu_info_vram_profiler"
+        }
+      ]
+    }
+  }
+}

tests/fixtures/dummy/credential.json

@@ -0,0 +1,21 @@
+{
+    "NAS_NODES": [
+        {
+            "NAME": "default",
+            "HOST": "localhost",
+            "PORT": "22",
+            "USERNAME": "admin",
+            "PASSWORD": "admin"
+        }
+    ],
+    "MAD_AWS_S3": {
+        "USERNAME": "admin",
+        "PASSWORD": "admin"
+    },
+    "MAD_MINIO": {
+        "USERNAME": "admin-access-key",
+        "PASSWORD": "admin-secret-key",
+        "MINIO_ENDPOINT": "http://127.0.1:9000",
+        "AWS_ENDPOINT_URL_S3": "http://127.0.1:9000"
+    }    
+}

tests/fixtures/dummy/data.json

@@ -0,0 +1,12 @@
+{
+    "dummy_data_local": {
+        "local": {
+            "path": "/tmp"
+        }
+    },
+    "dummy_data_local_fail": {
+        "local": {
+            "path": "/tmp/nonexistent"
+        }
+    }
+}

tests/fixtures/dummy/docker/dummy.ubuntu.amd.Dockerfile

@@ -0,0 +1,3 @@
+# CONTEXT {'gpu_vendor': 'AMD', 'guest_os': 'UBUNTU'}
+ARG BASE_DOCKER=rocm/pytorch
+FROM $BASE_DOCKER

tests/fixtures/dummy/docker/dummy.ubuntu.nvidia.Dockerfile

@@ -0,0 +1,38 @@
+# CONTEXT {'gpu_vendor': 'NVIDIA', 'guest_os': 'UBUNTU'}
+ARG BASE_DOCKER=nvidia/cuda:12.1.1-cudnn8-devel-ubuntu20.04
+FROM $BASE_DOCKER
+USER root
+ENV WORKSPACE_DIR=/workspace
+RUN mkdir -p $WORKSPACE_DIR
+WORKDIR $WORKSPACE_DIR
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt update && apt install -y \
+    unzip \
+    jq \
+    python3-pip \
+    git \
+    vim \
+    wget \
+    openmpi-bin libopenmpi-dev
+
+ENV PATH="/root/miniconda3/bin:${PATH}"
+ARG PATH="/root/miniconda3/bin:${PATH}"
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    mkdir /root/.conda && \
+    bash Miniconda3-latest-Linux-x86_64.sh -b && \
+    rm -rf Miniconda3-latest-Linux-x86_64.sh
+
+RUN conda --version && \
+    conda init
+RUN pip install --upgrade pip
+RUN pip install typing-extensions
+RUN pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu121
+
+RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - |  tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null && \
+    echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ bionic main' |  tee /etc/apt/sources.list.d/kitware.list >/dev/null && \
+    apt-get update && \
+    apt-get install -y cmake
+
+# record configuration for posterity
+RUN pip list

tests/fixtures/dummy/docker/dummy_ctxtest.ctx1.ubuntu.amd.Dockerfile

@@ -0,0 +1,5 @@
+# CONTEXT {'ctx_test': '1'}
+ARG BASE_DOCKER=rocm/pytorch
+FROM $BASE_DOCKER
+
+ENV ctxtest=1

tests/fixtures/dummy/docker/dummy_ctxtest.ctx2a.ubuntu.amd.Dockerfile

@@ -0,0 +1,5 @@
+# CONTEXT {'ctx_test': '2'}
+ARG BASE_DOCKER=rocm/pytorch
+FROM $BASE_DOCKER
+
+ENV ctxtest=2

tests/fixtures/dummy/docker/dummy_ctxtest.ctx2b.ubuntu.amd.Dockerfile

@@ -0,0 +1,5 @@
+# CONTEXT {'ctx_test': '2'}
+ARG BASE_DOCKER=rocm/pytorch
+FROM $BASE_DOCKER
+
+ENV ctxtest=2

tests/fixtures/dummy/docker/dummy_ctxtest.noctx.ubuntu.amd.Dockerfile

@@ -0,0 +1,5 @@
+# CONTEXT {'ctx_test': 'None'}
+ARG BASE_DOCKER=rocm/pytorch
+FROM $BASE_DOCKER
+
+ENV ctxtest=0