Add docs and example manifest on how to use NVIDIA DRA with DRANet

examples/demo_nvidia_dranet/README.md

@@ -0,0 +1,3 @@
+# GKE with NVIDIA DRA and DRANET
+
+This holds the YAMLs for the example given in /dranet/site/content/docs/user/nvidia-dranet.md

examples/demo_nvidia_dranet/deviceclass.yaml

@@ -0,0 +1,22 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: resource.k8s.io/v1beta1
+kind: DeviceClass
+metadata:
+  name: dranet
+spec:
+  selectors:
+    - cel:
+        expression: device.driver == "dra.net"

examples/demo_nvidia_dranet/resourceclaims.yaml

@@ -0,0 +1,50 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: resource.k8s.io/v1beta1
+kind: ResourceClaimTemplate
+metadata:
+  name: 2-gpu
+spec:
+  spec:
+    devices:
+      requests:
+      - name: gpu
+        deviceClassName: gpu.nvidia.com
+        count: 2
+        selectors:
+        - cel:
+            expression: |
+                device.attributes["gpu.nvidia.com"].index < 2
+
+---
+apiVersion: resource.k8s.io/v1beta1
+kind: ResourceClaimTemplate
+metadata:
+  name: 2-nic
+spec:
+  spec:
+    devices:
+      requests:
+      - name: nic
+        deviceClassName: dranet
+        count: 2
+        selectors:
+        - cel:
+            expression: device.attributes["dra.net"].rdma == true &&
+              (
+                (device.attributes["dra.net"].ifName.startsWith("gpu") &&
+                 device.attributes["dra.net"].ifName.endsWith("rdma0") &&
+                 int(device.attributes["dra.net"].ifName.substring(3, 4)) < 2)
+              )

examples/demo_nvidia_dranet/statefulset.yaml

@@ -0,0 +1,81 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: nccl-gib-test
+  labels:
+    name: nccl-gib-test
+spec:
+  replicas: 2
+  serviceName: nccl-gib-test
+  selector:
+    matchLabels:
+      name: nccl-gib-test
+  template:
+    metadata:
+      labels:
+        name: nccl-gib-test
+    spec:
+      containers:
+      - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic:v1.0.6
+        name: test
+        securityContext:
+          capabilities:
+            add: ["IPC_LOCK"]
+        volumeMounts:
+          # - name: library-dir-host
+          #   mountPath: /usr/local/nvidia
+          - name: gib
+            mountPath: /usr/local/gib
+          - name: shared-memory
+            mountPath: /dev/shm
+        env:
+          - name: LD_LIBRARY_PATH
+            value: /usr/local/nvidia/lib64
+        command: ["/bin/bash", "-c"]
+        args:
+          - |
+            # we use a headless service to identify the workers that has the format <hostname>.<service>.<ns>.svc.<zone>
+            # hence we need to allow to resolve fqdn 
+            nvidia-smi -L
+            echo -e "\norte_keep_fqdn_hostnames=t" >> /etc/openmpi/openmpi-mca-params.conf
+            /scripts/container_entry.sh shell
+            source /usr/local/gib/scripts/set_nccl_env.sh
+            sleep infinity
+        resources:
+          claims:
+          - name: gpu            
+      volumes:
+        - name: library-dir-host
+          hostPath:
+            path: /home/kubernetes/bin/nvidia
+        - name: gib
+          hostPath:
+            path: /home/kubernetes/bin/gib
+        - name: shared-memory
+          emptyDir:
+            medium: "Memory"
+            sizeLimit: 250Gi
+      resourceClaims:
+        - name: gpu
+          resourceClaimTemplateName: 2-gpu
+        - name: nic
+          resourceClaimTemplateName: 2-nic
+      tolerations:
+      - key: "nvidia.com/gpu"
+        operator: "Equal"
+        value: "present"
+        effect: "NoSchedule"