docs(training-guides): add GPU slicing with Dynamic Resource Allocation (DRA)

docs/en/training_guides/assets/dra/dra-sft-trainingruntime.yaml

@@ -0,0 +1,118 @@
+# Kubeflow Trainer v2 TrainingRuntime that runs a LoRA supervised-fine-tune INSIDE
+# a DRA GPU slice. The only difference from an ordinary GPU runtime is how the GPU
+# is requested:
+#
+#   * the Pod declares `spec.resourceClaims`, pointing at the ResourceClaimTemplate
+#     (mig-1g-6gb) instead of asking for a whole-GPU device-plugin resource;
+#   * the trainer container binds to it with `resources.claims` and carries NO
+#     `nvidia.com/gpu` / `nvidia.com/gpualloc` limit at all.
+#
+# The training script is self-contained: it builds a small causal-LM, wraps it in a
+# LoRA adapter, and trains on a synthetic dataset. Nothing is downloaded, so the
+# run works on air-gapped GPU nodes and finishes in ~1 minute — enough to prove the
+# slice is real, isolated, and trainable. Swap in a real base model + dataset (see
+# the guide) to turn this into a production fine-tune; the DRA plumbing is unchanged.
+apiVersion: trainer.kubeflow.org/v1alpha1
+kind: TrainingRuntime
+metadata:
+  name: dra-mig-sft
+  namespace: kubeflow-admin-cpaas-io
+  labels:
+    trainer.kubeflow.org/framework: torch
+    alauda.io/training-runtime: dra-mig-sft
+spec:
+  mlPolicy:
+    numNodes: 1
+    torch:
+      numProcPerNode: 1            # one MIG slice == one trainer process (int, not "1")
+  template:
+    spec:
+      replicatedJobs:
+        - name: node
+          template:
+            metadata:
+              labels:
+                trainer.kubeflow.org/trainjob-ancestor-step: trainer
+            spec:
+              backoffLimit: 0
+              template:
+                spec:
+                  securityContext:
+                    runAsNonRoot: true
+                    runAsUser: 1000
+                    runAsGroup: 1000
+                    fsGroup: 1000
+                  # (1) Pod-level: bind a per-Pod ResourceClaim from the template.
+                  resourceClaims:
+                    - name: gpu-slice
+                      resourceClaimTemplateName: mig-1g-6gb
+                  containers:
+                    - name: node
+                      image: docker.io/alaudadockerhub/llamafactory0.9-cu126-amd64:v0.1.0
+                      command:
+                        - bash
+                        - -c
+                        - |
+                          set -ex
+                          python - <<'PY'
+                          import random, torch
+                          from transformers import (GPT2Config, GPT2LMHeadModel,
+                                                    Trainer, TrainingArguments)
+                          from peft import LoraConfig, get_peft_model
+                          from datasets import Dataset
+
+                          assert torch.cuda.is_available(), "no CUDA device in this DRA slice"
+                          p = torch.cuda.get_device_properties(0)
+                          print(f"[slice] device={p.name} total_mem={p.total_memory/1024**3:.2f} GiB", flush=True)
+
+                          # Small causal-LM, randomly initialised -> no model download.
+                          cfg = GPT2Config(vocab_size=512, n_positions=64,
+                                           n_embd=256, n_layer=4, n_head=4)
+                          model = GPT2LMHeadModel(cfg)
+                          model = get_peft_model(model, LoraConfig(
+                              r=8, lora_alpha=16, target_modules=["c_attn"],
+                              task_type="CAUSAL_LM"))
+                          model.print_trainable_parameters()
+
+                          # Synthetic SFT corpus -> no dataset download.
+                          random.seed(0)
+                          rows = [{"input_ids": (s := [random.randrange(512) for _ in range(64)]),
+                                   "labels": s} for _ in range(128)]
+                          ds = Dataset.from_list(rows)
+
+                          args = TrainingArguments(
+                              output_dir="/workspace/out", per_device_train_batch_size=8,
+                              max_steps=30, logging_steps=10, fp16=True,
+                              save_strategy="no", report_to=[])
+                          Trainer(model=model, args=args, train_dataset=ds).train()
+
+                          peak = torch.cuda.max_memory_allocated() / 1024**3
+                          print(f"[slice] peak_mem={peak:.2f} GiB", flush=True)
+                          print("DRA LoRA SFT on GPU slice: OK", flush=True)
+                          PY
+                      resources:
+                        # (2) Container-level: consume the pod claim. No nvidia.com/*.
+                        claims:
+                          - name: gpu-slice
+                        requests:
+                          cpu: "1"
+                          memory: 4Gi
+                        limits:
+                          cpu: "4"
+                          memory: 8Gi
+                      securityContext:
+                        allowPrivilegeEscalation: false
+                        capabilities:
+                          drop: [ALL]
+                        runAsNonRoot: true
+                        seccompProfile:
+                          type: RuntimeDefault
+                      volumeMounts:
+                        - { name: workspace, mountPath: /workspace }
+                        - { name: dshm, mountPath: /dev/shm }
+                  volumes:
+                    - { name: workspace, emptyDir: {} }
+                    - name: dshm
+                      emptyDir:
+                        medium: Memory
+                        sizeLimit: 2Gi

docs/en/training_guides/assets/dra/dra-sft-trainjob.yaml

@@ -0,0 +1,17 @@
+# A TrainJob that runs the DRA-sliced LoRA SFT. It only references the runtime —
+# the DRA slice request lives in the TrainingRuntime's Pod spec, so day-to-day
+# submissions stay this small. Add `labels: { kueue.x-k8s.io/queue-name: ... }` to
+# gate it behind a Kueue LocalQueue if you also quota GPU slices.
+apiVersion: trainer.kubeflow.org/v1alpha1
+kind: TrainJob
+metadata:
+  generateName: dra-sft-
+  namespace: kubeflow-admin-cpaas-io
+spec:
+  runtimeRef:
+    apiGroup: trainer.kubeflow.org
+    kind: TrainingRuntime
+    name: dra-mig-sft
+  suspend: false
+  trainer:
+    numNodes: 1

docs/en/training_guides/assets/dra/dra-smoke-pod.yaml

@@ -0,0 +1,60 @@
+# Minimal DRA smoke: a single Pod that claims one GPU slice and prints what the
+# container can actually see. Run this before wiring DRA into a training job — it
+# proves the driver partitions and injects a slice, and shows you the slice's real
+# memory ceiling (e.g. ~6 GiB for a 1g.6gb MIG profile rather than the full card).
+#
+# The Pod references the ResourceClaimTemplate by name; the scheduler + DRA driver
+# allocate a device and inject it before the container starts. Point
+# `resourceClaimTemplateName` at either template:
+#   mig-1g-6gb            -> a hardware-isolated MIG slice
+#   shared-gpu-timeslice  -> a shared, time-sliced full GPU
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: dra-smoke-
+  namespace: kubeflow-admin-cpaas-io
+  labels:
+    app: dra-smoke
+spec:
+  restartPolicy: Never
+  resourceClaims:
+    - name: gpu-slice
+      resourceClaimTemplateName: mig-1g-6gb
+  containers:
+    - name: smoke
+      image: docker.io/alaudadockerhub/llamafactory0.9-cu126-amd64:v0.1.0
+      command:
+        - bash
+        - -c
+        - |
+          set -e
+          echo "===== nvidia-smi -L ====="
+          nvidia-smi -L
+          echo "===== nvidia-smi ====="
+          nvidia-smi
+          python - <<'PY'
+          import torch
+          ok = torch.cuda.is_available()
+          print("cuda_available:", ok)
+          if ok:
+              p = torch.cuda.get_device_properties(0)
+              print("device:", p.name)
+              print("slice_total_mem_GiB:", round(p.total_memory / 1024**3, 2))
+              x = torch.randn(2048, 2048, device="cuda"); (x @ x.T).sum().item()
+              print("matmul_on_slice: OK")
+          PY
+      resources:
+        claims:
+          - name: gpu-slice     # binds this container to the pod-level claim above
+        requests:
+          cpu: "500m"
+          memory: 2Gi
+        limits:
+          cpu: "2"
+          memory: 4Gi
+      securityContext:
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop: [ALL]
+        seccompProfile:
+          type: RuntimeDefault

docs/en/training_guides/assets/dra/mig-slice-resourceclaimtemplate.yaml

@@ -0,0 +1,37 @@
+# A ResourceClaimTemplate that asks the NVIDIA DRA driver for ONE MIG slice of a
+# GPU. Every Pod that references this template gets its own freshly partitioned
+# MIG device: a hardware-isolated slice with a fixed share of the GPU's compute
+# engines (SMs) and a fixed slice of VRAM. Two Pods that each claim `1g.6gb`
+# cannot touch each other's memory — this is the real "slice of a GPU".
+#
+# Requires:
+#   - the GPU is in MIG mode, and
+#   - the NVIDIA DRA driver (k8s-dra-driver-gpu) is advertising `mig.nvidia.com`
+#     devices in a ResourceSlice on the node (see the guide's Prerequisites).
+#
+# Size the slice by editing the profile in the CEL selector. On an A30 (24 GB):
+#   1g.6gb   -> up to 4 slices per physical GPU   (this file)
+#   2g.12gb  -> up to 2 slices per physical GPU
+#   4g.24gb  -> 1 slice (the whole card exposed as a single MIG device)
+# A100/H100 (80 GB) expose 1g.10gb / 2g.20gb / 3g.40gb / 7g.80gb and friends.
+#
+# The exact attribute key ("profile") and its values come from the driver. Always
+# confirm against your cluster:
+#   kubectl get resourceslices -o yaml | less   # look under spec.devices[].basic.attributes
+apiVersion: resource.k8s.io/v1
+kind: ResourceClaimTemplate
+metadata:
+  name: mig-1g-6gb
+  namespace: kubeflow-admin-cpaas-io   # must match the namespace where the Pods run
+spec:
+  spec:
+    devices:
+      requests:
+        - name: gpu-slice
+          exactly:
+            deviceClassName: mig.nvidia.com
+            allocationMode: ExactCount
+            count: 1
+            selectors:
+              - cel:
+                  expression: device.attributes["gpu.nvidia.com"].profile == "1g.6gb"

docs/en/training_guides/assets/dra/shared-gpu-resourceclaimtemplate.yaml

@@ -0,0 +1,37 @@
+# Alternative slicing strategy for GPUs that are NOT in MIG mode (older cards, or
+# a GPU you don't want to partition): request a FULL GPU but tell the NVIDIA DRA
+# driver to *share* it across claims by time-slicing. Multiple Pods that each
+# reference this template are placed on the same physical GPU and interleave in
+# time.
+#
+# Trade-off vs. MIG: time-slicing gives you scheduling density with NO memory or
+# fault isolation — every consumer sees the full VRAM and the whole card, and one
+# workload can OOM the others. Use MIG (mig-slice-resourceclaimtemplate.yaml) when
+# you need a guaranteed, isolated slice; use this when you only need to pack
+# several light jobs onto one card.
+#
+# `strategy: MPS` is also supported by the driver for concurrent (rather than
+# round-robin) sharing; swap TimeSlicing -> MPS if your workloads benefit from it.
+apiVersion: resource.k8s.io/v1
+kind: ResourceClaimTemplate
+metadata:
+  name: shared-gpu-timeslice
+  namespace: kubeflow-admin-cpaas-io   # must match the namespace where the Pods run
+spec:
+  spec:
+    devices:
+      requests:
+        - name: gpu-slice
+          exactly:
+            deviceClassName: gpu.nvidia.com
+            allocationMode: ExactCount
+            count: 1
+      config:
+        - requests: ["gpu-slice"]
+          opaque:
+            driver: gpu.nvidia.com
+            parameters:
+              apiVersion: resource.nvidia.com/v1beta1
+              kind: GpuConfig
+              sharing:
+                strategy: TimeSlicing