env:
  VLLM_WORKER_MULTIPROC_METHOD: spawn
  VLLM_USE_FLASHINFER_MOE_FP16: 0
  HF_HUB_DOWNLOAD_TIMEOUT: 300
  HF_HUB_ETAG_TIMEOUT: 60

steps:
  # Group: collapses under one heading in the Buildkite UI; child steps still run in parallel.
  - group: ":card_index_dividers: Omni Model Test"
    key: nightly-omni-test-group
    depends_on: upload-nightly-pipeline
    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
    steps:
      - label: ":full_moon: Omni · Function Test with H100"
        timeout_in_minutes: 90
        commands:
          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Omni · Function Test with L4"
        timeout_in_minutes: 90
        commands:
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
        agents:
          queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
        plugins:
          - docker#v5.2.0:
              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
              always-pull: true
              propagate-environment: true
              shm-size: "8gb"
              environment:
                - "HF_HOME=/fsx/hf_cache"
                - "HF_TOKEN"
              volumes:
                - "/fsx/hf_cache:/fsx/hf_cache"

      - label: ":full_moon: Omni · Doc Test with L4"
        timeout_in_minutes: 90
        commands:
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - pytest -s -v tests/examples/ -m "full_model and omni and L4" --run-level "full_model"
        agents:
          queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
        plugins:
          - docker#v5.2.0:
              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
              always-pull: true
              propagate-environment: true
              shm-size: "8gb"
              environment:
                - "HF_HOME=/fsx/hf_cache"
                - "HF_TOKEN"
              volumes:
                - "/fsx/hf_cache:/fsx/hf_cache"

      - label: ":full_moon: Omni · Doc Test with H100"
        timeout_in_minutes: 90
        commands:
          - pytest -s -v tests/examples/ -m "full_model and omni and H100" --run-level "full_model"
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Omni · Accuracy Test"
        timeout_in_minutes: 180
        commands:
          - export SEED_TTS_WER_EVAL=1
          - export SEED_TTS_EVAL_DEVICE=cuda:1
          - |
            set +e
            pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model" --run-level full_model
            EXIT=$$?
            buildkite-agent artifact upload "tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json"
            exit $$EXIT
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Omni · Perf Test · No Async Chunk"
        key: nightly-omni-performance-no-async-chunk
        timeout_in_minutes: 180
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
            exit $$EXIT
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Omni · Perf Test · Async Chunk"
        key: nightly-omni-performance-async-chunk
        timeout_in_minutes: 180
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
            exit $$EXIT
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Omni · Perf Test · vLLM Text"
        key: nightly-omni-performance-vllm-text
        timeout_in_minutes: 180
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
            exit $$EXIT
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Omni · Perf Test · Multi-Replica"
        key: nightly-omni-performance-multi-replicas
        timeout_in_minutes: 180
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
            exit $$EXIT
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 3
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate


      - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100"
        timeout_in_minutes: 45
        commands:
          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model" --run-level "core_model"
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 4
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

  - group: ":card_index_dividers: TTS Model Test"
    key: nightly-tts-test-group
    depends_on: upload-nightly-pipeline
    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
    steps:
      - label: ":full_moon: TTS · Function Test with L4"
        timeout_in_minutes: 90
        commands:
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_voxcpm2_tts_expansion.py
        agents:
          queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
        plugins:
          - docker#v5.2.0:
              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
              always-pull: true
              propagate-environment: true
              shm-size: "8gb"
              environment:
                - "HF_HOME=/fsx/hf_cache"
                - "HF_TOKEN"
              volumes:
                - "/fsx/hf_cache:/fsx/hf_cache"

      # - label: ":full_moon: TTS · Function Test with H100"
      #   timeout_in_minutes: 90
      #   commands:
      #     - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
      #     - pytest -s -v tests/e2e/ -m "full_model and H100 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
      #   agents:
      #     queue: "mithril-h100-pool"
      #   plugins:
      #     - kubernetes:
      #         podSpec:
      #           containers:
      #             - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
      #               resources:
      #                 limits:
      #                   nvidia.com/gpu: 1
      #               volumeMounts:
      #                 - name: devshm
      #                   mountPath: /dev/shm
      #                 - name: hf-cache
      #                   mountPath: /root/.cache/huggingface
      #               env:
      #                 - name: HF_HOME
      #                   value: /root/.cache/huggingface
      #                 - name: HF_TOKEN
      #                   valueFrom:
      #                     secretKeyRef:
      #                       name: hf-token-secret
      #                       key: token
      #           nodeSelector:
      #             node.kubernetes.io/instance-type: gpu-h100-sxm
      #           volumes:
      #             - name: devshm
      #               emptyDir:
      #                 medium: Memory
      #             - name: hf-cache
      #               hostPath:
      #                 path: /mnt/hf-cache
      #                 type: DirectoryOrCreate

      - label: ":full_moon: TTS · Accuracy Test · VoxCPM2"
        timeout_in_minutes: 30
        commands:
          - |
            timeout 30m bash -c "
              export VLLM_LOGGING_LEVEL=DEBUG
              export VLLM_WORKER_MULTIPROC_METHOD=spawn
              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
              pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts_expansion.py -m 'full_model and cuda and tts' --run-level 'full_model'
            "
        agents:
          queue: "gpu_1_queue"
        plugins:
          - docker#v5.2.0:
              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
              always-pull: true
              propagate-environment: true
              shm-size: "8gb"
              environment:
                - "HF_HOME=/fsx/hf_cache"
                - "HF_TOKEN"
              volumes:
                - "/fsx/hf_cache:/fsx/hf_cache"

      - label: ":full_moon: TTS · Perf Test"
        key: nightly-tts-performance
        timeout_in_minutes: 180
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
            exit $$EXIT
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: TTS · VoxCPM2 · Perf Test"
        key: nightly-tts-performance-voxcpm2
        timeout_in_minutes: 180
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_voxcpm2.json
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
            exit $$EXIT
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

  # Diffusion X2I suite: x2i / x2a / x2t and related non-video paths; x2v is only in "Diffusion X2V Model Test" below.
  - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
    key: nightly-diffusion-x2iat-group
    depends_on: upload-nightly-pipeline
    if: >-
      build.env("NIGHTLY") == "1" ||
      build.pull_request.labels includes "nightly-test" ||
      build.pull_request.labels includes "diffusion-x2iat-test"
    steps:
      # Use explicit file shards so single-card cases do not reserve 2+ GPUs,
      # and Bagel feature coverage is not duplicated via a broad tests/e2e sweep.
      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU"
        timeout_in_minutes: 120
        commands:
          - >-
            pytest -sv
            tests/e2e/online_serving/test_qwen_image_expansion.py
            tests/e2e/online_serving/test_qwen_image_edit_expansion.py
            tests/e2e/online_serving/test_qwen_image_layered_expansion.py
            tests/e2e/online_serving/test_longcat_image_expansion.py
            tests/e2e/online_serving/test_longcat_image_edit_expansion.py
            tests/e2e/online_serving/test_flux_2_dev_expansion.py
            tests/e2e/online_serving/test_bagel_expansion.py
            -m "full_model and diffusion and H100 and not distributed_cuda"
            --run-level "full_model"
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Qwen-Image"
        timeout_in_minutes: 120
        commands:
          - >-
            pytest -sv
            tests/e2e/online_serving/test_qwen_image_expansion.py
            tests/e2e/online_serving/test_qwen_image_edit_expansion.py
            -m "full_model and diffusion and H100 and distributed_cuda"
            --run-level "full_model"
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Layered/LongCat/FLUX"
        timeout_in_minutes: 90
        commands:
          - >-
            pytest -sv
            tests/e2e/online_serving/test_qwen_image_layered_expansion.py
            tests/e2e/online_serving/test_longcat_image_expansion.py
            tests/e2e/online_serving/test_longcat_image_edit_expansion.py
            tests/e2e/online_serving/test_flux_2_dev_expansion.py
            tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py
            -m "full_model and diffusion and H100 and distributed_cuda"
            --run-level "full_model"
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Bagel"
        timeout_in_minutes: 120
        commands:
          - >-
            pytest -sv
            tests/e2e/online_serving/test_bagel_expansion.py
            -m "full_model and diffusion and H100 and distributed_cuda"
            --run-level "full_model"
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU DreamZero"
        timeout_in_minutes: 120
        commands:
          - >-
            pytest -sv
            tests/e2e/online_serving/test_dreamzero_expansion.py
            -m "full_model and diffusion and H100 and distributed_cuda"
            --run-level "full_model"
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate


      - label: ":full_moon: Diffusion X2I(&A&T) · GLM-Image Function Test with H100"
        timeout_in_minutes: 120
        commands:
          - pytest -sv tests/e2e/online_serving/test_glm_image_expansion.py -m "full_model and diffusion and H100" --run-level "full_model"
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate


      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
        timeout_in_minutes: 60
        commands:
          - pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy
        agents:
          queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
        plugins:
          - docker#v5.2.0:
              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
              always-pull: true
              propagate-environment: true
              shm-size: "8gb"
              environment:
                - "HF_HOME=/fsx/hf_cache"
                - "HF_TOKEN"
              volumes:
                - "/fsx/hf_cache:/fsx/hf_cache"

      - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
        timeout_in_minutes: 60
        commands:
          - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model"
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test"
        timeout_in_minutes: 180
        commands:
          - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model
          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: VLLM_HTTP_TIMEOUT_KEEP_ALIVE
                        value: "120"
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
        timeout_in_minutes: 180
        commands:
          - export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct"
          - export HUNYUAN_IMAGE3_DEVICES="0,1,2,3"
          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m full_model --run-level full_model
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 4
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image"
        key: nightly-diffusion-x2iat-performance-qwen-image
        timeout_in_minutes: 180
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
          - export CACHE_DIT_VERSION=1.3.0
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
            exit $$EXIT
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 4
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit"
        key: nightly-diffusion-x2iat-performance-qwen-image-edit
        timeout_in_minutes: 180
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
          - export CACHE_DIT_VERSION=1.3.0
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
            exit $$EXIT
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 4
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511"
        key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511
        timeout_in_minutes: 180
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
          - export CACHE_DIT_VERSION=1.3.0
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
            exit $$EXIT
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 4
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Layered"
        key: nightly-diffusion-x2iat-performance-qwen-image-layered
        timeout_in_minutes: 120
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
          - export CACHE_DIT_VERSION=1.3.0
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
            exit $$EXIT
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · BAGEL"
        key: nightly-diffusion-x2iat-performance-bagel
        timeout_in_minutes: 180
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
          - export CACHE_DIT_VERSION=1.3.0
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
            exit $$EXIT
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

  # Diffusion x2v only (Wan, HunyuanVideo, …). x2i/x2a/x2t live in the X2I group above, not here.
  - group: ":card_index_dividers: Diffusion X2V Model Test"
    key: nightly-diffusion-x2v-group
    depends_on: upload-nightly-pipeline
    if: >-
      build.env("NIGHTLY") == "1" ||
      build.pull_request.labels includes "nightly-test" ||
      build.pull_request.labels includes "diffusion-x2v-test"
    steps:
      - label: ":full_moon: Diffusion X2V · Function Test"
        timeout_in_minutes: 90
        commands:
          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_hunyuan_video_15_expansion.py tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model"
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Diffusion X2V · Accuracy Test"
        timeout_in_minutes: 180
        commands:
          - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model
          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model
          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
          - pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m full_model --run-level full_model
          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Diffusion X2V · Perf Test"
        key: nightly-diffusion-x2v-performance
        timeout_in_minutes: 180
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json
            EXIT1=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
            exit $$EXIT1
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

  - group: ":card_index_dividers: Diffusion · Quantization Test"
    depends_on: upload-nightly-pipeline
    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test"
    steps:
      - label: ":full_moon: Quantization Test with H100"
        commands:
          - timeout 60m pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: ":full_moon: Quantization Test with L4"
        timeout_in_minutes: 60
        commands:
          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model"
        agents:
          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
        plugins:
          - docker#v5.2.0:
              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
              always-pull: true
              propagate-environment: true
              environment:
                - "HF_HOME=/fsx/hf_cache"
                - "HF_TOKEN"
              volumes:
                - "/fsx/hf_cache:/fsx/hf_cache"

  - label: ":bar_chart: Testcase Statistics"
    key: nightly-testcase-statistics
    timeout_in_minutes: 120
    depends_on: upload-nightly-pipeline
    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
    commands:
      - python tools/nightly/buildkite_testcase_statistics.py -o tests/dfx/perf/results/buildkite_testcase_statistics.html
      - buildkite-agent artifact upload "tests/dfx/perf/results/*.html"
    agents:
      queue: "mithril-h100-pool"
    plugins:
      - kubernetes:
          podSpec:
            containers:
              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                resources:
                  limits:
                    nvidia.com/gpu: 1
                volumeMounts:
                  - name: devshm
                    mountPath: /dev/shm
                  - name: hf-cache
                    mountPath: /root/.cache/huggingface
                env:
                  - name: HF_HOME
                    value: /root/.cache/huggingface
                  - name: HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
                        key: token
            nodeSelector:
              node.kubernetes.io/instance-type: gpu-h100-sxm
            volumes:
              - name: devshm
                emptyDir:
                  medium: Memory
              - name: hf-cache
                hostPath:
                  path: /mnt/hf-cache
                  type: DirectoryOrCreate

  # No need to run this step for PRs with label nightly-test
  - label: ":email: Nightly Collection & Email"
    key: nightly-perf-distribution
    depends_on:
      - nightly-omni-performance-no-async-chunk
      - nightly-omni-performance-async-chunk
      - nightly-omni-performance-vllm-text
      - nightly-omni-performance-multi-replicas
      - nightly-tts-performance
      - nightly-diffusion-x2iat-performance-qwen-image
      - nightly-diffusion-x2iat-performance-qwen-image-edit
      - nightly-diffusion-x2iat-performance-qwen-image-edit-2511
      - nightly-diffusion-x2iat-performance-qwen-image-layered
      - nightly-diffusion-x2iat-performance-bagel
      - nightly-diffusion-x2v-performance
      - nightly-testcase-statistics
    if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1"
    commands:
      - pip install openpyxl
      - export DEFAULT_INPUT_DIR=tests/dfx/perf/results
      - export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-multi-replicas
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2511
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-bagel
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance
      - buildkite-agent artifact download "tests/dfx/perf/results/*.html" . --step nightly-testcase-statistics
      - python tools/nightly/generate_nightly_perf_excel.py
      - python tools/nightly/generate_nightly_perf_html.py
      - python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html"
      - buildkite-agent artifact upload "tests/dfx/perf/results/*.xlsx"
      - buildkite-agent artifact upload "tests/dfx/perf/results/*.html"
    agents:
      queue: "cpu_queue_premerge"
