# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# One-shot Job that pre-downloads Qwen3-VL-30B-A3B-Instruct weights from HuggingFace into the
# qwen3-vl-pvc PVC in HF cache layout.
#
# Why: model-free-nim:2.0.2 expects the HF cache tree
# ($HF_HOME/hub/models--Qwen--Qwen3-VL-30B-A3B-Instruct/snapshots/<hash>/...) when given
# NIM_MODEL_PATH=hf://... with HF_HUB_OFFLINE=1. NIMCache's HF source produces a flat
# --local-dir layout instead and is incompatible (bug 6087790).
#
# The huggingface-cli (hf download) handles 429 rate limiting with backoff — which was an issue
# for the 13-shard 30B safetensors when we tried letting the NIM container download directly.
#
# install.sh applies this after pvc.yaml and before nimservice.yaml, then waits for
# condition=complete. Manual: kubectl wait -n osmo-nims --for=condition=complete job/qwen3-vl-hf-download --timeout=60m
apiVersion: batch/v1
kind: Job
metadata:
  name: qwen3-vl-hf-download
  namespace: osmo-nims
spec:
  backoffLimit: 2
  # Auto-delete the Job (and its pod) 60s after successful completion so the pod releases its
  # hold on the ReadWriteOnce PVC. Without this, a Completed pod keeps the volume attached on
  # its node, producing "Multi-Attach error" when the NIMService tries to mount the same PVC.
  ttlSecondsAfterFinished: 60
  template:
    spec:
      restartPolicy: OnFailure
      containers:
        - name: hf-download
          image: python:3.11-slim
          env:
            - name: HF_HOME
              # hf download places files under $HF_HOME/hub/models--<org>--<name>/... which is
              # exactly what the NIMService consumes with HF_HOME=/model-store/huggingface/hub.
              value: /model-store/huggingface
            - name: HF_HUB_ENABLE_HF_TRANSFER
              value: "1"
            - name: HF_TOKEN
              valueFrom:
                secretKeyRef:
                  name: hf-token-secret
                  key: HF_TOKEN
          command: ["bash", "-c"]
          args:
            - |
              set -euo pipefail
              pip install --quiet 'huggingface_hub[cli,hf_transfer]'
              echo "=== Downloading Qwen/Qwen3-VL-30B-A3B-Instruct to $HF_HOME/hub ==="
              hf download Qwen/Qwen3-VL-30B-A3B-Instruct
              echo "=== Download complete ==="
              du -sh "$HF_HOME"/hub/models--Qwen--Qwen3-VL-30B-A3B-Instruct || true
          volumeMounts:
            - name: model-store
              mountPath: /model-store
      volumes:
        - name: model-store
          persistentVolumeClaim:
            claimName: qwen3-vl-pvc
