NVIDIA/Nemotron-3.5-Lightning

# An InferenceClass describing a Nebius gpu-h100-sxm node with 1x NVIDIA
# H100 80GB. Nebius sizes nodes by platform + preset rather than an
# instance type; gpu-h100-sxm + 1gpu-16vcpu-200gb is one single-H100 SXM
# node. The devices block describes the hardware DRA-style; it is what
# the Nemotron ModelDeployment's nodeSelector matches on.
apiVersion: modelplane.ai/v1alpha1
kind: InferenceClass
metadata:
  name: nebius-h100-1x
spec:
  description: "Nebius gpu-h100-sxm, 1x NVIDIA H100 80GB"
  provisioning:
    provider: Nebius
    nebius:
      platform: gpu-h100-sxm
      preset: 1gpu-16vcpu-200gb
      diskSizeGb: 200
      driversPreset: cuda13.0
      accelerator:
        type: nvidia-h100
        count: 1
  devices:
  - name: gpu
    claim: DRA
    driver: gpu.nvidia.com
    deviceClassName: gpu.nvidia.com
    count: 1
    attributes:
      architecture: { string: Hopper }
      cudaComputeCapability: { version: "9.0.0" }
    capacity:
      # H100 80GB real usable VRAM (NVIDIA DRA driver), not nominal 80GB.
      memory: { value: "81559Mi" }

# An InferenceCluster backed by a Nebius mk8s cluster with a single 1x
# H100 GPU node group. Modelplane provisions the full mk8s cluster (VPC,
# control plane, system + GPU node groups) and installs the inference
# stack. ModelCache RWX storage is auto-provisioned via a Nebius shared
# filesystem, so model-cache.yaml works unchanged.
#
# Auth is the Nebius ClusterProviderConfig named default (service-account
# key + projectID); Modelplane reuses that identity to reach the cluster.
#
# eu-north1 (Finland) keeps inference in the EU for data residency.
#
# Clean teardown - delete the ML resources first (they hold a usage on the
# cluster), then the cluster:
#   kubectl delete modeldeployment,modelservice,modelcache nemotron-lightning -n ml-team
#   kubectl delete inferencecluster nebius-eu-north --cascade=foreground
apiVersion: modelplane.ai/v1alpha1
kind: InferenceCluster
metadata:
  name: nebius-eu-north
  labels:
    modelplane.ai/region: eu-north
spec:
  cluster:
    source: Nebius
    nebius: {}

  nodePools:
  - name: gpu-h100
    className: nebius-h100-1x
    nodeCount: 1
    minNodeCount: 1
    maxNodeCount: 1

# The model weights, staged once per cluster on a Nebius shared filesystem
# (RWX) and mounted at /mnt/models in the serving pod, so the engine reads
# the NVFP4 weights (~20 GiB) locally instead of pulling them from Hugging
# Face on every start.
#
# nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 is a public repo, so no
# authSecret / HF token is needed. Add one only if you point this at a gated
# repo.
apiVersion: modelplane.ai/v1alpha1
kind: ModelCache
metadata:
  name: nemotron-lightning
  namespace: ml-team
spec:
  source: HuggingFace
  huggingFace:
    repo: nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4
    sizeGiB: 50

# Nemotron-3.5-Lightning (30B total / 3B active MoE, hybrid
# Mamba-Transformer) served NVFP4 as a single Standalone vLLM engine on one
# H100 80GB, weights streamed from the shared ModelCache at /mnt/models. The
# NVFP4 checkpoint (~20 GiB) fits a single GPU with headroom for the KV and
# Mamba caches, so no tensor parallelism, no gang, and no prefill/decode
# disaggregation are needed.
#
# Notes on the engine flags:
#   Quantization needs no flag - the checkpoint auto-detects as
#     modelopt_mixed.
#   --moe-backend=humming and --linear-backend=humming are the cookbook's
#     base-configuration kernels.
#   The --mamba-* flags configure the hybrid model's state-space cache: the
#     flashinfer backend with an FP16 SSM cache, stochastic rounding, and
#     the cookbook's align mode and horizontal SSU algorithm.
#   --reasoning-parser=nemotron_v3 extracts the thinking block;
#     --tool-call-parser=qwen3_coder is the parser Nemotron ships with, and
#     --enable-auto-tool-choice turns on server-side tool selection.
apiVersion: modelplane.ai/v1alpha1
kind: ModelDeployment
metadata:
  name: nemotron-lightning
  namespace: ml-team
spec:
  replicas: 1
  template:
    spec:
      modelCacheRef:
        name: nemotron-lightning
      engines:
      - name: nemotron-lightning
        members:
        - role: Standalone
          nodeSelector:
            devices:
            - name: gpu
              count: 1
              selectors:
              - cel: |
                  device.driver == "gpu.nvidia.com" && device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("79Gi")) >= 0
          template:
            spec:
              containers:
              - name: engine
                image: vllm/vllm-openai:v0.27.1
                command: ["vllm", "serve", "/mnt/models"]
                args:
                - --served-model-name=nemotron-3.5-lightning
                - --moe-backend=humming
                - --linear-backend=humming
                - --max-num-seqs=256
                - --max-model-len=65536
                - --max-num-batched-tokens=32768
                - --enable-prefix-caching
                - --async-scheduling
                - --mamba-backend=flashinfer
                - --mamba-ssm-cache-dtype=float16
                - --enable-mamba-cache-stochastic-rounding
                - --mamba-cache-philox-rounds=5
                - --mamba-cache-mode=align
                - --mamba-ssu-algorithm=horizontal
                - --reasoning-parser=nemotron_v3
                - --enable-auto-tool-choice
                - --tool-call-parser=qwen3_coder

# One OpenAI-compatible endpoint for the deployment. Read its public address:
#   kubectl get ms nemotron-lightning -n ml-team -o jsonpath='{.status.address}'
# then call it, e.g.:
#   curl "$ADDR/v1/chat/completions" -H 'Content-Type: application/json' \
#     -d '{"model":"nemotron-3.5-lightning","messages":[{"role":"user","content":"hello"}]}'
apiVersion: modelplane.ai/v1alpha1
kind: ModelService
metadata:
  name: nemotron-lightning
  namespace: ml-team
spec:
  endpoints:
  - selector:
      matchLabels:
        modelplane.ai/deployment: nemotron-lightning