NVIDIA-Nemotron-3-Nano-4B-BF16

import torch
from transformers import AutoTokenizer, AutoModelForCausalLM

# Load tokenizer and model
tokenizer = AutoTokenizer.from_pretrained("nvidia/NVIDIA-Nemotron-3-Nano-4B")
model = AutoModelForCausalLM.from_pretrained(
    "nvidia/NVIDIA-Nemotron-3-Nano-4B",
    torch_dtype=torch.bfloat16,
    trust_remote_code=True,
    device_map="auto"
)

messages = [
    {"role": "system", "content": <system_prompt>},
    {"role": "user", "content": "Write a haiku about GPUs"},
]
tokenized_chat = tokenizer.apply_chat_template(
    messages,
    tokenize=True,
    add_generation_prompt=True,
    return_tensors="pt"
).to(model.device)

outputs = model.generate(
    tokenized_chat,
    max_new_tokens=32,
    eos_token_id=tokenizer.eos_token_id
)
print(tokenizer.decode(outputs[0]))

messages = [
    {"role": "system", "content": <system_prompt>},
    {"role": "user", "content": "Write a haiku about GPUs"},
]
tokenized_chat = tokenizer.apply_chat_template(
    messages,
    tokenize=True,
    enable_thinking=False,
    add_generation_prompt=True,
    return_tensors="pt"
).to(model.device)

outputs = model.generate(
    tokenized_chat,
    max_new_tokens=32,
    eos_token_id=tokenizer.eos_token_id
)
print(tokenizer.decode(outputs[0]))

pip install -U "vllm>=0.15.1"

wget https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/resolve/main/nano_v3_reasoning_parser.py

vllm serve nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16 \
  --served-model-name nemotron3-nano-4B-BF16\
  --max-num-seqs 8 \
  --tensor-parallel-size 1 \
  --max-model-len 262144 \
  --port 8000 \
  --trust-remote-code \
  --mamba_ssm_cache_dtype float32 \
  --enable-auto-tool-choice \
  --tool-call-parser qwen3_coder \
  --reasoning-parser-plugin nano_v3_reasoning_parser.py \
  --reasoning-parser nano_v3

from openai import OpenAI
import asyncio
from openai import AsyncOpenAI

# NOTE: Streaming is preferred for better performance and resource efficiency.
# It allows you to start processing responses as they arrive, reducing latency.

# Synchronous example (non-streaming)
client = OpenAI(
    api_key="your-nvapikey",
    base_url="base-url"
)

response = client.chat.completions.create(
    model="nemotron3-nano-4B-BF16",
    messages=[
        {
            "role": "user",
            "content": "Hello!"
        }
    ],
    temperature=0.7,
    max_tokens=256,
    top_p=0.7,
    stream=false
)

print(response.choices[0].message.content)

docker run -v /home/root/.cache/huggingface/:/root/.cache/huggingface/ --rm --ulimit memlock=-1 --ulimit stack=67108864 --gpus=all --ipc=host --network host -d -e MODEL=nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16 -e HF_TOKEN=$HF_TOKEN nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc6 bash -c '
cat > /tmp/extra-llm-api-config.yml <<EOF
kv_cache_config:
  dtype: "auto"
  enable_block_reuse: false
cuda_graph_config:
  max_batch_size: 32
  enable_padding: true
disable_overlap_scheduler: true
moe_config: 
  backend: CUTLASS
EOF

trtllm-serve  \
nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16 \
--host 0.0.0.0 \
--port 8123 \
--max_batch_size 32 \
--extra_llm_api_options /tmp/extra-llm-api-config.yml '

curl http://localhost:8123/v1/chat/completions -H "Content-Type: application/json"  -d '{
    "model": "NVIDIA-Nemotron-3-Nano-4B-BF16",
    "messages": [
        {
            "role": "user",
            "content": "Where is New York?"
        }
    ],
    "max_tokens": 1024,
    "top_p": 1.0
}' -w "\n"