Llama-3.2-3B-Instruct Ascend NPU 适配

# 从 ModelScope 下载（无需认证）
pip install modelscope
python3 -c "
from modelscope import snapshot_download
snapshot_download(
    'LLM-Research/Llama-3.2-3B-Instruct',
    cache_dir='./Llama-3.2-3B-Instruct'
)
"

huggingface-cli download meta-llama/Llama-3.2-3B-Instruct --local-dir ./Llama-3.2-3B-Instruct

python3 -m vllm.entrypoints.openai.api_server \
  --model ./Llama-3.2-3B-Instruct/LLM-Research/Llama-3___2-3B-Instruct \
  --dtype bfloat16 \
  --tensor-parallel-size 1 \
  --max-model-len 4096 \
  --max-num-seqs 8 \
  --port 8000 \
  --trust-remote-code

# 查看已加载模型
curl http://127.0.0.1:8000/v1/models

# 对话推理（推荐，Instruct 模型）
curl -s http://127.0.0.1:8000/v1/chat/completions \
  -H 'Content-Type: application/json' \
  -d '{
    "model": "./Llama-3.2-3B-Instruct/LLM-Research/Llama-3___2-3B-Instruct",
    "messages": [
      {"role": "system", "content": "You are a helpful assistant."},
      {"role": "user", "content": "What is the capital of France?"}
    ],
    "temperature": 0,
    "max_tokens": 64
  }'

# 文本续写推理
curl -s http://127.0.0.1:8000/v1/completions \
  -H 'Content-Type: application/json' \
  -d '{
    "model": "./Llama-3.2-3B-Instruct/LLM-Research/Llama-3___2-3B-Instruct",
    "prompt": "The capital of France is",
    "temperature": 0,
    "max_tokens": 64
  }'

python3 inference_npu.py

+------------------------------------------------------------------------------------------------+
| NPU   Name                | Health        | Power(W)    Temp(C)           Hugepages-Usage(page)|
| Chip  Phy-ID              | Bus-Id        | AICore(%)   Memory-Usage(MB)  HBM-Usage(MB)        |
+===========================+===============+====================================================+
| 0     Ascend910           | OK            | 165.6       45                0    / 0             |
| 0     0                   | 0000:0A:00.0  | 0           0    / 0          59751/ 65536         |
+===========================+===============+====================================================+
| NPU     Chip              | Process id    | Process name             | Process memory(MB)      |
+===========================+===============+====================================================+
| 0       0                 | 26961         | VLLMEngineCor            | 56698                   |
+===========================+===============+====================================================+

{
  "model": "Llama-3.2-3B-Instruct",
  "messages": [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "What is the capital of France?"}
  ],
  "temperature": 0,
  "max_tokens": 64
}

{
  "id": "chatcmpl-9463fa45a8602527",
  "choices": [{
    "message": {
      "role": "assistant",
      "content": "The capital of France is Paris."
    },
    "finish_reason": "stop"
  }],
  "usage": {"prompt_tokens": 48, "total_tokens": 56, "completion_tokens": 8}
}

{
  "model": "Llama-3.2-3B-Instruct",
  "messages": [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "Hello! Please introduce yourself briefly."}
  ],
  "temperature": 0.7,
  "max_tokens": 128
}

{
  "id": "chatcmpl-ea8d8d76433b7a1e",
  "choices": [{
    "message": {
      "role": "assistant",
      "content": "Hello. I'm an artificial intelligence assistant, here to provide information and help with any questions or tasks you may have. I don't have a personal name, but I'm often referred to as a conversational AI or a chatbot. I'm constantly learning and improving to provide the best possible assistance. How can I help you today?"
    },
    "finish_reason": "stop"
  }],
  "usage": {"prompt_tokens": 48, "total_tokens": 117, "completion_tokens": 69}
}

{
  "model": "Llama-3.2-3B-Instruct",
  "messages": [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "Explain what Python is in 2-3 sentences."}
  ],
  "temperature": 0,
  "max_tokens": 128
}

{
  "id": "chatcmpl-484f9c0255c93a7a",
  "choices": [{
    "message": {
      "role": "assistant",
      "content": "Python is a high-level, interpreted programming language that is widely used for various purposes such as web development, scientific computing, data analysis, artificial intelligence, and more. It is known for its simplicity, readability, and ease of use, making it a popular choice among beginners and experienced programmers alike. Python's syntax is designed to be intuitive and flexible, allowing developers to focus on writing code rather than managing complex syntax."
    },
    "finish_reason": "stop"
  }],
  "usage": {"prompt_tokens": 53, "total_tokens": 137, "completion_tokens": 84}
}

{
  "model": "Llama-3.2-3B-Instruct",
  "prompt": "The capital of France is",
  "temperature": 0,
  "max_tokens": 64
}

{
  "id": "cmpl-f6e68e0e72a67c3e",
  "choices": [{
    "text": " Paris. The capital of Italy is Rome. The capital of Spain is Madrid. The capital of Germany is Berlin. The capital of the United Kingdom is London. The capital of Australia is Canberra. The capital of China is Beijing. The capital of Japan is Tokyo. The capital of India is New Delhi. The capital of",
    "finish_reason": "length"
  }],
  "usage": {"prompt_tokens": 6, "total_tokens": 70, "completion_tokens": 64}
}

.
├── README.md              # 本文档
├── inference_npu.py       # NPU 推理脚本
├── start_server.sh        # 一键启动脚本
├── .gitignore             # Git 忽略规则
└── LICENSE