Llama-3.1-8B-Instruct Ascend NPU 适配

# 从 ModelScope 下载（无需认证）
pip install modelscope
python3 -c "
from modelscope import snapshot_download
snapshot_download(
    'LLM-Research/Meta-Llama-3.1-8B-Instruct',
    cache_dir='./Llama-3.1-8B-Instruct'
)
"

huggingface-cli download meta-llama/Llama-3.1-8B-Instruct --local-dir ./Llama-3.1-8B-Instruct

python3 -m vllm.entrypoints.openai.api_server \
  --model ./Llama-3.1-8B-Instruct/LLM-Research/Meta-Llama-3___1-8B-Instruct \
  --dtype bfloat16 \
  --tensor-parallel-size 1 \
  --max-model-len 4096 \
  --max-num-seqs 8 \
  --port 8000 \
  --trust-remote-code

# 查看已加载模型
curl http://127.0.0.1:8000/v1/models

# 对话推理（推荐，Instruct 模型）
curl -s http://127.0.0.1:8000/v1/chat/completions \
  -H 'Content-Type: application/json' \
  -d '{
    "model": "./Llama-3.1-8B-Instruct/LLM-Research/Meta-Llama-3___1-8B-Instruct",
    "messages": [
      {"role": "system", "content": "You are a helpful assistant."},
      {"role": "user", "content": "What is the capital of France?"}
    ],
    "temperature": 0,
    "max_tokens": 64
  }'

# 文本续写推理
curl -s http://127.0.0.1:8000/v1/completions \
  -H 'Content-Type: application/json' \
  -d '{
    "model": "./Llama-3.1-8B-Instruct/LLM-Research/Meta-Llama-3___1-8B-Instruct",
    "prompt": "The capital of France is",
    "temperature": 0,
    "max_tokens": 64
  }'

python3 inference_npu.py

+------------------------------------------------------------------------------------------------+
| NPU   Name                | Health        | Power(W)    Temp(C)           Hugepages-Usage(page)|
| Chip  Phy-ID              | Bus-Id        | AICore(%)   Memory-Usage(MB)  HBM-Usage(MB)        |
+===========================+===============+====================================================+
| 0     Ascend910           | OK            | 166.3       45                0    / 0             |
| 0     0                   | 0000:0A:00.0  | 0           0    / 0          59678/ 65536         |
+===========================+===============+====================================================+
| NPU     Chip              | Process id    | Process name             | Process memory(MB)      |
+===========================+===============+====================================================+
| 0       0                 | 32596         | VLLMEngineCor            | 56624                   |
+===========================+===============+====================================================+

{
  "model": "Llama-3.1-8B-Instruct",
  "messages": [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "What is the capital of France?"}
  ],
  "temperature": 0,
  "max_tokens": 64
}

{
  "id": "chatcmpl-58f59c7c4e55ecb4",
  "choices": [{
    "message": {
      "role": "assistant",
      "content": "The capital of France is Paris."
    },
    "finish_reason": "stop"
  }],
  "usage": {"prompt_tokens": 48, "total_tokens": 56, "completion_tokens": 8}
}

{
  "model": "Llama-3.1-8B-Instruct",
  "messages": [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "Hello! Please introduce yourself briefly."}
  ],
  "temperature": 0.7,
  "max_tokens": 128
}

{
  "id": "chatcmpl-0a012de5f5f4cd3e",
  "choices": [{
    "message": {
      "role": "assistant",
      "content": "Hello. I'm an AI assistant, here to provide information, answer questions, and help with tasks to the best of my abilities. I'm constantly learning and updating my knowledge to ensure I can assist you effectively. How can I help you today?"
    },
    "finish_reason": "stop"
  }],
  "usage": {"prompt_tokens": 48, "total_tokens": 99, "completion_tokens": 51}
}

{
  "model": "Llama-3.1-8B-Instruct",
  "messages": [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "Explain what Python is in 2-3 sentences."}
  ],
  "temperature": 0,
  "max_tokens": 128
}

{
  "id": "chatcmpl-a1a3f70aeaedc9be",
  "choices": [{
    "message": {
      "role": "assistant",
      "content": "Python is a high-level, interpreted programming language that is widely used for various purposes such as web development, data analysis, artificial intelligence, and more. It is known for its simplicity, readability, and ease of use, making it a popular choice among beginners and experienced developers alike. Python's syntax is concise and intuitive, allowing developers to focus on solving problems rather than getting bogged down in complex code."
    },
    "finish_reason": "stop"
  }],
  "usage": {"prompt_tokens": 53, "total_tokens": 135, "completion_tokens": 82}
}

{
  "model": "Llama-3.1-8B-Instruct",
  "prompt": "The capital of France is",
  "temperature": 0,
  "max_tokens": 64
}

{
  "id": "cmpl-ac5e2c6ee15bcc2c",
  "choices": [{
    "text": " a city of romance, art, fashion, and cuisine. Paris is a must-visit destination for anyone who loves history, architecture, and culture. From the iconic Eiffel Tower to the world-famous Louvre Museum, Paris has something to offer for every interest and age.\nThe city is divided into 20",
    "finish_reason": "length"
  }],
  "usage": {"prompt_tokens": 6, "total_tokens": 70, "completion_tokens": 64}
}

.
├── README.md              # 本文档
├── inference_npu.py       # NPU 推理脚本
├── start_server.sh        # 一键启动脚本
├── .gitignore             # Git 忽略规则
└── LICENSE