Infinity-Instruct-3M-0613-Llama3-70B-NPU

# 检查 NPU 状态
npu-smi info

# 设置环境变量
source /usr/local/Ascend/ascend-toolkit/set_env.sh

# 验证 CANN 版本
cat /usr/local/Ascend/ascend-toolkit/latest/version.cfg

# 安装驱动（根据实际版本调整路径）
chmod +x Ascend-hdk-910b-npu-driver-*.run
./Ascend-hdk-910b-npu-driver-*.run --full

# 安装 CANN 工具包
chmod +x Ascend-cann-toolkit-*.run
./Ascend-cann-toolkit-*.run --install

# 创建虚拟环境
conda create -n vllm-ascend python=3.10 -y
conda activate vllm-ascend

# 安装 vllm-ascend（推荐清华镜像）
pip install vllm-ascend -i https://pypi.tuna.tsinghua.edu.cn/simple/

# 验证安装
python -c "import vllm; print(vllm.__version__)"

# 指定使用的 NPU 设备（双卡）
export ASCEND_RT_VISIBLE_DEVICES=0,1

# 可选：调整内存分配比例（默认 0.9）
export VLLM_ASCEND_MEMORY_FRACTION=0.9

# 可选：启用更详细的日志
export VLLM_ASCEND_LOG_LEVEL=INFO

export ASCEND_RT_VISIBLE_DEVICES=0,1

python inference.py \
  --model BAAI/Infinity-Instruct-3M-0613-Llama3-70B \
  --prompt "Explain quantum computing." \
  --max-tokens 512 \
  --tensor-parallel-size 2

# batch_inference.py
import os
from vllm import LLM, SamplingParams

os.environ["ASCEND_RT_VISIBLE_DEVICES"] = "0,1"

# 初始化模型（双卡张量并行）
llm = LLM(
    model="BAAI/Infinity-Instruct-3M-0613-Llama3-70B",
    tensor_parallel_size=2,
    trust_remote_code=True,
    dtype="float16",
    max_model_len=8192,
)

# 采样参数
sampling_params = SamplingParams(
    temperature=0.7,
    top_p=0.9,
    max_tokens=512,
)

# 批量输入
prompts = [
    "Explain quantum computing in simple terms.",
    "Write a Python function to compute fibonacci numbers.",
    "What are the causes of climate change?",
    "Summarize the plot of Hamlet in 3 sentences.",
    "Prove the Pythagorean theorem.",
]

# 执行推理
outputs = llm.generate(prompts, sampling_params)

# 输出结果
for i, output in enumerate(outputs):
    prompt = output.prompt
    generated_text = output.outputs[0].text
    print(f"=== Prompt {i+1} ===")
    print(f"输入: {prompt}")
    print(f"输出: {generated_text}")
    print(f"生成 tokens: {len(output.outputs[0].token_ids)}")
    print()

export ASCEND_RT_VISIBLE_DEVICES=0,1
python batch_inference.py

export ASCEND_RT_VISIBLE_DEVICES=0,1

python -m vllm.entrypoints.openai.api_server \
    --model BAAI/Infinity-Instruct-3M-0613-Llama3-70B \
    --tensor-parallel-size 2 \
    --dtype float16 \
    --max-model-len 8192 \
    --port 8000

curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "BAAI/Infinity-Instruct-3M-0613-Llama3-70B",
    "messages": [{"role": "user", "content": "Hello! What is AI?"}],
    "temperature": 0.7,
    "max_tokens": 256
  }'

export ASCEND_RT_VISIBLE_DEVICES=0,1

python accuracy_run.py \
  --model BAAI/Infinity-Instruct-3M-0613-Llama3-70B \
  --tensor-parallel-size 2 \
  --output accuracy_report.json

Prompt:  What is the boiling point of water at sea level in Celsius?

Response: The boiling point of water at sea level is 100°C (212°F).

Prompt:  Write a Python function to check if a string is a palindrome.

Response: 
def is_palindrome(s):
    s = s.lower().replace(" ", "")
    return s == s[::-1]

Prompt:  If a train travels at 60 mph for 2.5 hours, how far does it travel?

Response: Distance = Speed × Time = 60 × 2.5 = 150 miles.

Prompt:  All men are mortal. Socrates is a man. Is Socrates mortal?

Response: Yes, Socrates is mortal. Since all men are mortal and Socrates is a man,
by modus ponens, Socrates must be mortal.

Prompt:  Write a haiku about artificial intelligence.

Response: 
Silicon dreams wake,
Patterns learning from the dark,
Machines begin to think.

export ASCEND_RT_VISIBLE_DEVICES=0,1

python accuracy_run_perf.py \
  --model BAAI/Infinity-Instruct-3M-0613-Llama3-70B \
  --tensor-parallel-size 2 \
  --output perf_report.json

.
├── inference.py
├── accuracy_run.py
├── accuracy_run_perf.py
├── accuracy_report.json
├── perf_report.json
└── README.md

# NPU 推理
python3 inference.py --device npu

# CPU 推理
python3 inference.py --device cpu