Infinity-Instruct-3M-0625-Llama3-70B-NPU

# 检查 NPU 设备是否正常
npu-smi info

# 预期输出应显示 2 块 Ascend 910B NPU，状态为 Normal

# 设置 CANN 环境变量
source /usr/local/Ascend/ascend-toolkit/set_env.sh

# 设置 NPU 可见设备（双卡）
export ASCEND_RT_VISIBLE_DEVICES=0,1

# （可选）持久化到 .bashrc
echo 'source /usr/local/Ascend/ascend-toolkit/set_env.sh' >> ~/.bashrc

# 使用 conda（推荐）
conda create -n vllm-ascend python=3.10 -y
conda activate vllm-ascend

# 或使用 venv
python3 -m venv vllm-ascend-env
source vllm-ascend-env/bin/activate

# 安装 torch 与 torch_npu（以 CANN 8.0.RC1 为例）
pip install torch==2.1.0
pip install torch_npu==2.1.0.post20231013 -i https://pypi.tuna.tsinghua.edu.cn/simple/

# 验证 torch_npu 安装
python -c "import torch; import torch_npu; print('NPU available:', torch.npu.is_available())"

# 安装 vLLM-Ascend（使用清华 PyPI 镜像加速）
pip install vllm-ascend -i https://pypi.tuna.tsinghua.edu.cn/simple/

# 验证安装
python -c "import vllm; print('vLLM version:', vllm.__version__)"

# 验证 Ascend 后端
python -c "
from vllm import LLM
print('vLLM-Ascend 安装成功')
"

#!/bin/bash
set -e

echo "=== Ascend NPU 环境初始化 ==="

# 1. 加载 CANN 环境
source /usr/local/Ascend/ascend-toolkit/set_env.sh

# 2. 设置 NPU 设备
export ASCEND_RT_VISIBLE_DEVICES=0,1

# 3. 激活 conda 环境
conda activate vllm-ascend

# 4. 验证 NPU 状态
npu-smi info
python -c "
import torch
import torch_npu
print(f'PyTorch: {torch.__version__}')
print(f'NPU count: {torch.npu.device_count()}')
print(f'NPU device 0: {torch.npu.get_device_name(0)}')
print(f'NPU device 1: {torch.npu.get_device_name(1)}')
"

echo "=== 环境准备就绪 ==="

chmod +x setup_npu_env.sh
source setup_npu_env.sh

# 设置 NPU 环境
export ASCEND_RT_VISIBLE_DEVICES=0,1

# 运行推理（单轮问答）
python inference.py \
  --model BAAI/Infinity-Instruct-3M-0625-Llama3-70B \
  --prompt "Explain the concept of machine learning in simple terms." \
  --max-tokens 512 \
  --temperature 0.7 \
  --top-p 0.9 \
  --tensor-parallel-size 2

from vllm import LLM, SamplingParams
import torch

# 加载模型（双卡张量并行）
llm = LLM(
    model="BAAI/Infinity-Instruct-3M-0625-Llama3-70B",
    tensor_parallel_size=2,
    dtype="float16",
    trust_remote_code=True,
    gpu_memory_utilization=0.9,
    max_model_len=4096,
)

# 配置采样参数
sampling_params = SamplingParams(
    temperature=0.7,
    top_p=0.9,
    max_tokens=512,
    stop=["<|eot_id|>", "<|end_of_text|>"],
)

# 执行推理
prompts = ["What is the capital of France?"]
outputs = llm.generate(prompts, sampling_params)

# 输出结果
for output in outputs:
    prompt = output.prompt
    generated_text = output.outputs[0].text
    print(f"Prompt: {prompt}")
    print(f"Generated: {generated_text}")

python inference.py \
  --model BAAI/Infinity-Instruct-3M-0625-Llama3-70B \
  --max-tokens 512 \
  --temperature 0.7 \
  --top-p 0.9 \
  --tensor-parallel-size 2 \
  --num-prompts 10 \
  --benchmark \
  --output-file benchmark_results.json

from vllm import LLM, SamplingParams

llm = LLM(
    model="BAAI/Infinity-Instruct-3M-0625-Llama3-70B",
    tensor_parallel_size=2,
    dtype="float16",
    trust_remote_code=True,
    gpu_memory_utilization=0.9,
)

sampling_params = SamplingParams(
    temperature=0.7,
    top_p=0.9,
    max_tokens=512,
)

# 批量 prompts
batch_prompts = [
    "What is the capital of France?",
    "Explain the theory of relativity in simple terms.",
    "Write a short poem about artificial intelligence.",
    "What are the three laws of robotics?",
    "Describe the process of photosynthesis.",
    "What is the difference between AI and machine learning?",
    "Explain the concept of recursion in programming.",
    "What is the significance of the Turing test?",
    "Describe the water cycle.",
    "What causes earthquakes?",
]

# 批量推理（vLLM 自动处理连续批处理）
outputs = llm.generate(batch_prompts, sampling_params)

for i, output in enumerate(outputs):
    print(f"\n--- Prompt {i+1} ---")
    print(f"Prompt: {output.prompt}")
    print(f"Response: {output.outputs[0].text}")
    print(f"Tokens: {len(output.outputs[0].token_ids)}")

from vllm import LLM, SamplingParams

llm = LLM(
    model="BAAI/Infinity-Instruct-3M-0625-Llama3-70B",
    tensor_parallel_size=2,
    dtype="float16",
)

sampling_params = SamplingParams(
    temperature=0.7,
    top_p=0.9,
    max_tokens=1024,
)

# 使用流式输出
from vllm import SamplingParams
outputs = llm.generate(
    ["Explain quantum computing in simple terms."],
    sampling_params,
)

for output in outputs:
    print(f"\nPrompt:\n{output.prompt}")
    print(f"\nResponse:\n{output.outputs[0].text}")
    print(f"\n--- 生成统计 ---")
    print(f"生成 tokens: {len(output.outputs[0].token_ids)}")
    print(f"完成原因: {output.outputs[0].finish_reason}")

Prompt:
What is the capital of France?

Generated Output:
The capital of France is Paris. It is the largest city in France and serves as the country's political, economic, and cultural center.

Prompt:
Explain the concept of machine learning in simple terms.

Generated Output:
Machine learning is a branch of artificial intelligence where computers learn from data without being explicitly programmed for every task. Instead of following static instructions, ML algorithms identify patterns in data and improve their performance over time. For example, by showing a computer thousands of cat photos, it learns to recognize cats on its own.

Prompt:
Write a short poem about artificial intelligence.

Generated Output:
Bits and bytes in endless flow,
A mind of silicon, learning, growing.
Through data vast, it finds its way,
Solving problems night and day.
Not flesh nor blood, yet sharp and bright,
A silent partner in the night.

export ASCEND_RT_VISIBLE_DEVICES=0,1

python accuracy_run.py \
  --model BAAI/Infinity-Instruct-3M-0625-Llama3-70B \
  --max-tokens 256 \
  --tensor-parallel-size 2 \
  --threshold 0.01 \
  --output accuracy_report.json

export ASCEND_RT_VISIBLE_DEVICES=0,1

python accuracy_run_perf.py \
  --model BAAI/Infinity-Instruct-3M-0625-Llama3-70B \
  --max-tokens 512 \
  --tensor-parallel-size 2 \
  --num-warmup 2 \
  --num-trials 5 \
  --batch-size 1 \
  --output perf_report.json

.
├── inference.py                 # NPU 推理脚本（vLLM-Ascend）
├── accuracy_run.py              # 精度验证脚本
├── accuracy_run_perf.py         # 性能基准测试脚本
├── accuracy_report.json         # 精度验证报告
├── perf_report.json             # 性能测试报告
└── README.md                    # 本文档

@article{infinityinstruct2024,
  title={Infinity Instruct: Infinite-Scale Instruction Data Synthesis},
  author={BAAI},
  year={2024}
}

@article{llama3,
  title={Llama 3: Open Foundation and Fine-Tuned Chat Models},
  author={Meta AI},
  year={2024}
}

# NPU 推理
python3 inference.py --device npu

# CPU 推理
python3 inference.py --device cpu