tara-1b 昇腾 NPU 部署文档

# 1. 加载 CANN 环境
source /usr/local/Ascend/ascend-toolkit/set_env.sh

# 2. 创建并激活 Python 虚拟环境
conda create -n tara python=3.11 -y
conda activate tara

# 3. 安装核心依赖
pip install torch==2.1.0
pip install torch_npu==2.1.0  # 请根据 CANN 版本选择对应 torch_npu
pip install vllm==0.17.0rc1
pip install vllm-ascend

# 4. 安装评测依赖（可选）
pip install openai requests datasets

# 5. 验证安装
python -c "import torch; import torch_npu; import vllm; print('All imports OK')"
npu-smi info

export VLLM_USE_MODELSCOPE=true
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export HCCL_BUFFSIZE=512
export OMP_PROC_BIND=false
export OMP_NUM_THREADS=1
export TASK_QUEUE_ENABLE=1
export VLLM_WORKER_MULTIPROC_METHOD=spawn

# 示例路径
MODEL_PATH=/data/models/tara-1b

# 确认目录结构
ls -la $MODEL_PATH
# 预期输出：config.json, tokenizer.json, model.safetensors / pytorch_model.bin, 等

/data/models/tara-1b/
├── config.json
├── tokenizer.json
├── tokenizer_config.json
├── generation_config.json
├── model.safetensors.index.json
├── model-00001-of-00002.safetensors
├── model-00002-of-00002.safetensors
└── special_tokens_map.json

cd tara-1b-deliverable
bash scripts/start_server.sh /data/models/tara-1b 8000

# 设置环境变量
source scripts/set_env.sh  # 或手动 export

# 启动 vLLM 服务
vllm serve /data/models/tara-1b \
  --host 0.0.0.0 \
  --port 8000 \
  --served-model-name tara-1b \
  --tensor-parallel-size 1 \
  --data-parallel-size 1 \
  --max-model-len 32768 \
  --max-num-seqs 64 \
  --max-num-batched-tokens 4096 \
  --gpu-memory-utilization 0.85 \
  --trust-remote-code \
  --dtype auto \
  --device npu \
  --seed 1024 \
  --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY"}' \
  --async-scheduling

# 健康检查
curl -s http://localhost:8000/health
# 预期返回：{"status":"ok"}

# 查看模型列表
curl -s http://localhost:8000/v1/models | python -m json.tool
# 预期返回：包含 "tara-1b"

curl http://localhost:8000/v1/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "tara-1b",
    "prompt": "The future of artificial intelligence is",
    "max_tokens": 100,
    "temperature": 0.7,
    "top_p": 0.9,
    "stop": ["\n\n"]
  }'

{
  "id": "cmpl-xxx",
  "object": "text_completion",
  "created": 1715420000,
  "model": "tara-1b",
  "choices": [
    {
      "text": " incredibly promising, with advances in...",
      "index": 0,
      "logprobs": null,
      "finish_reason": "length"
    }
  ],
  "usage": {
    "prompt_tokens": 8,
    "completion_tokens": 100,
    "total_tokens": 108
  }
}

curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "tara-1b",
    "messages": [
      {"role": "system", "content": "You are a helpful assistant."},
      {"role": "user", "content": "请用一句话解释什么是人工智能。"}
    ],
    "max_tokens": 128,
    "temperature": 0.7
  }'

{
  "id": "chatcmpl-xxx",
  "object": "chat.completion",
  "model": "tara-1b",
  "choices": [
    {
      "index": 0,
      "message": {
        "role": "assistant",
        "content": "人工智能是计算机科学的一个分支，旨在创建能够执行通常需要人类智能的任务的系统。"
      },
      "finish_reason": "stop"
    }
  ],
  "usage": {
    "prompt_tokens": 25,
    "completion_tokens": 32,
    "total_tokens": 57
  }
}

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="not-needed",
)

# Completions
response = client.completions.create(
    model="tara-1b",
    prompt="The future of AI is",
    max_tokens=100,
    temperature=0.7,
)
print(response.choices[0].text)

# Chat
response = client.chat.completions.create(
    model="tara-1b",
    messages=[
        {"role": "user", "content": "Hello!"}
    ],
    max_tokens=50,
)
print(response.choices[0].message.content)

# 单条推理
python scripts/inference.py \
  --model-path /data/models/tara-1b \
  --prompt "1+1等于几？" \
  --max-tokens 50

# 批量推理
python scripts/inference.py \
  --model-path /data/models/tara-1b \
  --batch-file prompts.txt \
  --output results.json

# 快速基准测试
python scripts/inference.py \
  --model-path /data/models/tara-1b \
  --benchmark \
  --num-prompts 100 \
  --input-len 128 \
  --output-len 128

# GSM8K 数学推理评测
python eval/accuracy_run.py \
  --server-url http://localhost:8000 \
  --dataset gsm8k \
  --num-samples 100

# HumanEval 代码生成评测
python eval/accuracy_run.py \
  --server-url http://localhost:8000 \
  --dataset humaneval \
  --num-samples 164

accuracy_results/
├── tara-1b_gsm8k_result.json     # 结构化结果
├── tara-1b_gsm8k_report.md       # Markdown 报告
└── tara-1b_humaneval_result.json

# 全部模式测试（API 模式）
python eval/performance_run.py \
  --server-url http://localhost:8000 \
  --mode all \
  --input-len 200 \
  --output-len 200 \
  --num-prompts 200

# 单独测试 Serve 吞吐
python eval/performance_run.py \
  --server-url http://localhost:8000 \
  --mode serve \
  --input-len 200 \
  --output-len 200 \
  --num-prompts 200 \
  --concurrency 10

# 单独测试延迟
python eval/performance_run.py \
  --server-url http://localhost:8000 \
  --mode latency \
  --input-len 512 \
  --output-len 256 \
  --num-iterations 10

perf_results/
├── tara-1b_perf_report_20240101_120000.json
└── tara-1b_perf_summary.md

# 收集所有信息并生成完整报告
python eval/log_generator.py --collect-all --output-dir ./logs

# 仅收集环境信息
python eval/log_generator.py --collect-env --output-dir ./logs

logs/
├── tara-1b_deployment_report.json   # 结构化 JSON 报告
├── tara-1b_deployment_report.md     # Markdown 汇总
├── npu_smi.txt                      # NPU 状态
├── npu_smi_version.txt              # 驱动版本
├── pip_list.txt                     # Python 包列表
└── ascend_env.txt                   # 环境变量

# 1. 启动服务
bash scripts/start_server.sh /data/models/tara-1b 8000

# 2. 验证服务
curl http://localhost:8000/v1/completions \
  -H "Content-Type: application/json" \
  -d '{"model":"tara-1b","prompt":"Hello","max_tokens":50}'

# 3. 精度评测
python eval/accuracy_run.py --dataset gsm8k

# 4. 性能评测
python eval/performance_run.py --mode all

# 5. 生成报告
python eval/log_generator.py --collect-all