BitCPM4-CANN-1B (Ascend NPU)

export ASCEND_LOG_DEV_AND_USR=0
export VLLM_USE_PRECOMPILED=0
export NPU_VISIBLE_DEVICES=0

# 部署前需应用 monkey-patch（见下方说明）
python3 -c "
import warnings, logging
warnings.filterwarnings('ignore')
logging.getLogger().setLevel(logging.ERROR)

# Monkey-patch: handle MiniCPM weight tying
import vllm.model_executor.models.minicpm as _m
_o = _m.MiniCPMForCausalLM.load_weights
def _p(self, w):
    r = _o(self, w)
    if self.config.tie_word_embeddings:
        r.add('lm_head.weight')
    return r
_m.MiniCPMForCausalLM.load_weights = _p

from vllm.entrypoints.cli.main import main
import sys
sys.argv = [
    'vllm', 'serve', '/path/to/model/',
    '--trust-remote-code',
    '--tensor-parallel-size', '1',
    '--gpu-memory-utilization', '0.85',
    '--max-model-len', '2048',
    '--enforce-eager',
    '--dtype', 'bfloat16',
    '--port', '8000',
    '--host', '0.0.0.0',
]
sys.exit(main())
"

curl -sf http://127.0.0.1:8000/v1/models | jq .

curl -s http://127.0.0.1:8000/v1/completions \
  -H 'Content-Type: application/json' \
  -d '{
    "model":"/path/to/model/",
    "prompt":"你好，请介绍一下你自己",
    "temperature":0,
    "max_tokens":128
  }' | jq -r '.choices[0].text'

你好，我是MiniCPM系列模型，由面壁智能和OpenBMB开源社区开发。...

curl -s http://127.0.0.1:8000/v1/completions \
  -H 'Content-Type: application/json' \
  -d '{
    "model":"/path/to/model/",
    "prompt":"What is quantum computing?",
    "temperature":0,
    "max_tokens":128
  }' | jq -r '.choices[0].text'

# 安装依赖
pip install torch torch_npu transformers datasets accelerate

# CPU 基线
python3 << 'EOF'
import torch
import torch.nn.functional as F
from transformers import AutoModelForCausalLM, AutoTokenizer
from datasets import load_dataset

device = "cpu"
model = AutoModelForCausalLM.from_pretrained(
    "/path/to/model/",
    trust_remote_code=True,
    torch_dtype=torch.bfloat16,
).to(device).eval()
tokenizer = AutoTokenizer.from_pretrained("/path/to/model/", trust_remote_code=True)

# WikiText-2 test set
ds = load_dataset("wikitext", "wikitext-2-raw-v1", split="test")
texts = [t for t in ds["text"] if len(t.strip()) > 50][:50]

total_nll = 0.0
total_tokens = 0
with torch.no_grad():
    for text in texts:
        inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=512)
        labels = inputs["input_ids"]
        outputs = model(**inputs, labels=labels)
        total_nll += outputs.loss.item() * labels.size(1)
        total_tokens += labels.size(1)

ppl_cpu = round(torch.exp(torch.tensor(total_nll / total_tokens)).item(), 2)
print(f"CPU PPL: {ppl_cpu}")
EOF

# NPU (Ascend) 推理（使用 torch_npu）
python3 << 'EOF'
import torch
import torch_npu
import torch.nn.functional as F
from transformers import AutoModelForCausalLM, AutoTokenizer
from datasets import load_dataset

device = "npu:0"
model = AutoModelForCausalLM.from_pretrained(
    "/path/to/model/",
    trust_remote_code=True,
    torch_dtype=torch.bfloat16,
).to(device).eval()
tokenizer = AutoTokenizer.from_pretrained("/path/to/model/", trust_remote_code=True)

ds = load_dataset("wikitext", "wikitext-2-raw-v1", split="test")
texts = [t for t in ds["text"] if len(t.strip()) > 50][:50]

total_nll = 0.0
total_tokens = 0
with torch.no_grad():
    for text in texts:
        inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=512)
        inputs = {k: v.to(device) for k, v in inputs.items()}
        labels = inputs["input_ids"]
        outputs = model(**inputs, labels=labels)
        total_nll += outputs.loss.item() * labels.size(1)
        total_tokens += labels.size(1)

ppl_npu = round(torch.exp(torch.tensor(total_nll / total_tokens)).item(), 2)
print(f"NPU PPL: {ppl_npu}")
EOF

# 注册钩子后逐层对比 hidden_states
def collect_hidden(name, cache):
    def hook(m, inp, out):
        cache[name] = out.detach().cpu()
    return hook

# 在模型加载后注册所有 transformer layer 的钩子
# 使用同一输入分别在 CPU 和 NPU 上前向，收集各层输出
# 计算每层的 cosine_similarity 和 MSE