cross-encoder/ms-marco-MiniLM-L12-v2 on Ascend 910B3

import torch
import torch_npu
from transformers import AutoModelForSequenceClassification, AutoTokenizer

model = AutoModelForSequenceClassification.from_pretrained("cross-encoder/ms-marco-MiniLM-L12-v2")
model = model.npu()
model.eval()

tokenizer = AutoTokenizer.from_pretrained("cross-encoder/ms-marco-MiniLM-L12-v2")
inputs = tokenizer(["query"], ["passage"], return_tensors="pt")
inputs = {k: v.npu() for k, v in inputs.items()}

with torch.no_grad():
    outputs = model(**inputs)
    score = outputs.logits.cpu().item()

pip install torch torch_npu transformers -i https://repo.huaweicloud.com/repository/pypi/simple/
export HF_ENDPOINT=https://hf-mirror.com

# NPU 推理
python inference.py --query "How many people live in Berlin?" --passage "Berlin has 3.5 million people."

# 从 JSON 文件批量推理
python inference.py --input pairs.json --output scores.json

# CPU 推理
python inference.py --query "Q" --passage "P" --device cpu

[
    {"query": "question", "passage": "relevant passage"},
    {"query": "question", "passage": "irrelevant passage"}
]

python inference.py --query "How many people live in Berlin?" --passage "Berlin had a population of 3.5 million."