GLM-OCR-8bit 昇腾 NPU 适配文档

# 安装 vLLM-Ascend
pip install vllm==0.18.0
pip install vllm-ascend==0.18.0rc1

# 安装 transformers (支持 GLM-OCR)
pip install git+https://github.com/huggingface/transformers.git

# 安装其他依赖
pip install requests numpy pillow

from modelscope import snapshot_download
model_dir = snapshot_download('mlx-community/GLM-OCR-8bit', cache_dir='./models')

# 使用镜像源
export HF_ENDPOINT=https://hf-mirror.com
huggingface-cli download mlx-community/GLM-OCR-8bit --local-dir ./models/GLM-OCR-8bit

# 原始权重
git clone https://atomgit.com/zai-org/GLM-OCR.git

vllm serve /path/to/GLM-OCR-bf16 \
  --dtype bfloat16 \
  --tensor-parallel-size 1 \
  --max-model-len 4096 \
  --max-num-seqs 16 \
  --port 8000 \
  --trust-remote-code \
  --served-model-name glm-ocr-8bit \
  --enforce-eager

# 检查服务状态
curl http://localhost:8000/v1/models

# 测试推理
curl -X POST http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "glm-ocr-8bit",
    "messages": [{"role": "user", "content": "say hi"}],
    "max_tokens": 16
  }'

python inference.py \
  --image /path/to/image.jpg \
  --prompt "请识别图片中的文字内容" \
  --model glm-ocr-8bit \
  --output result.json

import requests
import base64

def ocr_image(image_path, prompt="请识别图片中的文字内容"):
    with open(image_path, "rb") as f:
        image_data = base64.b64encode(f.read()).decode()

    response = requests.post(
        "http://localhost:8000/v1/chat/completions",
        json={
            "model": "glm-ocr-8bit",
            "messages": [{
                "role": "user",
                "content": [
                    {"type": "text", "text": prompt},
                    {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_data}"}}
                ]
            }],
            "max_tokens": 2048
        }
    )

    return response.json()["choices"][0]["message"]["content"]

# 使用示例
result = ocr_image("test.jpg")
print(result)

A: 需要安装最新版本的 transformers:
pip install git+https://github.com/huggingface/transformers.git

A: 修改 processor_config.json，将 processor_class 改为 "Glm46VProcessor"

A: 使用 bf16 版本的模型，或禁用量化:
vllm serve ... --quantization None

A: 减少 max-model-len 或 max-num-seqs 参数:
vllm serve ... --max-model-len 2048 --max-num-seqs 8

vllm serve ... --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY"}'

vllm serve ... --enable-prefix-caching

vllm serve ... --max-num-batched-tokens 8192