KORani-v3-13B - AtomGit AI社区

1、适配昇腾处理器：Ascend310、Ascend910系列 2、开发环境：Ascend-cann-toolkit_xxx、Ascend-cann-kernels-xxx(可选)、python3.8 3、下载代码：git clone https://modelers.cn/ShanXi/KORani-v3-13B.git 4、安装依赖：pip install -r examples/requirements.txt 5、推理测试：python examples/inference.py 6、推理脚本：

import argparse import torch from openmind import pipeline, is_torch_npu_available from transformers import PreTrainedTokenizerFast from transformers import BartForConditionalGeneration from transformers import AutoTokenizer, AutoModelForCausalLM from openmind_hub import snapshot_download

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--model_name_or_path",type=str,help="模型路径",default="./",)
    args = parser.parse_args()
    return args


if is_torch_npu_available():
    device = "npu:0"
else:
    device = "cpu"

args = parse_args()

if args.model_name_or_path:
    model_path = args.model_name_or_path
else:
    model_path = snapshot_download('ShanXi/KORani-v3-13B',revision='main',resume_donwload=True,ignore_patterns=['*.h5','*.ot','*.msgpack'])

model = AutoModelForCausalLM.from_pretrained(model_path, device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True)
systemPrompt = "당신은 유능한 AI입니다."
prompt = "지렁이도 밟으면 꿈틀하나요?"
outputs = model.generate(
  **tokenizer(
    f"[INST] <<SYS>>\n{systemPrompt}\n<</SYS>>\n\n{prompt} [/INST] ",
    return_tensors='pt'
  ).to('npu'),
  max_new_tokens=256,
  temperature=0.2,
  top_p=1,
  do_sample=True
)
print(tokenizer.decode(outputs[0]))

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--model_name_or_path",type=str,help="模型路径",default="./",)
    args = parser.parse_args()
    return args


if is_torch_npu_available():
    device = "npu:0"
else:
    device = "cpu"

args = parse_args()

if args.model_name_or_path:
    model_path = args.model_name_or_path
else:
    model_path = snapshot_download('ShanXi/KORani-v3-13B',revision='main',resume_donwload=True,ignore_patterns=['*.h5','*.ot','*.msgpack'])

model = AutoModelForCausalLM.from_pretrained(model_path, device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True)
systemPrompt = "당신은 유능한 AI입니다."
prompt = "지렁이도 밟으면 꿈틀하나요?"
outputs = model.generate(
  **tokenizer(
    f"[INST] <<SYS>>\n{systemPrompt}\n<</SYS>>\n\n{prompt} [/INST] ",
    return_tensors='pt'
  ).to('npu'),
  max_new_tokens=256,
  temperature=0.2,
  top_p=1,
  do_sample=True
)
print(tokenizer.decode(outputs[0]))