HuggingFace镜像/cl100k_base
模型介绍文件和版本分析
下载使用量0

cl100k_base 作为 transformers GPT2 分词器

cl100k_base 词汇表通过 Xenova 的此代码从 tiktoken 转换为 hf。

from transformers import GPT2TokenizerFast, AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("BEE-spoke-data/cl100k_base")
# if issues, try GPT2TokenizerFast directly

详情

GPT2TokenizerFast(
    name_or_path="BEE-spoke-data/cl100k_base",
    vocab_size=100261,
    model_max_length=8192,
    is_fast=True,
    padding_side="right",
    truncation_side="right",
    special_tokens={
        "bos_token": "<|endoftext|>",
        "eos_token": "<|endoftext|>",
        "unk_token": "<|endoftext|>",
    },
    clean_up_tokenization_spaces=True,
    added_tokens_decoder={
        "100257": AddedToken(
            "<|endoftext|>",
            rstrip=False,
            lstrip=False,
            single_word=False,
            normalized=False,
            special=True,
        ),
        "100258": AddedToken(
            "<|fim_prefix|>",
            rstrip=False,
            lstrip=False,
            single_word=False,
            normalized=False,
            special=True,
        ),
        "100259": AddedToken(
            "<|fim_middle|>",
            rstrip=False,
            lstrip=False,
            single_word=False,
            normalized=False,
            special=True,
        ),
        "100260": AddedToken(
            "<|fim_suffix|>",
            rstrip=False,
            lstrip=False,
            single_word=False,
            normalized=False,
            special=True,
        ),
        "100276": AddedToken(
            "<|endofprompt|>",
            rstrip=False,
            lstrip=False,
            single_word=False,
            normalized=False,
            special=True,
        ),
    },
)