LongCat-Next

# (Install python=3.10, ffffmpeg<7, soundfile==0.13.1)
conda env create -f environment.yml -v

# (Install torch and other pip dependencies)
pip install -r requirements.txt && pip install -r requirements-post.txt --no-build-isolation

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, AutoProcessor

# Load model
model_name = "meituan-longcat/LongCat-Next"
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    torch_dtype=torch.bfloat16,
    device_map="auto",
    trust_remote_code=True,
)
model.eval()

tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, fix_mistral_regex=True)
model.text_tokenizer = tokenizer # Dynamic binding
processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)

# Set messages
messages = [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "What book is this?<longcat_img_start>./assets/book.png<longcat_img_end>"}
]

# Apply chat-template
text_input = tokenizer.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=True,
)
print(f"{text_input=}")

# Preprocessing
text_inputs, visual_inputs, audio_inputs = processor(text=text_input, return_tensors="pt")
text_inputs = text_inputs.to(model.device)
if visual_inputs is not None:
    visual_inputs = visual_inputs.to(model.device)
if audio_inputs is not None:
    audio_inputs = audio_inputs.to(model.device)

# AR
with torch.no_grad():
    outputs = model.generate(
        input_ids=text_inputs["input_ids"],
        visual_inputs=visual_inputs,
        audio_inputs=audio_inputs,
        return_dict_in_generate=True,
    )

# Text decoding
output_input_ids = outputs.sequences
text_output = tokenizer.decode(output_input_ids[0][len(text_inputs["input_ids"][0]):], skip_special_tokens=True)
print(f"{text_output=}")

# Images decoding
output_visual_ids = outputs.visual_ids
if output_visual_ids.size(0) > 0:
    image_path_list = model.model.decode_visual_ids_and_save(
        output_visual_ids,
        save_prefix="./output_image",
        **model.generation_config.visual_generation_config["custom_params"],
    )
    print(f"{image_path_list=}")

# Audio decoding
output_audio_text_ids = outputs.audio_text_ids
output_audio_ids = outputs.audio_ids
if output_audio_text_ids.size(-1) > 0:
    audio_text = tokenizer.decode(output_audio_text_ids[0], skip_special_tokens=True)
    print(f"{audio_text=}")
if output_audio_ids.size(0) > 0:
    audio_path_list = model.model.decode_audio_ids_and_save(
        output_audio_ids,
        save_prefix="./output_audio",
        **model.generation_config.audio_generation_config["custom_params"],
    )
    print(f"{audio_path_list=}")

from parse_model_response import parse_model_response

tools = [
    {
        "type": "function",
        "function": {
            "name": "func_add",
            "description": "Calculate the sum of two numbers",
            "parameters": {
                "type": "object",
                "properties": {
                    "x1": {"type": "number", "description": "The first addend"},
                    "x2": {"type": "number", "description": "The second addend"}
                },
                "required": ["x1", "x2"]
            }
        }
    }
]
messages = [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "Please tell me what is $$125679 + 234519$$?"},
    {
        "role": "assistant",
        "content": "I'll calculate the sum of 125679 and 234519 for you.",
        "tool_calls": [{"type": "function", "function": {"name": "func_add", "arguments": {"x1": 125679, "x2": 234519}}}]
    },
    {"role": "tool", "name": "func_add", "content": '{"ans": 360198}'}
]

text_input = tokenizer.apply_chat_template(
    messages,
    tools=tools, # add tools here
    tokenize=False,
    add_generation_prompt=True,
)
print(f"{text_input=}")


# Preprocessing - AR - Text decoding
...

# Results parsing
parsed_message = parse_model_response(text_output.strip("\n"), tools)
print(f"{parsed_message=}")

# Simply replace the messages in the main example with the messages below.
messages = [
    {"role": "user", "content": "What book is this?<longcat_img_start>./assets/book.png<longcat_img_end>"}
]

# Simply replace the messages in the main example with the messages below.
# Suffix user content with '<longcat_img_start>' to force image generation.
messages = [
    {"role": "system", "content": ""},
    {"role": "user", "content": "A small kitten sitting naturally on a moss-covered forest floor, centered in the frame, holding a rectangular wooden sign gently with its front paws resting over the top edge. The kitten has soft, fluffy fur, a natural relaxed posture, and a calm, curious expression with a slightly open mouth (not exaggerated), looking directly at the camera.\n\nThe sign is positioned firmly in front of the kitten\'s chest, supported by its paws, with realistic contact and no floating effect. The board reads \"LongCat-Next: Lexicalizing Modalities as Discrete Tokens\" in clean, sharp black text, perfectly legible.\n\nThe environment is a lush forest with tall trees, ferns, and soft green foliage. The ground is covered with moss and small plants. Background softly blurred with natural depth of field. Lighting is soft, diffused sunlight filtering through the trees, creating gentle highlights and shadows. Realistic photography style, natural colors, high detail, no cartoonish exaggeration.<longcat_img_start>"}
]

# Simply replace the messages in the main example with the messages below.
messages = [
    {"role": "user", "content": "<longcat_audio_start>./assets/math1.wav<longcat_audio_end>"}
]

# Simply replace the messages in the main example with the messages below.
# Suffix user content with '<longcat_audiogen_start>' to force audio generation.
messages = [
    {"role": "system", "content": "Replicate the voice in the audio clip to formulate an answer:<longcat_audio_start>./assets/system_audio.wav<longcat_audio_end>"},
    {"role": "user", "content": "<longcat_audio_start>./assets/math1.wav<longcat_audio_end><longcat_audiogen_start>"}
]

@misc{meituanlongcatteam2026longcatnextlexicalizingmodalitiesdiscrete,
      title={LongCat-Next: Lexicalizing Modalities as Discrete Tokens}, 
      author={Meituan LongCat Team},
      year={2026},
      eprint={2603.27538},
      archivePrefix={arXiv},
      primaryClass={cs.CV},
      url={https://arxiv.org/abs/2603.27538}, 
}