Qwen3-8B-MLX-4bit

# 1. 安装依赖
pip install mlx-lm transformers safetensors torch

# 2. 运行反量化脚本 (已包含在本仓库)
python dequantize_mlx.py \
    --input /path/to/Qwen3-8B-MLX-4bit \
    --output /path/to/Qwen3-8B-BF16

pip install modelscope
python -c "
from modelscope import snapshot_download
model_dir = snapshot_download('Qwen/Qwen3-8B', cache_dir='./Qwen3-8B-BF16')
print('下载完成:', model_dir)
"

# 安装 vLLM-Ascend
pip install vllm vllm-ascend

# 启动推理服务 (Atlas A2/A3 单卡)
vllm serve /path/to/Qwen3-8B-BF16 \
    --tensor-parallel-size 1 \
    --max-model-len 8192 \
    --dtype bfloat16 \
    --gpu-memory-utilization 0.9 \
    --trust-remote-code

# 测试推理
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "/path/to/Qwen3-8B-BF16",
    "messages": [{"role": "user", "content": "Hello!"}],
    "max_tokens": 100
  }'

KeyError: 'qwen3'

pip install --upgrade transformers mlx_lm

from mlx_lm import load, generate

model, tokenizer = load("Qwen/Qwen3-8B-MLX-4bit")
prompt = "Hello, please introduce yourself and tell me what you can do."

if tokenizer.chat_template is not None:
    messages = [{"role": "user", "content": prompt}]
    prompt = tokenizer.apply_chat_template(
        messages,
        add_generation_prompt=True
    )

response = generate(
    model,
    tokenizer,
    prompt=prompt,
    verbose=True,
    max_tokens=1024
)

print(response)

text = tokenizer.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=True,
    enable_thinking=True  # True is the default value for enable_thinking
)

text = tokenizer.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=True,
    enable_thinking=False  # Setting enable_thinking=False disables thinking mode
)

from mlx_lm import load, generate


class QwenChatbot:
    def __init__(self, model_name="Qwen/Qwen3-8B-MLX-4bit"):
        self.model, self.tokenizer = load(model_name)
        self.history = []

    def generate_response(self, user_input):
        messages = self.history + [{"role": "user", "content": user_input}]

        text = self.tokenizer.apply_chat_template(
            messages,
            tokenize=False,
            add_generation_prompt=True
        )

        response = generate(
            self.model,
            self.tokenizer,
            prompt=text,
            verbose=True,
            max_tokens=32768
        )
        # Update history
        self.history.append({"role": "user", "content": user_input})
        self.history.append({"role": "assistant", "content": response})

        return response


# Example Usage
if __name__ == "__main__":
    chatbot = QwenChatbot()

    # First input (without /think or /no_think tags, thinking mode is enabled by default)
    user_input_1 = "How many 'r's are in strawberries?"
    print(f"User: {user_input_1}")
    response_1 = chatbot.generate_response(user_input_1)
    print(f"Bot: {response_1}")
    print("----------------------")

    # Second input with /no_think
    user_input_2 = "Then, how many 'r's are in blueberries? /no_think"
    print(f"User: {user_input_2}")
    response_2 = chatbot.generate_response(user_input_2)
    print(f"Bot: {response_2}")
    print("----------------------")

    # Third input with /think
    user_input_3 = "Really? /think"
    print(f"User: {user_input_3}")
    response_3 = chatbot.generate_response(user_input_3)
    print(f"Bot: {response_3}")

from qwen_agent.agents import Assistant

# Define LLM
llm_cfg = {
    "model": "Qwen3-8B-MLX-4bit",

    # Use the endpoint provided by Alibaba Model Studio:
    # "model_type": "qwen_dashscope",
    # "api_key": os.getenv("DASHSCOPE_API_KEY"),

    # Use a custom endpoint compatible with OpenAI API:
    "model_server": "http://localhost:8000/v1",  # api_base
    "api_key": "EMPTY",

    # Other parameters:
    # "generate_cfg": {
    #     # Add: When the response content is `this is the answer;
    #     # Do not add: When the response has been separated by reasoning_content and content.
    #     "thought_in_content": True,
    # },
}

# Define Tools
tools = [
    {
        "mcpServers": {  # You can specify the MCP configuration file
            "time": {
                "command": "uvx",
                "args": ["mcp-server-time", "--local-timezone=Asia/Shanghai"],
            },
            "fetch": {
                "command": "uvx",
                "args": ["mcp-server-fetch"],
            },
        }
    },
    "code_interpreter",  # Built-in tools
]

# Define Agent
bot = Assistant(llm=llm_cfg, function_list=tools)

# Streaming generation
messages = [
    {
        "role": "user",
        "content": "https://qwenlm.github.io/blog/ Introduce the latest developments of Qwen",
    }
]

for responses in bot.run(messages=messages):
    pass

print(responses)

@misc{qwen3technicalreport,
      title={Qwen3 Technical Report}, 
      author={Qwen Team},
      year={2025},
      eprint={2505.09388},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2505.09388}, 
}