Fun-CosyVoice3-0.5B-2512: 基于昇腾 NPU 的高质量 Zero-Shot 语音合成

# 安装依赖
pip install pyworld diffusers lightning soundfile onnx onnxruntime
# NPU 环境依赖（已安装）
pip install torch-npu

pretrained_models/Fun-CosyVoice3-0.5B/
├── cosyvoice3.yaml                     # 模型配置
├── configuration.json                  # 框架配置
├── README.md                           # 官方 README
├── llm.pt                              # LLM 权重 (~1.9GB)
├── llm.rl.pt                           # LLM RL 权重 (~1.9GB)
├── flow.pt                             # 流模型权重 (~1.3GB)
├── flow.decoder.estimator.fp32.onnx    # 流解码器 ONNX (~1.3GB)
├── hift.pt                             # HiFT 声码器 (~80MB)
├── campplus.onnx                       # 说话人嵌入提取 (~27MB)
├── speech_tokenizer_v3.onnx            # 语音 tokenizer (~925MB)
├── speech_tokenizer_v3.batch.onnx      # 批量语音 tokenizer (~925MB)
├── CosyVoice-BlankEN/                  # Qwen2 基座权重
│   ├── model.safetensors               # (~943MB)
│   ├── config.json
│   ├── tokenizer_config.json
│   ├── vocab.json
│   ├── merges.txt
│   └── generation_config.json
└── asset/
    └── zero_shot_prompt.wav            # Zero-shot 参考音频

# Zero-Shot 语音克隆 (NPU)
python3 inference.py \
  --model_dir pretrained_models/Fun-CosyVoice3-0.5B \
  --mode zero_shot \
  --device npu \
  --tts_text "八百标兵奔北坡，北坡炮兵并排跑，炮兵怕把标兵碰，标兵怕碰炮兵炮。" \
  --prompt_text "You are a helpful assistant.<|endofprompt|>希望你以后能够做的比我还好呦。" \
  --prompt_wav pretrained_models/Fun-CosyVoice3-0.5B/asset/zero_shot_prompt.wav \
  --output_dir ./outputs

# CPU 基线测试
python3 inference.py \
  --model_dir pretrained_models/Fun-CosyVoice3-0.5B \
  --mode zero_shot \
  --device cpu \
  --tts_text "八百标兵奔北坡，北坡炮兵并排跑，炮兵怕把标兵碰，标兵怕碰炮兵炮。" \
  --prompt_text "You are a helpful assistant.<|endofprompt|>希望你以后能够做的比我还好呦。" \
  --prompt_wav pretrained_models/Fun-CosyVoice3-0.5B/asset/zero_shot_prompt.wav \
  --output_dir ./outputs

# 性能基准测试 (NPU)
python3 inference.py \
  --model_dir pretrained_models/Fun-CosyVoice3-0.5B \
  --mode zero_shot \
  --device npu \
  --tts_text "八百标兵奔北坡，北坡炮兵并排跑，炮兵怕把标兵碰，标兵怕碰炮兵炮。" \
  --prompt_text "You are a helpful assistant.<|endofprompt|>希望你以后能够做的比我还好呦。" \
  --prompt_wav pretrained_models/Fun-CosyVoice3-0.5B/asset/zero_shot_prompt.wav \
  --output_dir ./outputs \
  --benchmark --warmup 1 --iterations 3

{
  "sample_count": 250560,
  "sample_rate": 24000,
  "metrics": {
    "mse": 0.000000,
    "mae": 0.000016,
    "max_diff": 0.002289,
    "relative_mae": 0.000009,
    "cosine_similarity": 0.999998,
    "snr_db": 57.589,
    "spectral_correlation": 0.999999,
    "mfcc_similarity": 1.000000
  }
}

[Info] Using device: npu:0
[Info] Model loaded in 20.82s
[Info] Model sample rate: 24000
[Info] Inference completed:
  Device: npu
  Audio duration: 10.440s
  Inference time: 14.582s
  RTF: 1.3968

[Info] Using device: cpu
[Info] Model loaded in 19.31s
[Info] Model sample rate: 24000
[Info] Inference completed:
  Device: cpu
  Audio duration: 10.440s
  Inference time: 15.801s
  RTF: 1.5135

=== Evaluation Results ===
Samples: 250560
  mse: 0.000000
  mae: 0.000016
  max_diff: 0.002289
  relative_mae: 0.000009
  cosine_similarity: 0.999998
  snr_db: 57.588930
  spectral_correlation: 0.999999
  mfcc_similarity: 1.000000

=== Pass Criteria ===
  relative_mae < 1%: PASS
  cosine_similarity > 0.99: PASS
  snr > 40dB: PASS
  spectral_correlation > 0.99: PASS

✅ NPU accuracy validation PASSED (error < 1%)

[Benchmark] Running 3 iterations...
  Iter 1/3: RTF=1.6181, Time=31.974s
  Iter 2/3: RTF=1.6683, Time=37.437s
  Iter 3/3: RTF=1.6094, Time=12.231s

Benchmark Summary:
{
  "rtf_mean": 1.632,
  "rtf_std": 0.026,
  "rtf_min": 1.609,
  "rtf_max": 1.668
}