Time-MoE 昇腾/NPU 迁移部署与 NV GPU 对比指南

docker pull quay.io/ascend/cann:8.5.0-910b-ubuntu22.04-py3.11

docker run -it -d \
    --shm-size=500g \
    --name TimeMoE \
    --privileged \
    --entrypoint /bin/bash \
    --net=host \
    --device /dev/davinci0 \
    --device /dev/davinci1 \
    --device /dev/davinci2 \
    --device /dev/davinci3 \
    --device /dev/davinci4 \
    --device /dev/davinci5 \
    --device /dev/davinci6 \
    --device /dev/davinci7 \
    --device /dev/davinci_manager \
    --device /dev/devmm_svm \
    --device /dev/hisi_hdc \
    -v /usr/local/dcmi:/usr/local/dcmi \
    -v /usr/local/Ascend/driver/tools/hccn_tool:/usr/local/Ascend/driver/tools/hccn_tool \
    -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
    -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
    -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
    -v /etc/ascend_install.info:/etc/ascend_install.info \
    -v /opt/:/opt/ \
    quay.io/ascend/cann:8.5.0-910b-ubuntu22.04-py3.11

docker exec -itu root TimeMoE bash

cd /home
git clone https://github.com/Time-MoE/Time-MoE.git
cd Time-MoE
git checkout 915bfda4c78a544d62a2bec6ab22948423059236

cd /home
git clone https://github.com/zhouhaoyi/ETDataset.git

mkdir Time-300B
cd Time-300B
pip install huggingface_hub
export HF_ENDPOINT=https://hf-mirror.com
export HF_HOME=/tmp/hf_cache
hf download Maple728/Time-300B --repo-type dataset --include "sales/*"

cd /home/Time-300B
cp -rL /tmp/hf_cache/hub/datasets--Maple728--Time-300B/snapshots/*/sales ./sales

sales/
├── dominick/
├── favorita_transactions/
├── favorita_sales/
......

cd /home/Time-MoE
HF_ENDPOINT=https://hf-mirror.com python -c "
from huggingface_hub import snapshot_download
snapshot_download(
    repo_id='Maple728/TimeMoE-50M',
    local_dir='./TimeMoE-50M',
    resume_download=True,
    local_dir_use_symlinks=False
)
"

wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh

chmod +x Miniconda3-latest-Linux-aarch64.sh

bash Miniconda3-latest-Linux-aarch64.sh

source ~/miniconda3/etc/profile.d/conda.sh

conda create -n Time-MoE python=3.11 -y

cd /home/Time-MoE
pip install -r requirements.txt
pip install torch==2.8.0 torch_npu==2.8.0
pip install transformers==4.57.0

from torch_npu.contrib import transfer_to_npu

from torch_npu.contrib import transfer_to_npu

# 执行推理
python run_eval.py -d /home/ETDataset/ETT-small/ETTh1.csv --model ./TimeMoE-50M -p 96

# evaluation_strategy=train_config.get("evaluation_strategy", 'no'),

  else:
      model = TimeMoeForPrediction.from_pretrained(model_path, **kwargs)
      # Guard against corrupted weight restore
      import torch.nn as nn
      object.__setattr__(model, 'loss_function', nn.HuberLoss(reduction='none', delta=2.0))

nohup torchrun --master_addr=127.0.0.1 --master_port=11011 --node_rank=0 --nproc_per_node=8 main.py --data_path /home/Time-300B/sales --model_path /home/Time-MoE/TimeMoE-50M > run.log 2>&1 &

{'loss': 0.4114, 'grad_norm': 0.25300517678260803, 'learning_rate': 0.0001, 'epoch': 0.0}
{'loss': 0.4517, 'grad_norm': 0.2968370318412781, 'learning_rate': 9.99999584434885e-05, 'epoch': 0.0}
{'loss': 0.4112, 'grad_norm': 0.1269938200712204, 'learning_rate': 9.999983377409216e-05, 'epoch': 0.0}
{'loss': 0.4163, 'grad_norm': 0.13979601860046387, 'learning_rate': 9.999962599222543e-05, 'epoch': 0.0}
......

......
{'loss': 0.3931, 'grad_norm': 0.1221962720155716, 'learning_rate': 5.000016622590785e-05, 'epoch': 1.0}
{'loss': 0.3587, 'grad_norm': 0.10206776112318039, 'learning_rate': 5.00000415565115e-05, 'epoch': 1.0}
{'train_runtime': 999.1059, 'train_samples_per_second': 110.309, 'train_steps_per_second': 1.725, 'train_loss': 0.3853655972940176, 'epoch': 1.0}
100%|██████████| 1723/1723 [16:39<00:00,  1.72it/s]

docker pull nvcr.io/nvidia/pytorch:25.12-py3

docker run -itd --runtime nvidia --gpus all --name TimeMoE --shm-size 500g --ulimit memlock=-1 --ulimit stack=67108864 --net host --entrypoint /bin/bash -v /home/:/home/ nvcr.io/nvidia/pytorch:25.12-py3
docker start TimeMoE
docker exec -itu root TimeMoE bash

cd /workspace
git clone https://github.com/Time-MoE/Time-MoE.git
cd Time-MoE
git checkout 915bfda4c78a544d62a2bec6ab22948423059236

cd /workspace
git clone https://github.com/zhouhaoyi/ETDataset.git

cd /workspace
mkdir Time-300B
cd Time-300B
pip install huggingface_hub
export HF_ENDPOINT=https://hf-mirror.com
export HF_HOME=/tmp/hf_cache
hf download Maple728/Time-300B --repo-type dataset --include "sales/*"

wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh

chmod +x Miniconda3-latest-Linux-x86_64.sh

bash Miniconda3-latest-Linux-x86_64.sh

source ~/miniconda3/etc/profile.d/conda.sh

conda create -n Time-MoE python=3.11 -y

cd /workspace/Time-MoE
pip install -r requirements.txt
pip install transformers==4.57.0

# 执行推理
python run_eval.py -d /workspace/ETDataset/ETT-small/ETTh1.csv -p 96

nohup torchrun --master_addr=127.0.0.1 --master_port=11011 --node_rank=0 --nproc_per_node=8 main.py --data_path /home/Time-300B/sales --model_path /home/Time-MoE/TimeMoE-50M > run.log 2>&1 &

{'loss': 0.4086, 'grad_norm': 0.23242798447608948, 'learning_rate': 0.0001, 'epoch': 0.0}
{'loss': 0.4247, 'grad_norm': 0.31524255871772766, 'learning_rate': 9.99999584434885e-05, 'epoch': 0.0}
{'loss': 0.4202, 'grad_norm': 0.2507534325122833, 'learning_rate': 9.999983377409216e-05, 'epoch': 0.0}
{'loss': 0.4035, 'grad_norm': 0.410408616065979, 'learning_rate': 9.999962599222543e-05, 'epoch': 0.0}
{'loss': 0.4008, 'grad_norm': 0.18601377308368683, 'learning_rate': 9.999933509857908e-05, 'epoch': 0.0}
......

......
{'loss': 0.3977, 'grad_norm': 0.11204443871974945, 'learning_rate': 5.000037400777459e-05, 'epoch': 1.0}
{'loss': 0.3645, 'grad_norm': 0.09452001005411148, 'learning_rate': 5.000016622590785e-05, 'epoch': 1.0}
{'loss': 0.3586, 'grad_norm': 0.1820800006389618, 'learning_rate': 5.00000415565115e-05, 'epoch': 1.0}
{'train_runtime': 339.0897, 'train_samples_per_second': 325.017, 'train_steps_per_second': 5.081, 'train_loss': 0.3852101174192379, 'epoch': 1.0}
100%|██████████| 1723/1723 [05:39<00:00,  5.08it/s]
2026-04-16 09:44:26,507 - log_util.py[pid:3606;line:52:log_in_local_rank_0] - INFO: Saving model to logs/time_moe

date,HUFL,HULL,MUFL,MULL,LUFL,LULL,OT

import torch
from safetensors.torch import load_file
from collections import defaultdict
from tqdm import tqdm

# ===== 配置 =====
gpu_path = "/workspace/Time-MoE/logs/time_moe/model.safetensors"
npu_path = "/workspace/weight/logs/time_moe/model.safetensors"

sample_per_tensor = 1000   # 用于quantile采样
rel_eps = 1e-3             # relative error过滤阈值
topk_num = 10

# ===== 加载 =====
print("Loading weights...")
gpu_weights = load_file(gpu_path)
npu_weights = load_file(npu_path)

gpu_keys = set(gpu_weights.keys())
npu_keys = set(npu_weights.keys())

print("参数名称一致:", gpu_keys == npu_keys)

common_keys = gpu_keys & npu_keys
print(f"参数数量: {len(common_keys)}")

# ===== 全局统计 =====
global_max = 0
global_mean = 0
global_rel_mean = 0
count = 0

# ===== TopK =====
topk = []

# ===== 分模块统计 =====
module_stats = defaultdict(list)

# ===== quantile采样 =====
samples = []

print("\nComparing...")

for key in tqdm(common_keys):
    g = gpu_weights[key].to(torch.float32).cpu()
    n = npu_weights[key].to(torch.float32).cpu()

    if g.shape != n.shape:
        print(f"[Shape mismatch] {key}: {g.shape} vs {n.shape}")
        continue

    diff = torch.abs(g - n)

    max_d = diff.max().item()
    mean_d = diff.mean().item()

    # ===== relative error（过滤小值）=====
    mask = torch.abs(g) > rel_eps
    if mask.sum() > 0:
        rel = diff[mask] / torch.abs(g[mask])
        rel_mean = rel.mean().item()
    else:
        rel_mean = 0

    # ===== 全局 =====
    global_max = max(global_max, max_d)
    global_mean += mean_d
    global_rel_mean += rel_mean
    count += 1

    # ===== TopK =====
    topk.append((max_d, key))

    # ===== 模块分类 =====
    if "self_attn" in key:
        module_stats["attention"].append(mean_d)
    elif "ffn_layer.experts" in key:
        module_stats["moe_expert"].append(mean_d)
    elif "shared_expert" in key:
        module_stats["shared_expert"].append(mean_d)
    elif "layernorm" in key:
        module_stats["layernorm"].append(mean_d)
    else:
        module_stats["others"].append(mean_d)

    # ===== quantile采样 =====
    flat = diff.view(-1)
    if flat.numel() > sample_per_tensor:
        idx = torch.randint(0, flat.numel(), (sample_per_tensor,))
        flat = flat[idx]
    samples.append(flat)

# ===== 汇总 =====
print("\n===== 全局统计 =====")
print(f"平均误差: {global_mean / count:.6e}")
print(f"平均相对误差(过滤<{rel_eps}): {global_rel_mean / count:.6e}")
print(f"最大误差: {global_max:.6e}")

# ===== 分位数（采样）=====
samples = torch.cat(samples)

q = torch.quantile(samples, torch.tensor([0.5, 0.9, 0.99]))

print("\n===== 分位数（采样） =====")
print(f"P50: {q[0].item():.6e}")
print(f"P90: {q[1].item():.6e}")
print(f"P99: {q[2].item():.6e}")

# ===== 模块统计 =====
print("\n===== 模块误差 =====")
for k, v in module_stats.items():
    if len(v) > 0:
        print(f"{k:15s}: mean={sum(v)/len(v):.6e}, count={len(v)}")

# ===== Top-K =====
topk.sort(reverse=True)

print(f"\n===== Top {topk_num} 最大误差参数 =====")
for d, k in topk[:topk_num]:
    print(f"{k}: {d:.6e}")

Loading weights...
参数名称一致: True
参数数量: 463

Comparing...
100%|████████████████████████████████████████████████████████████████████████████████████| 463/463 [00:00<00:00, 874.70it/s]

===== 全局统计 =====
平均误差: 7.039339e-03
平均相对误差(过滤<0.001): 2.721881e-01
最大误差: 1.691253e-01

===== 分位数（采样） =====
P50: 5.434263e-03
P90: 1.741766e-02
P99: 3.062531e-02

===== 模块误差 =====
moe_expert     : mean=8.156686e-03, count=288
attention      : mean=4.904049e-03, count=84
others         : mean=4.401606e-03, count=19
shared_expert  : mean=6.365904e-03, count=48
layernorm      : mean=4.539759e-03, count=24

===== Top 10 最大误差参数 =====
model.layers.11.ffn_layer.experts.7.down_proj.weight: 1.691253e-01
model.layers.11.ffn_layer.experts.7.gate_proj.weight: 1.189974e-01
model.layers.11.ffn_layer.shared_expert.up_proj.weight: 1.134196e-01
model.layers.11.ffn_layer.shared_expert.down_proj.weight: 1.080939e-01
model.layers.11.ffn_layer.experts.3.down_proj.weight: 9.563965e-02
model.layers.11.ffn_layer.shared_expert.gate_proj.weight: 9.188609e-02
model.layers.11.ffn_layer.experts.2.down_proj.weight: 9.044161e-02
model.layers.11.ffn_layer.experts.7.up_proj.weight: 8.590467e-02
model.layers.11.ffn_layer.experts.6.down_proj.weight: 8.155429e-02
model.layers.3.ffn_layer.experts.2.gate_proj.weight: 8.127359e-02

export PIP_INDEX_URL=https://mirrors.aliyun.com/pypi/simple
export PIP_TRUSTED_HOST=mirrors.aliyun.com

import torch
import torch.nn as nn

class NPUHuberLoss(nn.Module):
    def __init__(self, delta=2.0, reduction='none'):
        super().__init__()
        self.delta = delta
        self.reduction = reduction

    def forward(self, input, target):
        diff = input - target
        abs_diff = diff.abs()

        loss = torch.where(
            abs_diff <= self.delta,
            0.5 * diff * diff,
            self.delta * abs_diff - 0.5 * self.delta * self.delta
        )
        if self.reduction == 'mean':
            return loss.mean()
        elif self.reduction == 'sum':
            return loss.sum()
        else:
            return loss

from .HuberLoss import NPUHuberLoss

# 替换前
self.loss_function = torch.nn.HuberLoss(reduction='none', delta=2.0)

# 替换后
self.loss_function = NPUHuberLoss()