CoGenAV-Ascend：面向NPU的音视频表征学习

pip install torch==2.9.0 torch_npu==2.9.0 numpy

import torch
import torch_npu
from models.cogenav import CoGenAV

# Load model on NPU
device = torch.device("npu:0")
model = CoGenAV(cfg_file="config/base.yaml", model_tensor="weights/base_cogen.pt")
model = model.to(device)
model.eval()

# Prepare input
video = torch.randn(1, 96, 96, 1, 750).to(device)   # [B, H, W, C, T]
audio = torch.randn(1, 80, 3000).to(device)           # [B, mel_bins, T]

# Run inference
with torch.no_grad():
    output = model(video, audio)  # [B, T', D]
    features = output.permute(0, 2, 1)  # [B, D, T'] for downstream tasks

# Full evaluation with accuracy verification
python inference.py --mode eval --model_size base --input_type cogenav_av

# Simple inference
python inference.py --mode infer --model_size base --input_type cogenav_v

# With real video/audio inputs
python inference.py --mode infer --video_path /path/to/video.mp4 --audio_path /path/to/audio.wav

cogenav/
├── inference.py          # Main NPU inference script with evaluation
├── test_npu_load.py      # NPU loading and functionality test
├── fairseq_compat.py     # NPU-compatible fairseq replacements
├── config_loader.py      # YAML config loader (omegaconf fallback)
├── models/
│   ├── cogenav.py        # CoGenAV model definition (NPU-adapted)
│   ├── resnet.py         # ResNet video encoder
│   └── sepformer.py      # SepFormer for AVSE/AVSS
├── config/
│   ├── base.yaml         # Base model config (123M params)
│   └── large.yaml        # Large model config
├── weights/
│   ├── base_cogen.pt     # Base model weights
│   └── large_cogen.pt    # Large model weights
└── README.md             # This file

@article{cogenav2025,
  title={CoGenAV: Contrastive-Generative Audio-Visual Representation Learning},
  journal={arXiv:2505.03186},
  year={2025}
}