audio-flamingo-3-hf

pip install --upgrade pip
pip install transformers==5.0.0rc1 accelerate

VLLM_USE_PRECOMPILED=1 uv pip install -U --pre \
  --override <(printf 'transformers>=5.0.0rc1\n') \
  "vllm[audio] @ git+https://github.com/vllm-project/vllm.git"

import os
from pathlib import Path

from vllm import LLM, SamplingParams

os.environ["VLLM_ALLOW_LONG_MAX_MODEL_LEN"] = "1"

# audio_url = Path("./audio_file.mp3").expanduser().resolve().as_uri()   # local file -> file://...
audio_url = "https://huggingface.co/datasets/nvidia/AudioSkills/resolve/main/assets/WhDJDIviAOg_120_10.mp3"  # web URL -> https://...

prompt = "Transcribe the input speech."

llm = LLM(
    model="nvidia/audio-flamingo-3-hf",
    allowed_local_media_path=str(Path.cwd()),
    max_model_len=20000,
)
sp = SamplingParams(max_tokens=4096, temperature=0.0, repetition_penalty=1.2)

print(
    llm.chat(
        [
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": prompt},
                    {"type": "audio_url", "audio_url": {"url": audio_url}},
                ],
            }
        ],
        sp,
    )[0]
    .outputs[0]
    .text
)

pip install flash-attn --no-build-isolation

model = AudioFlamingo3ForConditionalGeneration.from_pretrained(
    model_id, torch_dtype=torch_dtype, low_cpu_mem_usage=True, attn_implementation="flash_attention_2"
).to(device)