final_vlm_moe-388

from transformers import LlavaForConditionalGeneration, AutoProcessor

# Load model
model = LlavaForConditionalGeneration.from_pretrained(
    "your-username/ndr-vlm-moe-8b-a2b",
    torch_dtype=torch.float16,
    device_map="auto",
)
processor = AutoProcessor.from_pretrained("your-username/ndr-vlm-moe-8b-a2b")

# Inference with image
from PIL import Image
image = Image.open("path/to/image.jpg")
prompt = "What's in this image?"

inputs = processor(images=image, text=prompt, return_tensors="pt").to("cuda")
outputs = model.generate(**inputs, max_new_tokens=256)
response = processor.decode(outputs[0], skip_special_tokens=True)
print(response)