DeepSeek-R1-NVFP4-FP8-BLOCK

from vllm import LLM, SamplingParams

prompts = ["The Swiss Alps are", "Brad Marchand is", "The Toronto Maple Leafs are"]

# Create a sampling params object for greedy sampling
sampling_params = SamplingParams(
    temperature=0.80, top_p=0.95, max_tokens=40, min_tokens=10
)
llm = LLM(
    "bdellabe/DeepSeek-R1-NVFP4-FP8-BLOCK",
    tensor_parallel_size=4,
    max_model_len=4096,
    enforce_eager=True,
)
output = llm.generate(prompts, sampling_params)
for out in output:
    print(out.outputs[0].text)