Qwen3-Next-80B-A3B-Thinking-NVFP4

from tensorrt_llm import LLM, SamplingParams, KvCacheConfig


def main():

    prompts = [
        "Hello, my name is",
        "The president of the United States is",
        "The capital of France is",
        "The future of AI is",
    ]
    sampling_params = SamplingParams(temperature=0.6, top_p=0.95)
    kv_cache_config = KvCacheConfig(enable_block_reuse=False)

    llm = LLM(model="nvidia/Qwen3-Next-80B-A3B-Thinking-NVFP4", tensor_parallel_size=4, kv_cache_config=kv_cache_config)

    outputs = llm.generate(prompts, sampling_params)

    # Print the outputs.
    for output in outputs:
        prompt = output.prompt
        generated_text = output.outputs[0].text
        print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")


# The entry point of the program needs to be protected for spawning processes.
if __name__ == '__main__':
    main()