Qwen3-8B-int4-ov

pip install optimum[openvino]

pip install openvino-genai huggingface_hub

import huggingface_hub as hf_hub

model_id = "OpenVINO/qwen3-8b-int4-ov"
model_path = "qwen3-8b-int4-ov"

hf_hub.snapshot_download(model_id, local_dir=model_path)

pip install openai

from openai import OpenAI
client = OpenAI(
  base_url="http://localhost:8000/v3",
  api_key="unused"
)

stream = client.chat.completions.create(
    model="OpenVINO/Qwen3-8B-int4-ov",
    messages=[{"role": "user", "content": "Hello."}],
    stream=True,
    extra_body={"chat_template_kwargs": {"enable_thinking": False}},
    tools=[],
)
for chunk in stream:
    if chunk.choices[0].delta.content is not None:
        print(chunk.choices[0].delta.content, end="")