cogito-671b-v2.1-FP8

pip install transformers hf_transfer accelerate vllm
pip install sglang[all]
hf download deepcogito/cogito-671b-v2.1-FP8

# H200s
python3 -m sglang.launch_server --model deepcogito/cogito-671b-v2.1-FP8 --tp 8

# B200s
python3 -m sglang.launch_server --model deepcogito/cogito-671b-v2.1-FP8 --tp 8 --quantization compressed-tensors --moe-runner-backend triton

import openai
client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="EMPTY")

response = client.chat.completions.create(
    model="default",
    messages=[
        {"role": "system", "content": "Always respond in 1-2 words."},
        {"role": "user", "content": "Who created you?"},
    ],
    temperature=0.6,
    max_tokens=8192,
    extra_body = {"chat_template_kwargs": {"enable_thinking": False}}
)
# To enable reasoning, set `enable_thinking=True` above.
print(response.choices[0].message.content)