nomos-1-AWQ-8bit

from transformers import AutoTokenizer, AutoModelForCausalLM
import torch

model_id = "NousResearch/nomos-1"

tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype=torch.float16,
    device_map="auto",
    trust_remote_code=True,
)

messages = [
    {
        "role": "user",
        "content": (
            "Solve the following problem and show your reasoning:\n\n"
            "Let a, b, c be positive real numbers such that abc = 1. "
            "Prove that\n"
            "\\[\n"
            "  \\frac{1}{1+a} + \\frac{1}{1+b} + \\frac{1}{1+c} \\ge 1.\n"
            "\\]"
        ),
    },
]

inputs = tokenizer.apply_chat_template(
    messages,
    add_generation_prompt=True,
    return_tensors="pt",
).to(model.device)

with torch.no_grad():
    outputs = model.generate(
        **inputs,
        max_new_tokens=400,
        temperature=0.6,
        top_p=0.95,
        top_k=20,
        do_sample=True,
    )

print(tokenizer.decode(outputs[0], skip_special_tokens=True))

python -m sglang.launch_server \
  --model-path NousResearch/nomos-1 \
  --tp-size 8

vllm serve \
  --model NousResearch/nomos-1 \
  --tensor-parallel-size 8