Solar-Open-69B-REAP

from llama_cpp import Llama
from transformers import AutoTokenizer

# 1. Pull the latest fixed template from this HF Repo
repo_id = "Akicou/Solar-Open-69B-REAP" 
tokenizer = AutoTokenizer.from_pretrained(repo_id)
fixed_template = tokenizer.chat_template

# 2. Initialize the Llama model
llm = Llama(
    model_path="./Solar-Open-69B-REAP.Q4_K_M.gguf",
    n_ctx=4096,
    n_gpu_layers=-1 # Use -1 to offload all layers to GPU
)

# 3. Create completion with the injected template
messages = [
    {"role": "system", "content": "You are a concise and helpful assistant."},
    {"role": "user", "content": "Explain the benefits of MoE pruning."}
]

response = llm.create_chat_completion(
    messages=messages,
    chat_template=fixed_template, # Force-uses the latest Jan 7th fix
    temp=0.7
)

print(response["choices"][0]["message"]["content"])