LumiChats-Instruct-4B_lora

pip install unsloth transformers bitsandbytes accelerate peft

from unsloth import FastLanguageModel
from transformers import TextStreamer

model, tokenizer = FastLanguageModel.from_pretrained(
    model_name     = "adityakum667388/LumiChats-Instruct-14B_lora",
    max_seq_length = 2048,
    load_in_4bit   = True,
)
FastLanguageModel.for_inference(model)

messages = [
    {"role": "user", "content": "Explain quantum entanglement in simple terms."}
]

inputs = tokenizer.apply_chat_template(
    messages,
    tokenize              = True,
    add_generation_prompt = True,
    return_tensors        = "pt",
).to("cuda")

streamer = TextStreamer(tokenizer, skip_prompt=True)
_ = model.generate(
    input_ids      = inputs,
    streamer       = streamer,
    max_new_tokens = 256,
    temperature    = 1.5,
    min_p          = 0.1,
    use_cache      = True,
)