GLM-4.5-Air-NotaMoeQuant-Int4

import torch
from transformers import AutoTokenizer, AutoModelForCausalLM

MODEL_NAME = "nota-ai/GLM-4.5-Air-NotaMoeQuant-Int4"

# load the tokenizer and the model
tokenizer = AutoTokenizer.from_pretrained(
    MODEL_NAME,
    dtype="auto",
    trust_remote_code=True,
)

model = AutoModelForCausalLM.from_pretrained(
    MODEL_NAME,
    dtype=torch.bfloat16,
    device_map="auto",
    trust_remote_code=True,
)

model.eval()

# prepare the model input
prompt = "What is large language model?"
messages = [
    {"role": "user", "content": prompt}
]
text = tokenizer.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=True,
    return_token_type_ids=False
)
model_inputs = tokenizer([text], return_tensors="pt").to(model.device)

# conduct text completion
generated_ids = model.generate(
    model_inputs['input_ids'],
    attention_mask=model_inputs['attention_mask'], 
    max_new_tokens=150
)