XVERSE-Ent-A5.7B

import torch
from transformers import AutoTokenizer, AutoModelForCausalLM
tokenizer = AutoTokenizer.from_pretrained("xverse/XVERSE-Ent-A5.7B")
model = AutoModelForCausalLM.from_pretrained("xverse/XVERSE-Ent-A5.7B", trust_remote_code=True, torch_dtype=torch.bfloat16, device_map='auto')
model = model.eval()
inputs = tokenizer('Time crept by, second after second. The sound of rain, the refrigerator’s occasional hum, the rush of water through unseen pipes in the walls—everything seemed amplified. Lin Yu realized he was counting his breaths, as if the moment he stopped, the room would be filled with a presence that did not belong to him.', return_tensors='pt').input_ids
inputs = inputs.cuda()
generated_ids = model.generate(inputs, max_new_tokens=70, eos_token_id=tokenizer.eos_token_id, repetition_penalty=1.1)
print(tokenizer.batch_decode(generated_ids, skip_special_tokens=True))