m5-encoder

from transformers import AutoConfig, AutoModel

config = AutoConfig.from_pretrained("IlPakoZ/m5-encoder", trust_remote_code=True)
model  = AutoModel.from_pretrained("IlPakoZ/m5-encoder", trust_remote_code=True)

from transformers import AutoModelForSequenceClassification

regression_model = AutoModelForSequenceClassification.from_pretrained(
    "IlPakoZ/m5-encoder", trust_remote_code=True
)

import torch
from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("IlPakoZ/m5-encoder", trust_remote_code=True)

smiles = "CCO"

# seed = 0 produces the canonical SELFIES, other values generate random reproducible variations
selfies, pos_encod, _ = model.get_positional_encodings_and_align(smiles, seed=0)

encoding    = tokenizer(selfies, return_tensors="pt")
input_ids   = encoding["input_ids"]
attn_mask   = encoding["attention_mask"]

rel_pos     = torch.tensor(pos_encod).unsqueeze(0)   # (1, seq_len, seq_len)

outputs = model(input_ids=input_ids, attention_mask=attn_mask, relative_position=rel_pos)
hidden  = outputs.last_hidden_state   # (1, seq_len, 512)