octen-embedding-0.6b-onnx-int4

import onnxruntime as ort
import numpy as np
from tokenizers import Tokenizer

tokenizer = Tokenizer.from_file("tokenizer.json")
tokenizer.enable_truncation(max_length=512)

# CPUExecutionProvider supports MatMulNBits 4-bit
session = ort.InferenceSession("model.int4.onnx", providers=["CPUExecutionProvider"])

text = "semantic search example"
enc  = tokenizer.encode(text)
ids  = np.array([enc.ids],             dtype=np.int64)
mask = np.array([enc.attention_mask],  dtype=np.int64)

lhs  = session.run(None, {"input_ids": ids, "attention_mask": mask})[0]  # [1, seq, 1024]
emb  = lhs[0, mask[0].sum() - 1]   # last non-padding token
emb  = emb / np.linalg.norm(emb)
print(emb.shape)  # (1024,)