BGE-VL-large

pip install sentence_transformers[image]

from sentence_transformers import SentenceTransformer

model = SentenceTransformer("BAAI/BGE-VL-large", trust_remote_code=True)

query_image = "https://huggingface.co/BAAI/BGE-VL-large/resolve/main/assets/cir_query.png"
candidate_1 = "https://huggingface.co/BAAI/BGE-VL-large/resolve/main/assets/cir_candi_1.png"
candidate_2 = "https://huggingface.co/BAAI/BGE-VL-large/resolve/main/assets/cir_candi_2.png"

# Encode text
text_embeddings = model.encode(["A dog sitting on a bench", "A cat sleeping on a couch"])
print(text_embeddings.shape)
# (2, 768)

# Encode images
image_embeddings = model.encode([query_image, candidate_1])
print(image_embeddings.shape)
# (2, 768)

# Compute similarities
similarities = model.similarity(text_embeddings, image_embeddings)
print(similarities)
# tensor([[0.1255, 0.1018],
#         [0.0161, 0.0271]])

# Composed image retrieval: encode image+text query, compare with image candidates
query_embeddings = model.encode([{
    "image": query_image,
    "text": "Make the background dark, as if the camera has taken the photo at night",
}])
candidate_embeddings = model.encode([candidate_1, candidate_2])
scores = model.similarity(query_embeddings, candidate_embeddings)
print(scores)
# tensor([[0.3696, 0.1714]])