BGE-VL-Screenshot

pip install "sentence_transformers[image]"

from sentence_transformers import SentenceTransformer

model = SentenceTransformer("BAAI/BGE-VL-Screenshot")

# Queries: composed image + text inputs (prefix text with "Query:")
query_inputs = [
    {"text": "Query:After a 17% drop, what is Nvidia's closing stock price?", "image": "https://huggingface.co/BAAI/BGE-VL-Screenshot/resolve/main/assets/query_1.png"},
    {"text": "Query:I would like to see a detailed and intuitive performance comparison between the two models.", "image": "https://huggingface.co/BAAI/BGE-VL-Screenshot/resolve/main/assets/query_2.png"},
]
query_embeddings = model.encode_query(query_inputs)
print(query_embeddings.shape)
# (2, 2048)

# Candidates: screenshot images
candidate_inputs = [
    "https://huggingface.co/BAAI/BGE-VL-Screenshot/resolve/main/assets/positive_1.jpeg",
    "https://huggingface.co/BAAI/BGE-VL-Screenshot/resolve/main/assets/neg_1.jpeg",
    "https://huggingface.co/BAAI/BGE-VL-Screenshot/resolve/main/assets/positive_2.jpeg",
    "https://huggingface.co/BAAI/BGE-VL-Screenshot/resolve/main/assets/neg_2.jpeg",
]
candidate_embeddings = model.encode_document(candidate_inputs)
print(candidate_embeddings.shape)
# (4, 2048)

similarities = model.similarity(query_embeddings, candidate_embeddings)
print(similarities)
# tensor([[0.5725, 0.3449, 0.1913, 0.1497],
#         [0.1457, 0.0795, 0.4243, 0.4177]])