huggingface/sentence-transformers v5.2.0

v5.2.0 - CrossEncoder multi-processing, multilingual NanoBEIR evaluators, similarity score in `mine_hard_negatives`, Transformers v5 support

on GitHub

from sentence_transformers.cross_encoder import CrossEncoder

def main():
	model = CrossEncoder('cross-encoder/ms-marco-MiniLM-L6-v2')
	
	# Start a pool of workers
	pool = model.start_multi_process_pool()
	
	# Use the pool for faster inference
	scores = model.predict(sentence_pairs, pool=pool)
	rankings = model.rank(query, documents, pool=pool)
	
	# Clean up when done
	model.stop_multi_process_pool(pool)

if __name__ == "__main__":
    main()

from sentence_transformers.cross_encoder import CrossEncoder

def main():
	model = CrossEncoder('cross-encoder/ms-marco-MiniLM-L6-v2', device="cpu")
	
	# Use 4 processes
	scores = model.predict(sentence_pairs, device=["cpu"] * 4)
	rankings = model.rank(query, documents, device=["cpu"] * 4)

if __name__ == "__main__":
    main()

import logging
from pprint import pprint

from sentence_transformers import SentenceTransformer
from sentence_transformers.evaluation import NanoBEIREvaluator

logging.basicConfig(format="%(asctime)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO)

# Load a model to evaluate
model = SentenceTransformer("google/embeddinggemma-300m")
# Use a Serbian translation of NanoBEIR
evaluator = NanoBEIREvaluator(
    ["msmarco", "nq"],
    dataset_id="Serbian-AI-Society/NanoBEIR-sr"
)
results = evaluator(model)
print(results[evaluator.primary_metric])
pprint({key: value for key, value in results.items() if "ndcg@10" in key})
"""
{'NanoBEIR_mean_cosine_ndcg@10': 0.44754032737278326,
 'NanoMSMARCO_cosine_ndcg@10': 0.4424192627754922,
 'NanoNQ_cosine_ndcg@10': 0.45266139197007427}
"""

from sentence_transformers.util import mine_hard_negatives
from sentence_transformers import SentenceTransformer
from datasets import load_dataset

# Load a Sentence Transformer model
model = SentenceTransformer("sentence-transformers/static-retrieval-mrl-en-v1")

# Load a dataset to mine hard negatives from
dataset = load_dataset("sentence-transformers/natural-questions", split="train").select(range(10000))
print(dataset)
"""
Dataset({
    features: ['query', 'answer'],
    num_rows: 10000
})
"""

# Mine hard negatives into num_negatives + 3 columns:
# 'query', 'answer', 'negative_1', 'negative_2', ..., 'score'
# where 'score' is a list of similarity scores for the query-answer plus each query-negative pair.
dataset = mine_hard_negatives(
    dataset=dataset,
    model=model,
    num_negatives=5,
    sampling_strategy="top",
    relative_margin=0.05,
    batch_size=128,
    use_faiss=True,
    output_format="labeled-list",
    output_scores=True,
)
"""
Negative candidates mined, preparing dataset...
Metric       Positive       Negative     Difference
Count          10,000         49,241
Mean           0.5884         0.3909         0.2033
Median         0.6005         0.3766         0.1837
Std            0.1467         0.1050         0.1337
Min            0.0272         0.1595         0.0088
25%            0.4918         0.3127         0.0903
50%            0.6005         0.3766         0.1837
75%            0.6974         0.4558         0.2924
Max            0.9679         0.8505         0.7281
Skipped 25,451 potential negatives (4.89%) due to the relative_margin of 0.05.
Could not find enough negatives for 148 samples (1.48%). Consider adjusting the range_max and relative_margin parameters if you'd like to find more valid negatives.
"""
print(dataset)
"""
Dataset({
    features: ['query', 'answer', 'scores'],
    num_rows: 9852
})
"""
print(dataset[0])
{
    "query": "when did richmond last play in a preliminary final",
    "answer": [
        "Richmond Football Club Richmond began 2017 with 5 straight wins, a feat it had not achieved since 1995. A series of close losses hampered the Tigers throughout the middle of the season, including a 5-point loss to the Western Bulldogs, 2-point loss to Fremantle, and a 3-point loss to the Giants. Richmond ended the season strongly with convincing victories over Fremantle and St Kilda in the final two rounds, elevating the club to 3rd on the ladder. Richmond's first final of the season against the Cats at the MCG attracted a record qualifying final crowd of 95,028; the Tigers won by 51 points. Having advanced to the first preliminary finals for the first time since 2001, Richmond defeated Greater Western Sydney by 36 points in front of a crowd of 94,258 to progress to the Grand Final against Adelaide, their first Grand Final appearance since 1982. The attendance was 100,021, the largest crowd to a grand final since 1986. The Crows led at quarter time and led by as many as 13, but the Tigers took over the game as it progressed and scored seven straight goals at one point. They eventually would win by 48 points – 16.12 (108) to Adelaide's 8.12 (60) – to end their 37-year flag drought.[22] Dustin Martin also became the first player to win a Premiership medal, the Brownlow Medal and the Norm Smith Medal in the same season, while Damien Hardwick was named AFL Coaches Association Coach of the Year. Richmond's jump from 13th to premiers also marked the biggest jump from one AFL season to the next.",
        "2017 AFL Grand Final The 2017 AFL Grand Final was an Australian rules football game contested between the Adelaide Crows and the Richmond Tigers, held at the Melbourne Cricket Ground on 30 September 2017. It was the 121st annual grand final of the Australian Football League (formerly the Victorian Football League), staged to determine the premiers for the 2017 AFL season.[1]. Richmond defeated Adelaide by 48 points, marking the club's eleventh premiership and first since 1980. Richmond's Dustin Martin won the Norm Smith Medal as the best player on the ground. The match was attended by 100,021 people, the largest crowd since the 1986 Grand Final.",
        "Raid of Richmond The Richmond Campaign was a group of British military actions against the capital of Virginia, Richmond, and the surrounding area, during the American Revolutionary War. Led by American turncoat Benedict Arnold, the Richmond Campaign is considered one of his greatest successes while serving under the British Army, and one of the most notorious actions that Arnold ever performed.",
        "2001 AFL Grand Final The 2001 AFL Grand Final was an Australian rules football game contested between the Essendon Football Club and the Brisbane Lions, held at the Melbourne Cricket Ground in Melbourne on 29 September 2001. It was the 105th annual Grand Final of the Australian Football League (formerly the Victorian Football League),[1] staged to determine the premiers for the 2001 AFL season. The match, attended by 91,482 spectators, was won by Brisbane by a margin of 26 points, marking that club's first premiership victory.",
        "1964 VFL Grand Final The 1964 VFL Grand Final was an Australian rules football game contested between the  Collingwood Football Club and Melbourne Football Club, held at the Melbourne Cricket Ground in Melbourne on 19 September 1964. It was the 68th annual Grand Final of the Victorian Football League, staged to determine the premiers for the 1964 VFL season. The match, attended by 102,471 spectators, was won by Melbourne by a margin of 4 points, marking that club's 12th (and to date, most recent) premiership victory.",
        "1998 AFL Grand Final The 1998 AFL Grand Final was an Australian rules football game contested between the Adelaide Crows and the North Melbourne Kangaroos, held at the Melbourne Cricket Ground in Melbourne on 26 September 1998. It was the 102nd annual Grand Final of the Australian Football League (formerly the Victorian Football League), staged to determine the premiers for the 1998 AFL season. The match, attended by 94,431 spectators, was won by Adelaide by a margin of 35 points marking that club's second consecutive premiership victory, and second premiership overall.",
    ],
    "scores": [
        0.5460646748542786,
        0.5105829238891602,
        0.4460095167160034,
        0.3221113085746765,
        0.3161606788635254,
        0.31184709072113037,
    ],
}
# dataset.push_to_hub("natural-questions-hard-negatives", "labeled-list-scores")