|
|
--- |
|
|
library_name: sentence-transformers |
|
|
pipeline_tag: sentence-similarity |
|
|
tags: |
|
|
- feature-extraction |
|
|
- sentence-similarity |
|
|
- mteb |
|
|
- transformers |
|
|
- transformers.js |
|
|
model-index: |
|
|
- name: epoch_0_model |
|
|
results: |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_counterfactual |
|
|
name: MTEB AmazonCounterfactualClassification (en) |
|
|
config: en |
|
|
split: test |
|
|
revision: e8379541af4e31359cca9fbcf4b00f2671dba205 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 75.20895522388058 |
|
|
- type: ap |
|
|
value: 38.57605549557802 |
|
|
- type: f1 |
|
|
value: 69.35586565857854 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_polarity |
|
|
name: MTEB AmazonPolarityClassification |
|
|
config: default |
|
|
split: test |
|
|
revision: e2d317d38cd51312af73b3d32a06d1a08b442046 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 91.8144 |
|
|
- type: ap |
|
|
value: 88.65222882032363 |
|
|
- type: f1 |
|
|
value: 91.80426301643274 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_reviews_multi |
|
|
name: MTEB AmazonReviewsClassification (en) |
|
|
config: en |
|
|
split: test |
|
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 47.162000000000006 |
|
|
- type: f1 |
|
|
value: 46.59329642263158 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: arguana |
|
|
name: MTEB ArguAna |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 24.253 |
|
|
- type: map_at_10 |
|
|
value: 38.962 |
|
|
- type: map_at_100 |
|
|
value: 40.081 |
|
|
- type: map_at_1000 |
|
|
value: 40.089000000000006 |
|
|
- type: map_at_3 |
|
|
value: 33.499 |
|
|
- type: map_at_5 |
|
|
value: 36.351 |
|
|
- type: mrr_at_1 |
|
|
value: 24.609 |
|
|
- type: mrr_at_10 |
|
|
value: 39.099000000000004 |
|
|
- type: mrr_at_100 |
|
|
value: 40.211000000000006 |
|
|
- type: mrr_at_1000 |
|
|
value: 40.219 |
|
|
- type: mrr_at_3 |
|
|
value: 33.677 |
|
|
- type: mrr_at_5 |
|
|
value: 36.469 |
|
|
- type: ndcg_at_1 |
|
|
value: 24.253 |
|
|
- type: ndcg_at_10 |
|
|
value: 48.010999999999996 |
|
|
- type: ndcg_at_100 |
|
|
value: 52.756 |
|
|
- type: ndcg_at_1000 |
|
|
value: 52.964999999999996 |
|
|
- type: ndcg_at_3 |
|
|
value: 36.564 |
|
|
- type: ndcg_at_5 |
|
|
value: 41.711999999999996 |
|
|
- type: precision_at_1 |
|
|
value: 24.253 |
|
|
- type: precision_at_10 |
|
|
value: 7.738 |
|
|
- type: precision_at_100 |
|
|
value: 0.98 |
|
|
- type: precision_at_1000 |
|
|
value: 0.1 |
|
|
- type: precision_at_3 |
|
|
value: 15.149000000000001 |
|
|
- type: precision_at_5 |
|
|
value: 11.593 |
|
|
- type: recall_at_1 |
|
|
value: 24.253 |
|
|
- type: recall_at_10 |
|
|
value: 77.383 |
|
|
- type: recall_at_100 |
|
|
value: 98.009 |
|
|
- type: recall_at_1000 |
|
|
value: 99.644 |
|
|
- type: recall_at_3 |
|
|
value: 45.448 |
|
|
- type: recall_at_5 |
|
|
value: 57.965999999999994 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/arxiv-clustering-p2p |
|
|
name: MTEB ArxivClusteringP2P |
|
|
config: default |
|
|
split: test |
|
|
revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 45.69069567851087 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/arxiv-clustering-s2s |
|
|
name: MTEB ArxivClusteringS2S |
|
|
config: default |
|
|
split: test |
|
|
revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 36.35185490976283 |
|
|
- task: |
|
|
type: Reranking |
|
|
dataset: |
|
|
type: mteb/askubuntudupquestions-reranking |
|
|
name: MTEB AskUbuntuDupQuestions |
|
|
config: default |
|
|
split: test |
|
|
revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 |
|
|
metrics: |
|
|
- type: map |
|
|
value: 61.71274951450321 |
|
|
- type: mrr |
|
|
value: 76.06032625423207 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/biosses-sts |
|
|
name: MTEB BIOSSES |
|
|
config: default |
|
|
split: test |
|
|
revision: d3fb88f8f02e40887cd149695127462bbcf29b4a |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 86.73980520022269 |
|
|
- type: cos_sim_spearman |
|
|
value: 84.24649792685918 |
|
|
- type: euclidean_pearson |
|
|
value: 85.85197641158186 |
|
|
- type: euclidean_spearman |
|
|
value: 84.24649792685918 |
|
|
- type: manhattan_pearson |
|
|
value: 86.26809552711346 |
|
|
- type: manhattan_spearman |
|
|
value: 84.56397504030865 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/banking77 |
|
|
name: MTEB Banking77Classification |
|
|
config: default |
|
|
split: test |
|
|
revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 84.25324675324674 |
|
|
- type: f1 |
|
|
value: 84.17872280892557 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/biorxiv-clustering-p2p |
|
|
name: MTEB BiorxivClusteringP2P |
|
|
config: default |
|
|
split: test |
|
|
revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 38.770253446400886 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/biorxiv-clustering-s2s |
|
|
name: MTEB BiorxivClusteringS2S |
|
|
config: default |
|
|
split: test |
|
|
revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 32.94307095497281 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: BeIR/cqadupstack |
|
|
name: MTEB CQADupstackAndroidRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 32.164 |
|
|
- type: map_at_10 |
|
|
value: 42.641 |
|
|
- type: map_at_100 |
|
|
value: 43.947 |
|
|
- type: map_at_1000 |
|
|
value: 44.074999999999996 |
|
|
- type: map_at_3 |
|
|
value: 39.592 |
|
|
- type: map_at_5 |
|
|
value: 41.204 |
|
|
- type: mrr_at_1 |
|
|
value: 39.628 |
|
|
- type: mrr_at_10 |
|
|
value: 48.625 |
|
|
- type: mrr_at_100 |
|
|
value: 49.368 |
|
|
- type: mrr_at_1000 |
|
|
value: 49.413000000000004 |
|
|
- type: mrr_at_3 |
|
|
value: 46.400000000000006 |
|
|
- type: mrr_at_5 |
|
|
value: 47.68 |
|
|
- type: ndcg_at_1 |
|
|
value: 39.628 |
|
|
- type: ndcg_at_10 |
|
|
value: 48.564 |
|
|
- type: ndcg_at_100 |
|
|
value: 53.507000000000005 |
|
|
- type: ndcg_at_1000 |
|
|
value: 55.635999999999996 |
|
|
- type: ndcg_at_3 |
|
|
value: 44.471 |
|
|
- type: ndcg_at_5 |
|
|
value: 46.137 |
|
|
- type: precision_at_1 |
|
|
value: 39.628 |
|
|
- type: precision_at_10 |
|
|
value: 8.856 |
|
|
- type: precision_at_100 |
|
|
value: 1.429 |
|
|
- type: precision_at_1000 |
|
|
value: 0.191 |
|
|
- type: precision_at_3 |
|
|
value: 21.268 |
|
|
- type: precision_at_5 |
|
|
value: 14.649000000000001 |
|
|
- type: recall_at_1 |
|
|
value: 32.164 |
|
|
- type: recall_at_10 |
|
|
value: 59.609 |
|
|
- type: recall_at_100 |
|
|
value: 80.521 |
|
|
- type: recall_at_1000 |
|
|
value: 94.245 |
|
|
- type: recall_at_3 |
|
|
value: 46.521 |
|
|
- type: recall_at_5 |
|
|
value: 52.083999999999996 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: BeIR/cqadupstack |
|
|
name: MTEB CQADupstackEnglishRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 31.526 |
|
|
- type: map_at_10 |
|
|
value: 41.581 |
|
|
- type: map_at_100 |
|
|
value: 42.815999999999995 |
|
|
- type: map_at_1000 |
|
|
value: 42.936 |
|
|
- type: map_at_3 |
|
|
value: 38.605000000000004 |
|
|
- type: map_at_5 |
|
|
value: 40.351 |
|
|
- type: mrr_at_1 |
|
|
value: 39.489999999999995 |
|
|
- type: mrr_at_10 |
|
|
value: 47.829 |
|
|
- type: mrr_at_100 |
|
|
value: 48.512 |
|
|
- type: mrr_at_1000 |
|
|
value: 48.552 |
|
|
- type: mrr_at_3 |
|
|
value: 45.754 |
|
|
- type: mrr_at_5 |
|
|
value: 46.986 |
|
|
- type: ndcg_at_1 |
|
|
value: 39.489999999999995 |
|
|
- type: ndcg_at_10 |
|
|
value: 47.269 |
|
|
- type: ndcg_at_100 |
|
|
value: 51.564 |
|
|
- type: ndcg_at_1000 |
|
|
value: 53.53099999999999 |
|
|
- type: ndcg_at_3 |
|
|
value: 43.301 |
|
|
- type: ndcg_at_5 |
|
|
value: 45.239000000000004 |
|
|
- type: precision_at_1 |
|
|
value: 39.489999999999995 |
|
|
- type: precision_at_10 |
|
|
value: 8.93 |
|
|
- type: precision_at_100 |
|
|
value: 1.415 |
|
|
- type: precision_at_1000 |
|
|
value: 0.188 |
|
|
- type: precision_at_3 |
|
|
value: 20.892 |
|
|
- type: precision_at_5 |
|
|
value: 14.865999999999998 |
|
|
- type: recall_at_1 |
|
|
value: 31.526 |
|
|
- type: recall_at_10 |
|
|
value: 56.76 |
|
|
- type: recall_at_100 |
|
|
value: 75.029 |
|
|
- type: recall_at_1000 |
|
|
value: 87.491 |
|
|
- type: recall_at_3 |
|
|
value: 44.786 |
|
|
- type: recall_at_5 |
|
|
value: 50.254 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: BeIR/cqadupstack |
|
|
name: MTEB CQADupstackGamingRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 40.987 |
|
|
- type: map_at_10 |
|
|
value: 52.827 |
|
|
- type: map_at_100 |
|
|
value: 53.751000000000005 |
|
|
- type: map_at_1000 |
|
|
value: 53.81 |
|
|
- type: map_at_3 |
|
|
value: 49.844 |
|
|
- type: map_at_5 |
|
|
value: 51.473 |
|
|
- type: mrr_at_1 |
|
|
value: 46.833999999999996 |
|
|
- type: mrr_at_10 |
|
|
value: 56.389 |
|
|
- type: mrr_at_100 |
|
|
value: 57.003 |
|
|
- type: mrr_at_1000 |
|
|
value: 57.034 |
|
|
- type: mrr_at_3 |
|
|
value: 54.17999999999999 |
|
|
- type: mrr_at_5 |
|
|
value: 55.486999999999995 |
|
|
- type: ndcg_at_1 |
|
|
value: 46.833999999999996 |
|
|
- type: ndcg_at_10 |
|
|
value: 58.372 |
|
|
- type: ndcg_at_100 |
|
|
value: 62.068 |
|
|
- type: ndcg_at_1000 |
|
|
value: 63.288 |
|
|
- type: ndcg_at_3 |
|
|
value: 53.400000000000006 |
|
|
- type: ndcg_at_5 |
|
|
value: 55.766000000000005 |
|
|
- type: precision_at_1 |
|
|
value: 46.833999999999996 |
|
|
- type: precision_at_10 |
|
|
value: 9.191 |
|
|
- type: precision_at_100 |
|
|
value: 1.192 |
|
|
- type: precision_at_1000 |
|
|
value: 0.134 |
|
|
- type: precision_at_3 |
|
|
value: 23.448 |
|
|
- type: precision_at_5 |
|
|
value: 15.862000000000002 |
|
|
- type: recall_at_1 |
|
|
value: 40.987 |
|
|
- type: recall_at_10 |
|
|
value: 71.146 |
|
|
- type: recall_at_100 |
|
|
value: 87.035 |
|
|
- type: recall_at_1000 |
|
|
value: 95.633 |
|
|
- type: recall_at_3 |
|
|
value: 58.025999999999996 |
|
|
- type: recall_at_5 |
|
|
value: 63.815999999999995 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: BeIR/cqadupstack |
|
|
name: MTEB CQADupstackGisRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 24.587 |
|
|
- type: map_at_10 |
|
|
value: 33.114 |
|
|
- type: map_at_100 |
|
|
value: 34.043 |
|
|
- type: map_at_1000 |
|
|
value: 34.123999999999995 |
|
|
- type: map_at_3 |
|
|
value: 30.45 |
|
|
- type: map_at_5 |
|
|
value: 31.813999999999997 |
|
|
- type: mrr_at_1 |
|
|
value: 26.554 |
|
|
- type: mrr_at_10 |
|
|
value: 35.148 |
|
|
- type: mrr_at_100 |
|
|
value: 35.926 |
|
|
- type: mrr_at_1000 |
|
|
value: 35.991 |
|
|
- type: mrr_at_3 |
|
|
value: 32.599000000000004 |
|
|
- type: mrr_at_5 |
|
|
value: 33.893 |
|
|
- type: ndcg_at_1 |
|
|
value: 26.554 |
|
|
- type: ndcg_at_10 |
|
|
value: 38.132 |
|
|
- type: ndcg_at_100 |
|
|
value: 42.78 |
|
|
- type: ndcg_at_1000 |
|
|
value: 44.919 |
|
|
- type: ndcg_at_3 |
|
|
value: 32.833 |
|
|
- type: ndcg_at_5 |
|
|
value: 35.168 |
|
|
- type: precision_at_1 |
|
|
value: 26.554 |
|
|
- type: precision_at_10 |
|
|
value: 5.921 |
|
|
- type: precision_at_100 |
|
|
value: 0.8659999999999999 |
|
|
- type: precision_at_1000 |
|
|
value: 0.109 |
|
|
- type: precision_at_3 |
|
|
value: 13.861 |
|
|
- type: precision_at_5 |
|
|
value: 9.605 |
|
|
- type: recall_at_1 |
|
|
value: 24.587 |
|
|
- type: recall_at_10 |
|
|
value: 51.690000000000005 |
|
|
- type: recall_at_100 |
|
|
value: 73.428 |
|
|
- type: recall_at_1000 |
|
|
value: 89.551 |
|
|
- type: recall_at_3 |
|
|
value: 37.336999999999996 |
|
|
- type: recall_at_5 |
|
|
value: 43.047000000000004 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: BeIR/cqadupstack |
|
|
name: MTEB CQADupstackMathematicaRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 16.715 |
|
|
- type: map_at_10 |
|
|
value: 24.251 |
|
|
- type: map_at_100 |
|
|
value: 25.326999999999998 |
|
|
- type: map_at_1000 |
|
|
value: 25.455 |
|
|
- type: map_at_3 |
|
|
value: 21.912000000000003 |
|
|
- type: map_at_5 |
|
|
value: 23.257 |
|
|
- type: mrr_at_1 |
|
|
value: 20.274 |
|
|
- type: mrr_at_10 |
|
|
value: 28.552 |
|
|
- type: mrr_at_100 |
|
|
value: 29.42 |
|
|
- type: mrr_at_1000 |
|
|
value: 29.497 |
|
|
- type: mrr_at_3 |
|
|
value: 26.14 |
|
|
- type: mrr_at_5 |
|
|
value: 27.502 |
|
|
- type: ndcg_at_1 |
|
|
value: 20.274 |
|
|
- type: ndcg_at_10 |
|
|
value: 29.088 |
|
|
- type: ndcg_at_100 |
|
|
value: 34.293 |
|
|
- type: ndcg_at_1000 |
|
|
value: 37.271 |
|
|
- type: ndcg_at_3 |
|
|
value: 24.708 |
|
|
- type: ndcg_at_5 |
|
|
value: 26.809 |
|
|
- type: precision_at_1 |
|
|
value: 20.274 |
|
|
- type: precision_at_10 |
|
|
value: 5.361 |
|
|
- type: precision_at_100 |
|
|
value: 0.915 |
|
|
- type: precision_at_1000 |
|
|
value: 0.13 |
|
|
- type: precision_at_3 |
|
|
value: 11.733 |
|
|
- type: precision_at_5 |
|
|
value: 8.556999999999999 |
|
|
- type: recall_at_1 |
|
|
value: 16.715 |
|
|
- type: recall_at_10 |
|
|
value: 39.587 |
|
|
- type: recall_at_100 |
|
|
value: 62.336000000000006 |
|
|
- type: recall_at_1000 |
|
|
value: 83.453 |
|
|
- type: recall_at_3 |
|
|
value: 27.839999999999996 |
|
|
- type: recall_at_5 |
|
|
value: 32.952999999999996 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: BeIR/cqadupstack |
|
|
name: MTEB CQADupstackPhysicsRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 28.793000000000003 |
|
|
- type: map_at_10 |
|
|
value: 38.582 |
|
|
- type: map_at_100 |
|
|
value: 39.881 |
|
|
- type: map_at_1000 |
|
|
value: 39.987 |
|
|
- type: map_at_3 |
|
|
value: 35.851 |
|
|
- type: map_at_5 |
|
|
value: 37.289 |
|
|
- type: mrr_at_1 |
|
|
value: 34.455999999999996 |
|
|
- type: mrr_at_10 |
|
|
value: 43.909 |
|
|
- type: mrr_at_100 |
|
|
value: 44.74 |
|
|
- type: mrr_at_1000 |
|
|
value: 44.786 |
|
|
- type: mrr_at_3 |
|
|
value: 41.659 |
|
|
- type: mrr_at_5 |
|
|
value: 43.010999999999996 |
|
|
- type: ndcg_at_1 |
|
|
value: 34.455999999999996 |
|
|
- type: ndcg_at_10 |
|
|
value: 44.266 |
|
|
- type: ndcg_at_100 |
|
|
value: 49.639 |
|
|
- type: ndcg_at_1000 |
|
|
value: 51.644 |
|
|
- type: ndcg_at_3 |
|
|
value: 39.865 |
|
|
- type: ndcg_at_5 |
|
|
value: 41.887 |
|
|
- type: precision_at_1 |
|
|
value: 34.455999999999996 |
|
|
- type: precision_at_10 |
|
|
value: 7.843999999999999 |
|
|
- type: precision_at_100 |
|
|
value: 1.243 |
|
|
- type: precision_at_1000 |
|
|
value: 0.158 |
|
|
- type: precision_at_3 |
|
|
value: 18.831999999999997 |
|
|
- type: precision_at_5 |
|
|
value: 13.147 |
|
|
- type: recall_at_1 |
|
|
value: 28.793000000000003 |
|
|
- type: recall_at_10 |
|
|
value: 55.68300000000001 |
|
|
- type: recall_at_100 |
|
|
value: 77.99000000000001 |
|
|
- type: recall_at_1000 |
|
|
value: 91.183 |
|
|
- type: recall_at_3 |
|
|
value: 43.293 |
|
|
- type: recall_at_5 |
|
|
value: 48.618 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: BeIR/cqadupstack |
|
|
name: MTEB CQADupstackProgrammersRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 25.907000000000004 |
|
|
- type: map_at_10 |
|
|
value: 35.519 |
|
|
- type: map_at_100 |
|
|
value: 36.806 |
|
|
- type: map_at_1000 |
|
|
value: 36.912 |
|
|
- type: map_at_3 |
|
|
value: 32.748 |
|
|
- type: map_at_5 |
|
|
value: 34.232 |
|
|
- type: mrr_at_1 |
|
|
value: 31.621 |
|
|
- type: mrr_at_10 |
|
|
value: 40.687 |
|
|
- type: mrr_at_100 |
|
|
value: 41.583 |
|
|
- type: mrr_at_1000 |
|
|
value: 41.638999999999996 |
|
|
- type: mrr_at_3 |
|
|
value: 38.527 |
|
|
- type: mrr_at_5 |
|
|
value: 39.612 |
|
|
- type: ndcg_at_1 |
|
|
value: 31.621 |
|
|
- type: ndcg_at_10 |
|
|
value: 41.003 |
|
|
- type: ndcg_at_100 |
|
|
value: 46.617999999999995 |
|
|
- type: ndcg_at_1000 |
|
|
value: 48.82 |
|
|
- type: ndcg_at_3 |
|
|
value: 36.542 |
|
|
- type: ndcg_at_5 |
|
|
value: 38.368 |
|
|
- type: precision_at_1 |
|
|
value: 31.621 |
|
|
- type: precision_at_10 |
|
|
value: 7.396999999999999 |
|
|
- type: precision_at_100 |
|
|
value: 1.191 |
|
|
- type: precision_at_1000 |
|
|
value: 0.153 |
|
|
- type: precision_at_3 |
|
|
value: 17.39 |
|
|
- type: precision_at_5 |
|
|
value: 12.1 |
|
|
- type: recall_at_1 |
|
|
value: 25.907000000000004 |
|
|
- type: recall_at_10 |
|
|
value: 52.115 |
|
|
- type: recall_at_100 |
|
|
value: 76.238 |
|
|
- type: recall_at_1000 |
|
|
value: 91.218 |
|
|
- type: recall_at_3 |
|
|
value: 39.417 |
|
|
- type: recall_at_5 |
|
|
value: 44.435 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: BeIR/cqadupstack |
|
|
name: MTEB CQADupstackRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 25.732166666666668 |
|
|
- type: map_at_10 |
|
|
value: 34.51616666666667 |
|
|
- type: map_at_100 |
|
|
value: 35.67241666666666 |
|
|
- type: map_at_1000 |
|
|
value: 35.78675 |
|
|
- type: map_at_3 |
|
|
value: 31.953416666666662 |
|
|
- type: map_at_5 |
|
|
value: 33.333 |
|
|
- type: mrr_at_1 |
|
|
value: 30.300166666666673 |
|
|
- type: mrr_at_10 |
|
|
value: 38.6255 |
|
|
- type: mrr_at_100 |
|
|
value: 39.46183333333334 |
|
|
- type: mrr_at_1000 |
|
|
value: 39.519999999999996 |
|
|
- type: mrr_at_3 |
|
|
value: 36.41299999999999 |
|
|
- type: mrr_at_5 |
|
|
value: 37.6365 |
|
|
- type: ndcg_at_1 |
|
|
value: 30.300166666666673 |
|
|
- type: ndcg_at_10 |
|
|
value: 39.61466666666667 |
|
|
- type: ndcg_at_100 |
|
|
value: 44.60808333333334 |
|
|
- type: ndcg_at_1000 |
|
|
value: 46.91708333333334 |
|
|
- type: ndcg_at_3 |
|
|
value: 35.26558333333333 |
|
|
- type: ndcg_at_5 |
|
|
value: 37.220000000000006 |
|
|
- type: precision_at_1 |
|
|
value: 30.300166666666673 |
|
|
- type: precision_at_10 |
|
|
value: 6.837416666666667 |
|
|
- type: precision_at_100 |
|
|
value: 1.10425 |
|
|
- type: precision_at_1000 |
|
|
value: 0.14875 |
|
|
- type: precision_at_3 |
|
|
value: 16.13716666666667 |
|
|
- type: precision_at_5 |
|
|
value: 11.2815 |
|
|
- type: recall_at_1 |
|
|
value: 25.732166666666668 |
|
|
- type: recall_at_10 |
|
|
value: 50.578916666666665 |
|
|
- type: recall_at_100 |
|
|
value: 72.42183333333334 |
|
|
- type: recall_at_1000 |
|
|
value: 88.48766666666667 |
|
|
- type: recall_at_3 |
|
|
value: 38.41325 |
|
|
- type: recall_at_5 |
|
|
value: 43.515750000000004 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: BeIR/cqadupstack |
|
|
name: MTEB CQADupstackStatsRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 23.951 |
|
|
- type: map_at_10 |
|
|
value: 30.974 |
|
|
- type: map_at_100 |
|
|
value: 31.804 |
|
|
- type: map_at_1000 |
|
|
value: 31.900000000000002 |
|
|
- type: map_at_3 |
|
|
value: 28.762 |
|
|
- type: map_at_5 |
|
|
value: 29.94 |
|
|
- type: mrr_at_1 |
|
|
value: 26.534000000000002 |
|
|
- type: mrr_at_10 |
|
|
value: 33.553 |
|
|
- type: mrr_at_100 |
|
|
value: 34.297 |
|
|
- type: mrr_at_1000 |
|
|
value: 34.36 |
|
|
- type: mrr_at_3 |
|
|
value: 31.391000000000002 |
|
|
- type: mrr_at_5 |
|
|
value: 32.525999999999996 |
|
|
- type: ndcg_at_1 |
|
|
value: 26.534000000000002 |
|
|
- type: ndcg_at_10 |
|
|
value: 35.112 |
|
|
- type: ndcg_at_100 |
|
|
value: 39.28 |
|
|
- type: ndcg_at_1000 |
|
|
value: 41.723 |
|
|
- type: ndcg_at_3 |
|
|
value: 30.902 |
|
|
- type: ndcg_at_5 |
|
|
value: 32.759 |
|
|
- type: precision_at_1 |
|
|
value: 26.534000000000002 |
|
|
- type: precision_at_10 |
|
|
value: 5.445 |
|
|
- type: precision_at_100 |
|
|
value: 0.819 |
|
|
- type: precision_at_1000 |
|
|
value: 0.11 |
|
|
- type: precision_at_3 |
|
|
value: 12.986 |
|
|
- type: precision_at_5 |
|
|
value: 9.049 |
|
|
- type: recall_at_1 |
|
|
value: 23.951 |
|
|
- type: recall_at_10 |
|
|
value: 45.24 |
|
|
- type: recall_at_100 |
|
|
value: 64.12299999999999 |
|
|
- type: recall_at_1000 |
|
|
value: 82.28999999999999 |
|
|
- type: recall_at_3 |
|
|
value: 33.806000000000004 |
|
|
- type: recall_at_5 |
|
|
value: 38.277 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: BeIR/cqadupstack |
|
|
name: MTEB CQADupstackTexRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 16.829 |
|
|
- type: map_at_10 |
|
|
value: 23.684 |
|
|
- type: map_at_100 |
|
|
value: 24.683 |
|
|
- type: map_at_1000 |
|
|
value: 24.81 |
|
|
- type: map_at_3 |
|
|
value: 21.554000000000002 |
|
|
- type: map_at_5 |
|
|
value: 22.768 |
|
|
- type: mrr_at_1 |
|
|
value: 20.096 |
|
|
- type: mrr_at_10 |
|
|
value: 27.230999999999998 |
|
|
- type: mrr_at_100 |
|
|
value: 28.083999999999996 |
|
|
- type: mrr_at_1000 |
|
|
value: 28.166000000000004 |
|
|
- type: mrr_at_3 |
|
|
value: 25.212 |
|
|
- type: mrr_at_5 |
|
|
value: 26.32 |
|
|
- type: ndcg_at_1 |
|
|
value: 20.096 |
|
|
- type: ndcg_at_10 |
|
|
value: 27.989000000000004 |
|
|
- type: ndcg_at_100 |
|
|
value: 32.847 |
|
|
- type: ndcg_at_1000 |
|
|
value: 35.896 |
|
|
- type: ndcg_at_3 |
|
|
value: 24.116 |
|
|
- type: ndcg_at_5 |
|
|
value: 25.964 |
|
|
- type: precision_at_1 |
|
|
value: 20.096 |
|
|
- type: precision_at_10 |
|
|
value: 5 |
|
|
- type: precision_at_100 |
|
|
value: 0.8750000000000001 |
|
|
- type: precision_at_1000 |
|
|
value: 0.131 |
|
|
- type: precision_at_3 |
|
|
value: 11.207 |
|
|
- type: precision_at_5 |
|
|
value: 8.08 |
|
|
- type: recall_at_1 |
|
|
value: 16.829 |
|
|
- type: recall_at_10 |
|
|
value: 37.407000000000004 |
|
|
- type: recall_at_100 |
|
|
value: 59.101000000000006 |
|
|
- type: recall_at_1000 |
|
|
value: 81.024 |
|
|
- type: recall_at_3 |
|
|
value: 26.739 |
|
|
- type: recall_at_5 |
|
|
value: 31.524 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: BeIR/cqadupstack |
|
|
name: MTEB CQADupstackUnixRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 24.138 |
|
|
- type: map_at_10 |
|
|
value: 32.275999999999996 |
|
|
- type: map_at_100 |
|
|
value: 33.416000000000004 |
|
|
- type: map_at_1000 |
|
|
value: 33.527 |
|
|
- type: map_at_3 |
|
|
value: 29.854000000000003 |
|
|
- type: map_at_5 |
|
|
value: 31.096 |
|
|
- type: mrr_at_1 |
|
|
value: 28.450999999999997 |
|
|
- type: mrr_at_10 |
|
|
value: 36.214 |
|
|
- type: mrr_at_100 |
|
|
value: 37.134 |
|
|
- type: mrr_at_1000 |
|
|
value: 37.198 |
|
|
- type: mrr_at_3 |
|
|
value: 34.001999999999995 |
|
|
- type: mrr_at_5 |
|
|
value: 35.187000000000005 |
|
|
- type: ndcg_at_1 |
|
|
value: 28.450999999999997 |
|
|
- type: ndcg_at_10 |
|
|
value: 37.166 |
|
|
- type: ndcg_at_100 |
|
|
value: 42.454 |
|
|
- type: ndcg_at_1000 |
|
|
value: 44.976 |
|
|
- type: ndcg_at_3 |
|
|
value: 32.796 |
|
|
- type: ndcg_at_5 |
|
|
value: 34.631 |
|
|
- type: precision_at_1 |
|
|
value: 28.450999999999997 |
|
|
- type: precision_at_10 |
|
|
value: 6.241 |
|
|
- type: precision_at_100 |
|
|
value: 0.9950000000000001 |
|
|
- type: precision_at_1000 |
|
|
value: 0.133 |
|
|
- type: precision_at_3 |
|
|
value: 14.801 |
|
|
- type: precision_at_5 |
|
|
value: 10.280000000000001 |
|
|
- type: recall_at_1 |
|
|
value: 24.138 |
|
|
- type: recall_at_10 |
|
|
value: 48.111 |
|
|
- type: recall_at_100 |
|
|
value: 71.245 |
|
|
- type: recall_at_1000 |
|
|
value: 88.986 |
|
|
- type: recall_at_3 |
|
|
value: 36.119 |
|
|
- type: recall_at_5 |
|
|
value: 40.846 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: BeIR/cqadupstack |
|
|
name: MTEB CQADupstackWebmastersRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 23.244 |
|
|
- type: map_at_10 |
|
|
value: 31.227 |
|
|
- type: map_at_100 |
|
|
value: 33.007 |
|
|
- type: map_at_1000 |
|
|
value: 33.223 |
|
|
- type: map_at_3 |
|
|
value: 28.924 |
|
|
- type: map_at_5 |
|
|
value: 30.017 |
|
|
- type: mrr_at_1 |
|
|
value: 27.668 |
|
|
- type: mrr_at_10 |
|
|
value: 35.524 |
|
|
- type: mrr_at_100 |
|
|
value: 36.699 |
|
|
- type: mrr_at_1000 |
|
|
value: 36.759 |
|
|
- type: mrr_at_3 |
|
|
value: 33.366 |
|
|
- type: mrr_at_5 |
|
|
value: 34.552 |
|
|
- type: ndcg_at_1 |
|
|
value: 27.668 |
|
|
- type: ndcg_at_10 |
|
|
value: 36.381 |
|
|
- type: ndcg_at_100 |
|
|
value: 43.062 |
|
|
- type: ndcg_at_1000 |
|
|
value: 45.656 |
|
|
- type: ndcg_at_3 |
|
|
value: 32.501999999999995 |
|
|
- type: ndcg_at_5 |
|
|
value: 34.105999999999995 |
|
|
- type: precision_at_1 |
|
|
value: 27.668 |
|
|
- type: precision_at_10 |
|
|
value: 6.798 |
|
|
- type: precision_at_100 |
|
|
value: 1.492 |
|
|
- type: precision_at_1000 |
|
|
value: 0.234 |
|
|
- type: precision_at_3 |
|
|
value: 15.152 |
|
|
- type: precision_at_5 |
|
|
value: 10.791 |
|
|
- type: recall_at_1 |
|
|
value: 23.244 |
|
|
- type: recall_at_10 |
|
|
value: 45.979 |
|
|
- type: recall_at_100 |
|
|
value: 74.822 |
|
|
- type: recall_at_1000 |
|
|
value: 91.078 |
|
|
- type: recall_at_3 |
|
|
value: 34.925 |
|
|
- type: recall_at_5 |
|
|
value: 39.126 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: BeIR/cqadupstack |
|
|
name: MTEB CQADupstackWordpressRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 19.945 |
|
|
- type: map_at_10 |
|
|
value: 27.517999999999997 |
|
|
- type: map_at_100 |
|
|
value: 28.588 |
|
|
- type: map_at_1000 |
|
|
value: 28.682000000000002 |
|
|
- type: map_at_3 |
|
|
value: 25.345000000000002 |
|
|
- type: map_at_5 |
|
|
value: 26.555 |
|
|
- type: mrr_at_1 |
|
|
value: 21.996 |
|
|
- type: mrr_at_10 |
|
|
value: 29.845 |
|
|
- type: mrr_at_100 |
|
|
value: 30.775999999999996 |
|
|
- type: mrr_at_1000 |
|
|
value: 30.845 |
|
|
- type: mrr_at_3 |
|
|
value: 27.726 |
|
|
- type: mrr_at_5 |
|
|
value: 28.882 |
|
|
- type: ndcg_at_1 |
|
|
value: 21.996 |
|
|
- type: ndcg_at_10 |
|
|
value: 32.034 |
|
|
- type: ndcg_at_100 |
|
|
value: 37.185 |
|
|
- type: ndcg_at_1000 |
|
|
value: 39.645 |
|
|
- type: ndcg_at_3 |
|
|
value: 27.750999999999998 |
|
|
- type: ndcg_at_5 |
|
|
value: 29.805999999999997 |
|
|
- type: precision_at_1 |
|
|
value: 21.996 |
|
|
- type: precision_at_10 |
|
|
value: 5.065 |
|
|
- type: precision_at_100 |
|
|
value: 0.819 |
|
|
- type: precision_at_1000 |
|
|
value: 0.11399999999999999 |
|
|
- type: precision_at_3 |
|
|
value: 12.076 |
|
|
- type: precision_at_5 |
|
|
value: 8.392 |
|
|
- type: recall_at_1 |
|
|
value: 19.945 |
|
|
- type: recall_at_10 |
|
|
value: 43.62 |
|
|
- type: recall_at_100 |
|
|
value: 67.194 |
|
|
- type: recall_at_1000 |
|
|
value: 85.7 |
|
|
- type: recall_at_3 |
|
|
value: 32.15 |
|
|
- type: recall_at_5 |
|
|
value: 37.208999999999996 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: climate-fever |
|
|
name: MTEB ClimateFEVER |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 18.279 |
|
|
- type: map_at_10 |
|
|
value: 31.052999999999997 |
|
|
- type: map_at_100 |
|
|
value: 33.125 |
|
|
- type: map_at_1000 |
|
|
value: 33.306000000000004 |
|
|
- type: map_at_3 |
|
|
value: 26.208 |
|
|
- type: map_at_5 |
|
|
value: 28.857 |
|
|
- type: mrr_at_1 |
|
|
value: 42.671 |
|
|
- type: mrr_at_10 |
|
|
value: 54.557 |
|
|
- type: mrr_at_100 |
|
|
value: 55.142 |
|
|
- type: mrr_at_1000 |
|
|
value: 55.169000000000004 |
|
|
- type: mrr_at_3 |
|
|
value: 51.488 |
|
|
- type: mrr_at_5 |
|
|
value: 53.439 |
|
|
- type: ndcg_at_1 |
|
|
value: 42.671 |
|
|
- type: ndcg_at_10 |
|
|
value: 41.276 |
|
|
- type: ndcg_at_100 |
|
|
value: 48.376000000000005 |
|
|
- type: ndcg_at_1000 |
|
|
value: 51.318 |
|
|
- type: ndcg_at_3 |
|
|
value: 35.068 |
|
|
- type: ndcg_at_5 |
|
|
value: 37.242 |
|
|
- type: precision_at_1 |
|
|
value: 42.671 |
|
|
- type: precision_at_10 |
|
|
value: 12.638 |
|
|
- type: precision_at_100 |
|
|
value: 2.045 |
|
|
- type: precision_at_1000 |
|
|
value: 0.26 |
|
|
- type: precision_at_3 |
|
|
value: 26.08 |
|
|
- type: precision_at_5 |
|
|
value: 19.805 |
|
|
- type: recall_at_1 |
|
|
value: 18.279 |
|
|
- type: recall_at_10 |
|
|
value: 46.946 |
|
|
- type: recall_at_100 |
|
|
value: 70.97200000000001 |
|
|
- type: recall_at_1000 |
|
|
value: 87.107 |
|
|
- type: recall_at_3 |
|
|
value: 31.147999999999996 |
|
|
- type: recall_at_5 |
|
|
value: 38.099 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: dbpedia-entity |
|
|
name: MTEB DBPedia |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 8.573 |
|
|
- type: map_at_10 |
|
|
value: 19.747 |
|
|
- type: map_at_100 |
|
|
value: 28.205000000000002 |
|
|
- type: map_at_1000 |
|
|
value: 29.831000000000003 |
|
|
- type: map_at_3 |
|
|
value: 14.109 |
|
|
- type: map_at_5 |
|
|
value: 16.448999999999998 |
|
|
- type: mrr_at_1 |
|
|
value: 71 |
|
|
- type: mrr_at_10 |
|
|
value: 77.68599999999999 |
|
|
- type: mrr_at_100 |
|
|
value: 77.995 |
|
|
- type: mrr_at_1000 |
|
|
value: 78.00200000000001 |
|
|
- type: mrr_at_3 |
|
|
value: 76.292 |
|
|
- type: mrr_at_5 |
|
|
value: 77.029 |
|
|
- type: ndcg_at_1 |
|
|
value: 59.12500000000001 |
|
|
- type: ndcg_at_10 |
|
|
value: 43.9 |
|
|
- type: ndcg_at_100 |
|
|
value: 47.863 |
|
|
- type: ndcg_at_1000 |
|
|
value: 54.848 |
|
|
- type: ndcg_at_3 |
|
|
value: 49.803999999999995 |
|
|
- type: ndcg_at_5 |
|
|
value: 46.317 |
|
|
- type: precision_at_1 |
|
|
value: 71 |
|
|
- type: precision_at_10 |
|
|
value: 34.4 |
|
|
- type: precision_at_100 |
|
|
value: 11.063 |
|
|
- type: precision_at_1000 |
|
|
value: 1.989 |
|
|
- type: precision_at_3 |
|
|
value: 52.333 |
|
|
- type: precision_at_5 |
|
|
value: 43.7 |
|
|
- type: recall_at_1 |
|
|
value: 8.573 |
|
|
- type: recall_at_10 |
|
|
value: 25.615 |
|
|
- type: recall_at_100 |
|
|
value: 53.385000000000005 |
|
|
- type: recall_at_1000 |
|
|
value: 75.46000000000001 |
|
|
- type: recall_at_3 |
|
|
value: 15.429 |
|
|
- type: recall_at_5 |
|
|
value: 19.357 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/emotion |
|
|
name: MTEB EmotionClassification |
|
|
config: default |
|
|
split: test |
|
|
revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 47.989999999999995 |
|
|
- type: f1 |
|
|
value: 42.776314451497555 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: fever |
|
|
name: MTEB FEVER |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 74.13499999999999 |
|
|
- type: map_at_10 |
|
|
value: 82.825 |
|
|
- type: map_at_100 |
|
|
value: 83.096 |
|
|
- type: map_at_1000 |
|
|
value: 83.111 |
|
|
- type: map_at_3 |
|
|
value: 81.748 |
|
|
- type: map_at_5 |
|
|
value: 82.446 |
|
|
- type: mrr_at_1 |
|
|
value: 79.553 |
|
|
- type: mrr_at_10 |
|
|
value: 86.654 |
|
|
- type: mrr_at_100 |
|
|
value: 86.774 |
|
|
- type: mrr_at_1000 |
|
|
value: 86.778 |
|
|
- type: mrr_at_3 |
|
|
value: 85.981 |
|
|
- type: mrr_at_5 |
|
|
value: 86.462 |
|
|
- type: ndcg_at_1 |
|
|
value: 79.553 |
|
|
- type: ndcg_at_10 |
|
|
value: 86.345 |
|
|
- type: ndcg_at_100 |
|
|
value: 87.32 |
|
|
- type: ndcg_at_1000 |
|
|
value: 87.58200000000001 |
|
|
- type: ndcg_at_3 |
|
|
value: 84.719 |
|
|
- type: ndcg_at_5 |
|
|
value: 85.677 |
|
|
- type: precision_at_1 |
|
|
value: 79.553 |
|
|
- type: precision_at_10 |
|
|
value: 10.402000000000001 |
|
|
- type: precision_at_100 |
|
|
value: 1.1119999999999999 |
|
|
- type: precision_at_1000 |
|
|
value: 0.11499999999999999 |
|
|
- type: precision_at_3 |
|
|
value: 32.413 |
|
|
- type: precision_at_5 |
|
|
value: 20.138 |
|
|
- type: recall_at_1 |
|
|
value: 74.13499999999999 |
|
|
- type: recall_at_10 |
|
|
value: 93.215 |
|
|
- type: recall_at_100 |
|
|
value: 97.083 |
|
|
- type: recall_at_1000 |
|
|
value: 98.732 |
|
|
- type: recall_at_3 |
|
|
value: 88.79 |
|
|
- type: recall_at_5 |
|
|
value: 91.259 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: fiqa |
|
|
name: MTEB FiQA2018 |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 18.298000000000002 |
|
|
- type: map_at_10 |
|
|
value: 29.901 |
|
|
- type: map_at_100 |
|
|
value: 31.528 |
|
|
- type: map_at_1000 |
|
|
value: 31.713 |
|
|
- type: map_at_3 |
|
|
value: 25.740000000000002 |
|
|
- type: map_at_5 |
|
|
value: 28.227999999999998 |
|
|
- type: mrr_at_1 |
|
|
value: 36.728 |
|
|
- type: mrr_at_10 |
|
|
value: 45.401 |
|
|
- type: mrr_at_100 |
|
|
value: 46.27 |
|
|
- type: mrr_at_1000 |
|
|
value: 46.315 |
|
|
- type: mrr_at_3 |
|
|
value: 42.978 |
|
|
- type: mrr_at_5 |
|
|
value: 44.29 |
|
|
- type: ndcg_at_1 |
|
|
value: 36.728 |
|
|
- type: ndcg_at_10 |
|
|
value: 37.456 |
|
|
- type: ndcg_at_100 |
|
|
value: 43.832 |
|
|
- type: ndcg_at_1000 |
|
|
value: 47 |
|
|
- type: ndcg_at_3 |
|
|
value: 33.694 |
|
|
- type: ndcg_at_5 |
|
|
value: 35.085 |
|
|
- type: precision_at_1 |
|
|
value: 36.728 |
|
|
- type: precision_at_10 |
|
|
value: 10.386 |
|
|
- type: precision_at_100 |
|
|
value: 1.701 |
|
|
- type: precision_at_1000 |
|
|
value: 0.22599999999999998 |
|
|
- type: precision_at_3 |
|
|
value: 22.479 |
|
|
- type: precision_at_5 |
|
|
value: 16.605 |
|
|
- type: recall_at_1 |
|
|
value: 18.298000000000002 |
|
|
- type: recall_at_10 |
|
|
value: 44.369 |
|
|
- type: recall_at_100 |
|
|
value: 68.098 |
|
|
- type: recall_at_1000 |
|
|
value: 87.21900000000001 |
|
|
- type: recall_at_3 |
|
|
value: 30.215999999999998 |
|
|
- type: recall_at_5 |
|
|
value: 36.861 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: hotpotqa |
|
|
name: MTEB HotpotQA |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 39.568 |
|
|
- type: map_at_10 |
|
|
value: 65.061 |
|
|
- type: map_at_100 |
|
|
value: 65.896 |
|
|
- type: map_at_1000 |
|
|
value: 65.95100000000001 |
|
|
- type: map_at_3 |
|
|
value: 61.831 |
|
|
- type: map_at_5 |
|
|
value: 63.849000000000004 |
|
|
- type: mrr_at_1 |
|
|
value: 79.136 |
|
|
- type: mrr_at_10 |
|
|
value: 84.58200000000001 |
|
|
- type: mrr_at_100 |
|
|
value: 84.765 |
|
|
- type: mrr_at_1000 |
|
|
value: 84.772 |
|
|
- type: mrr_at_3 |
|
|
value: 83.684 |
|
|
- type: mrr_at_5 |
|
|
value: 84.223 |
|
|
- type: ndcg_at_1 |
|
|
value: 79.136 |
|
|
- type: ndcg_at_10 |
|
|
value: 72.622 |
|
|
- type: ndcg_at_100 |
|
|
value: 75.539 |
|
|
- type: ndcg_at_1000 |
|
|
value: 76.613 |
|
|
- type: ndcg_at_3 |
|
|
value: 68.065 |
|
|
- type: ndcg_at_5 |
|
|
value: 70.58 |
|
|
- type: precision_at_1 |
|
|
value: 79.136 |
|
|
- type: precision_at_10 |
|
|
value: 15.215 |
|
|
- type: precision_at_100 |
|
|
value: 1.7500000000000002 |
|
|
- type: precision_at_1000 |
|
|
value: 0.189 |
|
|
- type: precision_at_3 |
|
|
value: 44.011 |
|
|
- type: precision_at_5 |
|
|
value: 28.388999999999996 |
|
|
- type: recall_at_1 |
|
|
value: 39.568 |
|
|
- type: recall_at_10 |
|
|
value: 76.077 |
|
|
- type: recall_at_100 |
|
|
value: 87.481 |
|
|
- type: recall_at_1000 |
|
|
value: 94.56400000000001 |
|
|
- type: recall_at_3 |
|
|
value: 66.01599999999999 |
|
|
- type: recall_at_5 |
|
|
value: 70.97200000000001 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/imdb |
|
|
name: MTEB ImdbClassification |
|
|
config: default |
|
|
split: test |
|
|
revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 85.312 |
|
|
- type: ap |
|
|
value: 80.36296867333715 |
|
|
- type: f1 |
|
|
value: 85.26613311552218 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: msmarco |
|
|
name: MTEB MSMARCO |
|
|
config: default |
|
|
split: dev |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 23.363999999999997 |
|
|
- type: map_at_10 |
|
|
value: 35.711999999999996 |
|
|
- type: map_at_100 |
|
|
value: 36.876999999999995 |
|
|
- type: map_at_1000 |
|
|
value: 36.923 |
|
|
- type: map_at_3 |
|
|
value: 32.034 |
|
|
- type: map_at_5 |
|
|
value: 34.159 |
|
|
- type: mrr_at_1 |
|
|
value: 24.04 |
|
|
- type: mrr_at_10 |
|
|
value: 36.345 |
|
|
- type: mrr_at_100 |
|
|
value: 37.441 |
|
|
- type: mrr_at_1000 |
|
|
value: 37.480000000000004 |
|
|
- type: mrr_at_3 |
|
|
value: 32.713 |
|
|
- type: mrr_at_5 |
|
|
value: 34.824 |
|
|
- type: ndcg_at_1 |
|
|
value: 24.026 |
|
|
- type: ndcg_at_10 |
|
|
value: 42.531 |
|
|
- type: ndcg_at_100 |
|
|
value: 48.081 |
|
|
- type: ndcg_at_1000 |
|
|
value: 49.213 |
|
|
- type: ndcg_at_3 |
|
|
value: 35.044 |
|
|
- type: ndcg_at_5 |
|
|
value: 38.834 |
|
|
- type: precision_at_1 |
|
|
value: 24.026 |
|
|
- type: precision_at_10 |
|
|
value: 6.622999999999999 |
|
|
- type: precision_at_100 |
|
|
value: 0.941 |
|
|
- type: precision_at_1000 |
|
|
value: 0.104 |
|
|
- type: precision_at_3 |
|
|
value: 14.909 |
|
|
- type: precision_at_5 |
|
|
value: 10.871 |
|
|
- type: recall_at_1 |
|
|
value: 23.363999999999997 |
|
|
- type: recall_at_10 |
|
|
value: 63.426 |
|
|
- type: recall_at_100 |
|
|
value: 88.96300000000001 |
|
|
- type: recall_at_1000 |
|
|
value: 97.637 |
|
|
- type: recall_at_3 |
|
|
value: 43.095 |
|
|
- type: recall_at_5 |
|
|
value: 52.178000000000004 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/mtop_domain |
|
|
name: MTEB MTOPDomainClassification (en) |
|
|
config: en |
|
|
split: test |
|
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 93.0095759233926 |
|
|
- type: f1 |
|
|
value: 92.78387794667408 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/mtop_intent |
|
|
name: MTEB MTOPIntentClassification (en) |
|
|
config: en |
|
|
split: test |
|
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 75.0296397628819 |
|
|
- type: f1 |
|
|
value: 58.45699589820874 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (en) |
|
|
config: en |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 73.45662407531944 |
|
|
- type: f1 |
|
|
value: 71.42364781421813 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (en) |
|
|
config: en |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 77.07800941492937 |
|
|
- type: f1 |
|
|
value: 77.22799045640845 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/medrxiv-clustering-p2p |
|
|
name: MTEB MedrxivClusteringP2P |
|
|
config: default |
|
|
split: test |
|
|
revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 34.531234379250606 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/medrxiv-clustering-s2s |
|
|
name: MTEB MedrxivClusteringS2S |
|
|
config: default |
|
|
split: test |
|
|
revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 30.941490381193802 |
|
|
- task: |
|
|
type: Reranking |
|
|
dataset: |
|
|
type: mteb/mind_small |
|
|
name: MTEB MindSmallReranking |
|
|
config: default |
|
|
split: test |
|
|
revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69 |
|
|
metrics: |
|
|
- type: map |
|
|
value: 30.3115090856725 |
|
|
- type: mrr |
|
|
value: 31.290667638675757 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: nfcorpus |
|
|
name: MTEB NFCorpus |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 5.465 |
|
|
- type: map_at_10 |
|
|
value: 13.03 |
|
|
- type: map_at_100 |
|
|
value: 16.057 |
|
|
- type: map_at_1000 |
|
|
value: 17.49 |
|
|
- type: map_at_3 |
|
|
value: 9.553 |
|
|
- type: map_at_5 |
|
|
value: 11.204 |
|
|
- type: mrr_at_1 |
|
|
value: 43.653 |
|
|
- type: mrr_at_10 |
|
|
value: 53.269 |
|
|
- type: mrr_at_100 |
|
|
value: 53.72 |
|
|
- type: mrr_at_1000 |
|
|
value: 53.761 |
|
|
- type: mrr_at_3 |
|
|
value: 50.929 |
|
|
- type: mrr_at_5 |
|
|
value: 52.461 |
|
|
- type: ndcg_at_1 |
|
|
value: 42.26 |
|
|
- type: ndcg_at_10 |
|
|
value: 34.673 |
|
|
- type: ndcg_at_100 |
|
|
value: 30.759999999999998 |
|
|
- type: ndcg_at_1000 |
|
|
value: 39.728 |
|
|
- type: ndcg_at_3 |
|
|
value: 40.349000000000004 |
|
|
- type: ndcg_at_5 |
|
|
value: 37.915 |
|
|
- type: precision_at_1 |
|
|
value: 43.653 |
|
|
- type: precision_at_10 |
|
|
value: 25.789 |
|
|
- type: precision_at_100 |
|
|
value: 7.754999999999999 |
|
|
- type: precision_at_1000 |
|
|
value: 2.07 |
|
|
- type: precision_at_3 |
|
|
value: 38.596000000000004 |
|
|
- type: precision_at_5 |
|
|
value: 33.251 |
|
|
- type: recall_at_1 |
|
|
value: 5.465 |
|
|
- type: recall_at_10 |
|
|
value: 17.148 |
|
|
- type: recall_at_100 |
|
|
value: 29.768 |
|
|
- type: recall_at_1000 |
|
|
value: 62.239 |
|
|
- type: recall_at_3 |
|
|
value: 10.577 |
|
|
- type: recall_at_5 |
|
|
value: 13.315 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: nq |
|
|
name: MTEB NQ |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 37.008 |
|
|
- type: map_at_10 |
|
|
value: 52.467 |
|
|
- type: map_at_100 |
|
|
value: 53.342999999999996 |
|
|
- type: map_at_1000 |
|
|
value: 53.366 |
|
|
- type: map_at_3 |
|
|
value: 48.412 |
|
|
- type: map_at_5 |
|
|
value: 50.875 |
|
|
- type: mrr_at_1 |
|
|
value: 41.541 |
|
|
- type: mrr_at_10 |
|
|
value: 54.967 |
|
|
- type: mrr_at_100 |
|
|
value: 55.611 |
|
|
- type: mrr_at_1000 |
|
|
value: 55.627 |
|
|
- type: mrr_at_3 |
|
|
value: 51.824999999999996 |
|
|
- type: mrr_at_5 |
|
|
value: 53.763000000000005 |
|
|
- type: ndcg_at_1 |
|
|
value: 41.541 |
|
|
- type: ndcg_at_10 |
|
|
value: 59.724999999999994 |
|
|
- type: ndcg_at_100 |
|
|
value: 63.38700000000001 |
|
|
- type: ndcg_at_1000 |
|
|
value: 63.883 |
|
|
- type: ndcg_at_3 |
|
|
value: 52.331 |
|
|
- type: ndcg_at_5 |
|
|
value: 56.327000000000005 |
|
|
- type: precision_at_1 |
|
|
value: 41.541 |
|
|
- type: precision_at_10 |
|
|
value: 9.447 |
|
|
- type: precision_at_100 |
|
|
value: 1.1520000000000001 |
|
|
- type: precision_at_1000 |
|
|
value: 0.12 |
|
|
- type: precision_at_3 |
|
|
value: 23.262 |
|
|
- type: precision_at_5 |
|
|
value: 16.314999999999998 |
|
|
- type: recall_at_1 |
|
|
value: 37.008 |
|
|
- type: recall_at_10 |
|
|
value: 79.145 |
|
|
- type: recall_at_100 |
|
|
value: 94.986 |
|
|
- type: recall_at_1000 |
|
|
value: 98.607 |
|
|
- type: recall_at_3 |
|
|
value: 60.277 |
|
|
- type: recall_at_5 |
|
|
value: 69.407 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: quora |
|
|
name: MTEB QuoraRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 70.402 |
|
|
- type: map_at_10 |
|
|
value: 84.181 |
|
|
- type: map_at_100 |
|
|
value: 84.796 |
|
|
- type: map_at_1000 |
|
|
value: 84.81400000000001 |
|
|
- type: map_at_3 |
|
|
value: 81.209 |
|
|
- type: map_at_5 |
|
|
value: 83.085 |
|
|
- type: mrr_at_1 |
|
|
value: 81.02000000000001 |
|
|
- type: mrr_at_10 |
|
|
value: 87.263 |
|
|
- type: mrr_at_100 |
|
|
value: 87.36 |
|
|
- type: mrr_at_1000 |
|
|
value: 87.36 |
|
|
- type: mrr_at_3 |
|
|
value: 86.235 |
|
|
- type: mrr_at_5 |
|
|
value: 86.945 |
|
|
- type: ndcg_at_1 |
|
|
value: 81.01 |
|
|
- type: ndcg_at_10 |
|
|
value: 87.99900000000001 |
|
|
- type: ndcg_at_100 |
|
|
value: 89.217 |
|
|
- type: ndcg_at_1000 |
|
|
value: 89.33 |
|
|
- type: ndcg_at_3 |
|
|
value: 85.053 |
|
|
- type: ndcg_at_5 |
|
|
value: 86.703 |
|
|
- type: precision_at_1 |
|
|
value: 81.01 |
|
|
- type: precision_at_10 |
|
|
value: 13.336 |
|
|
- type: precision_at_100 |
|
|
value: 1.52 |
|
|
- type: precision_at_1000 |
|
|
value: 0.156 |
|
|
- type: precision_at_3 |
|
|
value: 37.14 |
|
|
- type: precision_at_5 |
|
|
value: 24.44 |
|
|
- type: recall_at_1 |
|
|
value: 70.402 |
|
|
- type: recall_at_10 |
|
|
value: 95.214 |
|
|
- type: recall_at_100 |
|
|
value: 99.438 |
|
|
- type: recall_at_1000 |
|
|
value: 99.928 |
|
|
- type: recall_at_3 |
|
|
value: 86.75699999999999 |
|
|
- type: recall_at_5 |
|
|
value: 91.44099999999999 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/reddit-clustering |
|
|
name: MTEB RedditClustering |
|
|
config: default |
|
|
split: test |
|
|
revision: 24640382cdbf8abc73003fb0fa6d111a705499eb |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 56.51721502758904 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/reddit-clustering-p2p |
|
|
name: MTEB RedditClusteringP2P |
|
|
config: default |
|
|
split: test |
|
|
revision: 282350215ef01743dc01b456c7f5241fa8937f16 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 61.054808572333016 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: scidocs |
|
|
name: MTEB SCIDOCS |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 4.578 |
|
|
- type: map_at_10 |
|
|
value: 11.036999999999999 |
|
|
- type: map_at_100 |
|
|
value: 12.879999999999999 |
|
|
- type: map_at_1000 |
|
|
value: 13.150999999999998 |
|
|
- type: map_at_3 |
|
|
value: 8.133 |
|
|
- type: map_at_5 |
|
|
value: 9.559 |
|
|
- type: mrr_at_1 |
|
|
value: 22.6 |
|
|
- type: mrr_at_10 |
|
|
value: 32.68 |
|
|
- type: mrr_at_100 |
|
|
value: 33.789 |
|
|
- type: mrr_at_1000 |
|
|
value: 33.854 |
|
|
- type: mrr_at_3 |
|
|
value: 29.7 |
|
|
- type: mrr_at_5 |
|
|
value: 31.480000000000004 |
|
|
- type: ndcg_at_1 |
|
|
value: 22.6 |
|
|
- type: ndcg_at_10 |
|
|
value: 18.616 |
|
|
- type: ndcg_at_100 |
|
|
value: 25.883 |
|
|
- type: ndcg_at_1000 |
|
|
value: 30.944 |
|
|
- type: ndcg_at_3 |
|
|
value: 18.136 |
|
|
- type: ndcg_at_5 |
|
|
value: 15.625 |
|
|
- type: precision_at_1 |
|
|
value: 22.6 |
|
|
- type: precision_at_10 |
|
|
value: 9.48 |
|
|
- type: precision_at_100 |
|
|
value: 1.991 |
|
|
- type: precision_at_1000 |
|
|
value: 0.321 |
|
|
- type: precision_at_3 |
|
|
value: 16.8 |
|
|
- type: precision_at_5 |
|
|
value: 13.54 |
|
|
- type: recall_at_1 |
|
|
value: 4.578 |
|
|
- type: recall_at_10 |
|
|
value: 19.213 |
|
|
- type: recall_at_100 |
|
|
value: 40.397 |
|
|
- type: recall_at_1000 |
|
|
value: 65.2 |
|
|
- type: recall_at_3 |
|
|
value: 10.208 |
|
|
- type: recall_at_5 |
|
|
value: 13.718 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/sickr-sts |
|
|
name: MTEB SICK-R |
|
|
config: default |
|
|
split: test |
|
|
revision: a6ea5a8cab320b040a23452cc28066d9beae2cee |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 83.44288351714071 |
|
|
- type: cos_sim_spearman |
|
|
value: 79.37995604564952 |
|
|
- type: euclidean_pearson |
|
|
value: 81.1078874670718 |
|
|
- type: euclidean_spearman |
|
|
value: 79.37995905980499 |
|
|
- type: manhattan_pearson |
|
|
value: 81.03697527288986 |
|
|
- type: manhattan_spearman |
|
|
value: 79.33490235296236 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/sts12-sts |
|
|
name: MTEB STS12 |
|
|
config: default |
|
|
split: test |
|
|
revision: a0d554a64d88156834ff5ae9920b964011b16384 |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 84.95557650436523 |
|
|
- type: cos_sim_spearman |
|
|
value: 78.5190672399868 |
|
|
- type: euclidean_pearson |
|
|
value: 81.58064025904707 |
|
|
- type: euclidean_spearman |
|
|
value: 78.5190672399868 |
|
|
- type: manhattan_pearson |
|
|
value: 81.52857930619889 |
|
|
- type: manhattan_spearman |
|
|
value: 78.50421361308034 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/sts13-sts |
|
|
name: MTEB STS13 |
|
|
config: default |
|
|
split: test |
|
|
revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 84.79128416228737 |
|
|
- type: cos_sim_spearman |
|
|
value: 86.05402451477147 |
|
|
- type: euclidean_pearson |
|
|
value: 85.46280267054289 |
|
|
- type: euclidean_spearman |
|
|
value: 86.05402451477147 |
|
|
- type: manhattan_pearson |
|
|
value: 85.46278563858236 |
|
|
- type: manhattan_spearman |
|
|
value: 86.08079590861004 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/sts14-sts |
|
|
name: MTEB STS14 |
|
|
config: default |
|
|
split: test |
|
|
revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 83.20623089568763 |
|
|
- type: cos_sim_spearman |
|
|
value: 81.53786907061009 |
|
|
- type: euclidean_pearson |
|
|
value: 82.82272250091494 |
|
|
- type: euclidean_spearman |
|
|
value: 81.53786907061009 |
|
|
- type: manhattan_pearson |
|
|
value: 82.78850494027013 |
|
|
- type: manhattan_spearman |
|
|
value: 81.5135618083407 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/sts15-sts |
|
|
name: MTEB STS15 |
|
|
config: default |
|
|
split: test |
|
|
revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 85.46366618397936 |
|
|
- type: cos_sim_spearman |
|
|
value: 86.96566013336908 |
|
|
- type: euclidean_pearson |
|
|
value: 86.62651697548931 |
|
|
- type: euclidean_spearman |
|
|
value: 86.96565526364454 |
|
|
- type: manhattan_pearson |
|
|
value: 86.58812160258009 |
|
|
- type: manhattan_spearman |
|
|
value: 86.9336484321288 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/sts16-sts |
|
|
name: MTEB STS16 |
|
|
config: default |
|
|
split: test |
|
|
revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 82.51858358641559 |
|
|
- type: cos_sim_spearman |
|
|
value: 84.7652527954999 |
|
|
- type: euclidean_pearson |
|
|
value: 84.23914783766861 |
|
|
- type: euclidean_spearman |
|
|
value: 84.7652527954999 |
|
|
- type: manhattan_pearson |
|
|
value: 84.22749648503171 |
|
|
- type: manhattan_spearman |
|
|
value: 84.74527996746386 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/sts17-crosslingual-sts |
|
|
name: MTEB STS17 (en-en) |
|
|
config: en-en |
|
|
split: test |
|
|
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 87.28026563313065 |
|
|
- type: cos_sim_spearman |
|
|
value: 87.46928143824915 |
|
|
- type: euclidean_pearson |
|
|
value: 88.30558762000372 |
|
|
- type: euclidean_spearman |
|
|
value: 87.46928143824915 |
|
|
- type: manhattan_pearson |
|
|
value: 88.10513330809331 |
|
|
- type: manhattan_spearman |
|
|
value: 87.21069787834173 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/sts22-crosslingual-sts |
|
|
name: MTEB STS22 (en) |
|
|
config: en |
|
|
split: test |
|
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 62.376497134587375 |
|
|
- type: cos_sim_spearman |
|
|
value: 65.0159550112516 |
|
|
- type: euclidean_pearson |
|
|
value: 65.64572120879598 |
|
|
- type: euclidean_spearman |
|
|
value: 65.0159550112516 |
|
|
- type: manhattan_pearson |
|
|
value: 65.88143604989976 |
|
|
- type: manhattan_spearman |
|
|
value: 65.17547297222434 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/stsbenchmark-sts |
|
|
name: MTEB STSBenchmark |
|
|
config: default |
|
|
split: test |
|
|
revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 84.22876368947644 |
|
|
- type: cos_sim_spearman |
|
|
value: 85.46935577445318 |
|
|
- type: euclidean_pearson |
|
|
value: 85.32830231392005 |
|
|
- type: euclidean_spearman |
|
|
value: 85.46935577445318 |
|
|
- type: manhattan_pearson |
|
|
value: 85.30353211758495 |
|
|
- type: manhattan_spearman |
|
|
value: 85.42821085956945 |
|
|
- task: |
|
|
type: Reranking |
|
|
dataset: |
|
|
type: mteb/scidocs-reranking |
|
|
name: MTEB SciDocsRR |
|
|
config: default |
|
|
split: test |
|
|
revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab |
|
|
metrics: |
|
|
- type: map |
|
|
value: 80.60986667767133 |
|
|
- type: mrr |
|
|
value: 94.29432314236236 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: scifact |
|
|
name: MTEB SciFact |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 54.528 |
|
|
- type: map_at_10 |
|
|
value: 65.187 |
|
|
- type: map_at_100 |
|
|
value: 65.62599999999999 |
|
|
- type: map_at_1000 |
|
|
value: 65.657 |
|
|
- type: map_at_3 |
|
|
value: 62.352 |
|
|
- type: map_at_5 |
|
|
value: 64.025 |
|
|
- type: mrr_at_1 |
|
|
value: 57.333 |
|
|
- type: mrr_at_10 |
|
|
value: 66.577 |
|
|
- type: mrr_at_100 |
|
|
value: 66.88 |
|
|
- type: mrr_at_1000 |
|
|
value: 66.908 |
|
|
- type: mrr_at_3 |
|
|
value: 64.556 |
|
|
- type: mrr_at_5 |
|
|
value: 65.739 |
|
|
- type: ndcg_at_1 |
|
|
value: 57.333 |
|
|
- type: ndcg_at_10 |
|
|
value: 70.275 |
|
|
- type: ndcg_at_100 |
|
|
value: 72.136 |
|
|
- type: ndcg_at_1000 |
|
|
value: 72.963 |
|
|
- type: ndcg_at_3 |
|
|
value: 65.414 |
|
|
- type: ndcg_at_5 |
|
|
value: 67.831 |
|
|
- type: precision_at_1 |
|
|
value: 57.333 |
|
|
- type: precision_at_10 |
|
|
value: 9.5 |
|
|
- type: precision_at_100 |
|
|
value: 1.057 |
|
|
- type: precision_at_1000 |
|
|
value: 0.11199999999999999 |
|
|
- type: precision_at_3 |
|
|
value: 25.778000000000002 |
|
|
- type: precision_at_5 |
|
|
value: 17.2 |
|
|
- type: recall_at_1 |
|
|
value: 54.528 |
|
|
- type: recall_at_10 |
|
|
value: 84.356 |
|
|
- type: recall_at_100 |
|
|
value: 92.833 |
|
|
- type: recall_at_1000 |
|
|
value: 99.333 |
|
|
- type: recall_at_3 |
|
|
value: 71.283 |
|
|
- type: recall_at_5 |
|
|
value: 77.14999999999999 |
|
|
- task: |
|
|
type: PairClassification |
|
|
dataset: |
|
|
type: mteb/sprintduplicatequestions-pairclassification |
|
|
name: MTEB SprintDuplicateQuestions |
|
|
config: default |
|
|
split: test |
|
|
revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 |
|
|
metrics: |
|
|
- type: cos_sim_accuracy |
|
|
value: 99.74158415841585 |
|
|
- type: cos_sim_ap |
|
|
value: 92.90048959850317 |
|
|
- type: cos_sim_f1 |
|
|
value: 86.35650810245687 |
|
|
- type: cos_sim_precision |
|
|
value: 90.4709748083242 |
|
|
- type: cos_sim_recall |
|
|
value: 82.6 |
|
|
- type: dot_accuracy |
|
|
value: 99.74158415841585 |
|
|
- type: dot_ap |
|
|
value: 92.90048959850317 |
|
|
- type: dot_f1 |
|
|
value: 86.35650810245687 |
|
|
- type: dot_precision |
|
|
value: 90.4709748083242 |
|
|
- type: dot_recall |
|
|
value: 82.6 |
|
|
- type: euclidean_accuracy |
|
|
value: 99.74158415841585 |
|
|
- type: euclidean_ap |
|
|
value: 92.90048959850317 |
|
|
- type: euclidean_f1 |
|
|
value: 86.35650810245687 |
|
|
- type: euclidean_precision |
|
|
value: 90.4709748083242 |
|
|
- type: euclidean_recall |
|
|
value: 82.6 |
|
|
- type: manhattan_accuracy |
|
|
value: 99.74158415841585 |
|
|
- type: manhattan_ap |
|
|
value: 92.87344692947894 |
|
|
- type: manhattan_f1 |
|
|
value: 86.38497652582159 |
|
|
- type: manhattan_precision |
|
|
value: 90.29443838604145 |
|
|
- type: manhattan_recall |
|
|
value: 82.8 |
|
|
- type: max_accuracy |
|
|
value: 99.74158415841585 |
|
|
- type: max_ap |
|
|
value: 92.90048959850317 |
|
|
- type: max_f1 |
|
|
value: 86.38497652582159 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/stackexchange-clustering |
|
|
name: MTEB StackExchangeClustering |
|
|
config: default |
|
|
split: test |
|
|
revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 63.191648770424216 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/stackexchange-clustering-p2p |
|
|
name: MTEB StackExchangeClusteringP2P |
|
|
config: default |
|
|
split: test |
|
|
revision: 815ca46b2622cec33ccafc3735d572c266efdb44 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 34.02944668730218 |
|
|
- task: |
|
|
type: Reranking |
|
|
dataset: |
|
|
type: mteb/stackoverflowdupquestions-reranking |
|
|
name: MTEB StackOverflowDupQuestions |
|
|
config: default |
|
|
split: test |
|
|
revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 |
|
|
metrics: |
|
|
- type: map |
|
|
value: 50.466386167525265 |
|
|
- type: mrr |
|
|
value: 51.19071492233257 |
|
|
- task: |
|
|
type: Summarization |
|
|
dataset: |
|
|
type: mteb/summeval |
|
|
name: MTEB SummEval |
|
|
config: default |
|
|
split: test |
|
|
revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 30.198022505886435 |
|
|
- type: cos_sim_spearman |
|
|
value: 30.40170257939193 |
|
|
- type: dot_pearson |
|
|
value: 30.198015316402614 |
|
|
- type: dot_spearman |
|
|
value: 30.40170257939193 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: trec-covid |
|
|
name: MTEB TRECCOVID |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 0.242 |
|
|
- type: map_at_10 |
|
|
value: 2.17 |
|
|
- type: map_at_100 |
|
|
value: 12.221 |
|
|
- type: map_at_1000 |
|
|
value: 28.63 |
|
|
- type: map_at_3 |
|
|
value: 0.728 |
|
|
- type: map_at_5 |
|
|
value: 1.185 |
|
|
- type: mrr_at_1 |
|
|
value: 94 |
|
|
- type: mrr_at_10 |
|
|
value: 97 |
|
|
- type: mrr_at_100 |
|
|
value: 97 |
|
|
- type: mrr_at_1000 |
|
|
value: 97 |
|
|
- type: mrr_at_3 |
|
|
value: 97 |
|
|
- type: mrr_at_5 |
|
|
value: 97 |
|
|
- type: ndcg_at_1 |
|
|
value: 89 |
|
|
- type: ndcg_at_10 |
|
|
value: 82.30499999999999 |
|
|
- type: ndcg_at_100 |
|
|
value: 61.839999999999996 |
|
|
- type: ndcg_at_1000 |
|
|
value: 53.381 |
|
|
- type: ndcg_at_3 |
|
|
value: 88.877 |
|
|
- type: ndcg_at_5 |
|
|
value: 86.05199999999999 |
|
|
- type: precision_at_1 |
|
|
value: 94 |
|
|
- type: precision_at_10 |
|
|
value: 87 |
|
|
- type: precision_at_100 |
|
|
value: 63.38 |
|
|
- type: precision_at_1000 |
|
|
value: 23.498 |
|
|
- type: precision_at_3 |
|
|
value: 94 |
|
|
- type: precision_at_5 |
|
|
value: 92 |
|
|
- type: recall_at_1 |
|
|
value: 0.242 |
|
|
- type: recall_at_10 |
|
|
value: 2.302 |
|
|
- type: recall_at_100 |
|
|
value: 14.979000000000001 |
|
|
- type: recall_at_1000 |
|
|
value: 49.638 |
|
|
- type: recall_at_3 |
|
|
value: 0.753 |
|
|
- type: recall_at_5 |
|
|
value: 1.226 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: webis-touche2020 |
|
|
name: MTEB Touche2020 |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 3.006 |
|
|
- type: map_at_10 |
|
|
value: 11.805 |
|
|
- type: map_at_100 |
|
|
value: 18.146 |
|
|
- type: map_at_1000 |
|
|
value: 19.788 |
|
|
- type: map_at_3 |
|
|
value: 5.914 |
|
|
- type: map_at_5 |
|
|
value: 8.801 |
|
|
- type: mrr_at_1 |
|
|
value: 40.816 |
|
|
- type: mrr_at_10 |
|
|
value: 56.36600000000001 |
|
|
- type: mrr_at_100 |
|
|
value: 56.721999999999994 |
|
|
- type: mrr_at_1000 |
|
|
value: 56.721999999999994 |
|
|
- type: mrr_at_3 |
|
|
value: 52.041000000000004 |
|
|
- type: mrr_at_5 |
|
|
value: 54.796 |
|
|
- type: ndcg_at_1 |
|
|
value: 37.755 |
|
|
- type: ndcg_at_10 |
|
|
value: 29.863 |
|
|
- type: ndcg_at_100 |
|
|
value: 39.571 |
|
|
- type: ndcg_at_1000 |
|
|
value: 51.385999999999996 |
|
|
- type: ndcg_at_3 |
|
|
value: 32.578 |
|
|
- type: ndcg_at_5 |
|
|
value: 32.351 |
|
|
- type: precision_at_1 |
|
|
value: 40.816 |
|
|
- type: precision_at_10 |
|
|
value: 26.531 |
|
|
- type: precision_at_100 |
|
|
value: 7.796 |
|
|
- type: precision_at_1000 |
|
|
value: 1.555 |
|
|
- type: precision_at_3 |
|
|
value: 32.653 |
|
|
- type: precision_at_5 |
|
|
value: 33.061 |
|
|
- type: recall_at_1 |
|
|
value: 3.006 |
|
|
- type: recall_at_10 |
|
|
value: 18.738 |
|
|
- type: recall_at_100 |
|
|
value: 48.058 |
|
|
- type: recall_at_1000 |
|
|
value: 83.41300000000001 |
|
|
- type: recall_at_3 |
|
|
value: 7.166 |
|
|
- type: recall_at_5 |
|
|
value: 12.102 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/toxic_conversations_50k |
|
|
name: MTEB ToxicConversationsClassification |
|
|
config: default |
|
|
split: test |
|
|
revision: d7c0de2777da35d6aae2200a62c6e0e5af397c4c |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 71.4178 |
|
|
- type: ap |
|
|
value: 14.648781342150446 |
|
|
- type: f1 |
|
|
value: 55.07299194946378 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/tweet_sentiment_extraction |
|
|
name: MTEB TweetSentimentExtractionClassification |
|
|
config: default |
|
|
split: test |
|
|
revision: d604517c81ca91fe16a244d1248fc021f9ecee7a |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 60.919637804187886 |
|
|
- type: f1 |
|
|
value: 61.24122013967399 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/twentynewsgroups-clustering |
|
|
name: MTEB TwentyNewsgroupsClustering |
|
|
config: default |
|
|
split: test |
|
|
revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 49.207896583685695 |
|
|
- task: |
|
|
type: PairClassification |
|
|
dataset: |
|
|
type: mteb/twittersemeval2015-pairclassification |
|
|
name: MTEB TwitterSemEval2015 |
|
|
config: default |
|
|
split: test |
|
|
revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 |
|
|
metrics: |
|
|
- type: cos_sim_accuracy |
|
|
value: 86.23114978840078 |
|
|
- type: cos_sim_ap |
|
|
value: 74.26624727825818 |
|
|
- type: cos_sim_f1 |
|
|
value: 68.72377190817083 |
|
|
- type: cos_sim_precision |
|
|
value: 64.56400742115028 |
|
|
- type: cos_sim_recall |
|
|
value: 73.45646437994723 |
|
|
- type: dot_accuracy |
|
|
value: 86.23114978840078 |
|
|
- type: dot_ap |
|
|
value: 74.26624032659652 |
|
|
- type: dot_f1 |
|
|
value: 68.72377190817083 |
|
|
- type: dot_precision |
|
|
value: 64.56400742115028 |
|
|
- type: dot_recall |
|
|
value: 73.45646437994723 |
|
|
- type: euclidean_accuracy |
|
|
value: 86.23114978840078 |
|
|
- type: euclidean_ap |
|
|
value: 74.26624714480556 |
|
|
- type: euclidean_f1 |
|
|
value: 68.72377190817083 |
|
|
- type: euclidean_precision |
|
|
value: 64.56400742115028 |
|
|
- type: euclidean_recall |
|
|
value: 73.45646437994723 |
|
|
- type: manhattan_accuracy |
|
|
value: 86.16558383501221 |
|
|
- type: manhattan_ap |
|
|
value: 74.2091943976357 |
|
|
- type: manhattan_f1 |
|
|
value: 68.64221520524654 |
|
|
- type: manhattan_precision |
|
|
value: 63.59135913591359 |
|
|
- type: manhattan_recall |
|
|
value: 74.5646437994723 |
|
|
- type: max_accuracy |
|
|
value: 86.23114978840078 |
|
|
- type: max_ap |
|
|
value: 74.26624727825818 |
|
|
- type: max_f1 |
|
|
value: 68.72377190817083 |
|
|
- task: |
|
|
type: PairClassification |
|
|
dataset: |
|
|
type: mteb/twitterurlcorpus-pairclassification |
|
|
name: MTEB TwitterURLCorpus |
|
|
config: default |
|
|
split: test |
|
|
revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf |
|
|
metrics: |
|
|
- type: cos_sim_accuracy |
|
|
value: 89.3681841114604 |
|
|
- type: cos_sim_ap |
|
|
value: 86.65166387498546 |
|
|
- type: cos_sim_f1 |
|
|
value: 79.02581944698774 |
|
|
- type: cos_sim_precision |
|
|
value: 75.35796605434099 |
|
|
- type: cos_sim_recall |
|
|
value: 83.06898675700647 |
|
|
- type: dot_accuracy |
|
|
value: 89.3681841114604 |
|
|
- type: dot_ap |
|
|
value: 86.65166019802056 |
|
|
- type: dot_f1 |
|
|
value: 79.02581944698774 |
|
|
- type: dot_precision |
|
|
value: 75.35796605434099 |
|
|
- type: dot_recall |
|
|
value: 83.06898675700647 |
|
|
- type: euclidean_accuracy |
|
|
value: 89.3681841114604 |
|
|
- type: euclidean_ap |
|
|
value: 86.65166462876266 |
|
|
- type: euclidean_f1 |
|
|
value: 79.02581944698774 |
|
|
- type: euclidean_precision |
|
|
value: 75.35796605434099 |
|
|
- type: euclidean_recall |
|
|
value: 83.06898675700647 |
|
|
- type: manhattan_accuracy |
|
|
value: 89.36624364497226 |
|
|
- type: manhattan_ap |
|
|
value: 86.65076471274106 |
|
|
- type: manhattan_f1 |
|
|
value: 79.07408783532733 |
|
|
- type: manhattan_precision |
|
|
value: 76.41102972856527 |
|
|
- type: manhattan_recall |
|
|
value: 81.92947336002464 |
|
|
- type: max_accuracy |
|
|
value: 89.3681841114604 |
|
|
- type: max_ap |
|
|
value: 86.65166462876266 |
|
|
- type: max_f1 |
|
|
value: 79.07408783532733 |
|
|
license: apache-2.0 |
|
|
language: |
|
|
- en |
|
|
--- |
|
|
|
|
|
# nomic-embed-text-v1.5: Resizable Production Embeddings with Matryoshka Representation Learning |
|
|
|
|
|
[Blog](https://www.nomic.ai/blog/posts/nomic-embed-text-v1) | [Technical Report](https://arxiv.org/abs/2402.01613) | [AWS SageMaker](https://aws.amazon.com/marketplace/seller-profile?id=seller-tpqidcj54zawi) | [Nomic Platform](https://atlas.nomic.ai) |
|
|
|
|
|
**Exciting Update!**: `nomic-embed-text-v1.5` is now multimodal! [nomic-embed-vision-v1.5](https://huggingface.co/nomic-ai/nomic-embed-vision-v1.5) is aligned to the embedding space of `nomic-embed-text-v1.5`, meaning any text embedding is multimodal! |
|
|
|
|
|
## Usage |
|
|
|
|
|
**Important**: the text prompt *must* include a *task instruction prefix*, instructing the model which task is being performed. |
|
|
|
|
|
For example, if you are implementing a RAG application, you embed your documents as `search_document: <text here>` and embed your user queries as `search_query: <text here>`. |
|
|
|
|
|
## Task instruction prefixes |
|
|
|
|
|
### `search_document` |
|
|
|
|
|
#### Purpose: embed texts as documents from a dataset |
|
|
|
|
|
This prefix is used for embedding texts as documents, for example as documents for a RAG index. |
|
|
|
|
|
```python |
|
|
from sentence_transformers import SentenceTransformer |
|
|
|
|
|
model = SentenceTransformer("nomic-ai/nomic-embed-text-v1.5", trust_remote_code=True) |
|
|
sentences = ['search_document: TSNE is a dimensionality reduction algorithm created by Laurens van Der Maaten'] |
|
|
embeddings = model.encode(sentences) |
|
|
print(embeddings) |
|
|
``` |
|
|
|
|
|
### `search_query` |
|
|
|
|
|
#### Purpose: embed texts as questions to answer |
|
|
|
|
|
This prefix is used for embedding texts as questions that documents from a dataset could resolve, for example as queries to be answered by a RAG application. |
|
|
|
|
|
```python |
|
|
from sentence_transformers import SentenceTransformer |
|
|
|
|
|
model = SentenceTransformer("nomic-ai/nomic-embed-text-v1.5", trust_remote_code=True) |
|
|
sentences = ['search_query: Who is Laurens van Der Maaten?'] |
|
|
embeddings = model.encode(sentences) |
|
|
print(embeddings) |
|
|
``` |
|
|
|
|
|
### `clustering` |
|
|
|
|
|
#### Purpose: embed texts to group them into clusters |
|
|
|
|
|
This prefix is used for embedding texts in order to group them into clusters, discover common topics, or remove semantic duplicates. |
|
|
|
|
|
```python |
|
|
from sentence_transformers import SentenceTransformer |
|
|
|
|
|
model = SentenceTransformer("nomic-ai/nomic-embed-text-v1.5", trust_remote_code=True) |
|
|
sentences = ['clustering: the quick brown fox'] |
|
|
embeddings = model.encode(sentences) |
|
|
print(embeddings) |
|
|
``` |
|
|
|
|
|
### `classification` |
|
|
|
|
|
#### Purpose: embed texts to classify them |
|
|
|
|
|
This prefix is used for embedding texts into vectors that will be used as features for a classification model |
|
|
|
|
|
```python |
|
|
from sentence_transformers import SentenceTransformer |
|
|
|
|
|
model = SentenceTransformer("nomic-ai/nomic-embed-text-v1.5", trust_remote_code=True) |
|
|
sentences = ['classification: the quick brown fox'] |
|
|
embeddings = model.encode(sentences) |
|
|
print(embeddings) |
|
|
``` |
|
|
|
|
|
|
|
|
### Sentence Transformers |
|
|
```python |
|
|
import torch.nn.functional as F |
|
|
from sentence_transformers import SentenceTransformer |
|
|
|
|
|
matryoshka_dim = 512 |
|
|
|
|
|
model = SentenceTransformer("nomic-ai/nomic-embed-text-v1.5", trust_remote_code=True) |
|
|
sentences = ['search_query: What is TSNE?', 'search_query: Who is Laurens van der Maaten?'] |
|
|
embeddings = model.encode(sentences, convert_to_tensor=True) |
|
|
embeddings = F.layer_norm(embeddings, normalized_shape=(embeddings.shape[1],)) |
|
|
embeddings = embeddings[:, :matryoshka_dim] |
|
|
embeddings = F.normalize(embeddings, p=2, dim=1) |
|
|
print(embeddings) |
|
|
``` |
|
|
|
|
|
### Transformers |
|
|
|
|
|
```diff |
|
|
import torch |
|
|
import torch.nn.functional as F |
|
|
from transformers import AutoTokenizer, AutoModel |
|
|
|
|
|
def mean_pooling(model_output, attention_mask): |
|
|
token_embeddings = model_output[0] |
|
|
input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() |
|
|
return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) |
|
|
|
|
|
sentences = ['search_query: What is TSNE?', 'search_query: Who is Laurens van der Maaten?'] |
|
|
|
|
|
tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased') |
|
|
model = AutoModel.from_pretrained('nomic-ai/nomic-embed-text-v1.5', trust_remote_code=True, safe_serialization=True) |
|
|
model.eval() |
|
|
|
|
|
encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt') |
|
|
|
|
|
+ matryoshka_dim = 512 |
|
|
|
|
|
with torch.no_grad(): |
|
|
model_output = model(**encoded_input) |
|
|
|
|
|
embeddings = mean_pooling(model_output, encoded_input['attention_mask']) |
|
|
+ embeddings = F.layer_norm(embeddings, normalized_shape=(embeddings.shape[1],)) |
|
|
+ embeddings = embeddings[:, :matryoshka_dim] |
|
|
embeddings = F.normalize(embeddings, p=2, dim=1) |
|
|
print(embeddings) |
|
|
``` |
|
|
|
|
|
The model natively supports scaling of the sequence length past 2048 tokens. To do so, |
|
|
|
|
|
```diff |
|
|
- tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased') |
|
|
+ tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased', model_max_length=8192) |
|
|
|
|
|
|
|
|
- model = AutoModel.from_pretrained('nomic-ai/nomic-embed-text-v1.5', trust_remote_code=True) |
|
|
+ model = AutoModel.from_pretrained('nomic-ai/nomic-embed-text-v1.5', trust_remote_code=True, rotary_scaling_factor=2) |
|
|
``` |
|
|
|
|
|
### Transformers.js |
|
|
|
|
|
```js |
|
|
import { pipeline, layer_norm } from '@huggingface/transformers'; |
|
|
|
|
|
// Create a feature extraction pipeline |
|
|
const extractor = await pipeline('feature-extraction', 'nomic-ai/nomic-embed-text-v1.5'); |
|
|
|
|
|
// Define sentences |
|
|
const texts = ['search_query: What is TSNE?', 'search_query: Who is Laurens van der Maaten?']; |
|
|
|
|
|
// Compute sentence embeddings |
|
|
let embeddings = await extractor(texts, { pooling: 'mean' }); |
|
|
console.log(embeddings); // Tensor of shape [2, 768] |
|
|
|
|
|
const matryoshka_dim = 512; |
|
|
embeddings = layer_norm(embeddings, [embeddings.dims[1]]) |
|
|
.slice(null, [0, matryoshka_dim]) |
|
|
.normalize(2, -1); |
|
|
console.log(embeddings.tolist()); |
|
|
``` |
|
|
|
|
|
|
|
|
## Nomic API |
|
|
|
|
|
The easiest way to use Nomic Embed is through the Nomic Embedding API. |
|
|
|
|
|
Generating embeddings with the `nomic` Python client is as easy as |
|
|
|
|
|
```python |
|
|
from nomic import embed |
|
|
|
|
|
output = embed.text( |
|
|
texts=['Nomic Embedding API', '#keepAIOpen'], |
|
|
model='nomic-embed-text-v1.5', |
|
|
task_type='search_document', |
|
|
dimensionality=256, |
|
|
) |
|
|
|
|
|
print(output) |
|
|
``` |
|
|
|
|
|
For more information, see the [API reference](https://docs.nomic.ai/reference/endpoints/nomic-embed-text) |
|
|
|
|
|
|
|
|
## Infinity |
|
|
|
|
|
Usage with [Infinity](https://github.com/michaelfeil/infinity). |
|
|
|
|
|
```bash |
|
|
docker run --gpus all -v $PWD/data:/app/.cache -e HF_TOKEN=$HF_TOKEN -p "7997":"7997" \ |
|
|
michaelf34/infinity:0.0.70 \ |
|
|
v2 --model-id nomic-ai/nomic-embed-text-v1.5 --revision "main" --dtype float16 --batch-size 8 --engine torch --port 7997 --no-bettertransformer |
|
|
``` |
|
|
|
|
|
## Adjusting Dimensionality |
|
|
|
|
|
`nomic-embed-text-v1.5` is an improvement upon [Nomic Embed](https://huggingface.co/nomic-ai/nomic-embed-text-v1) that utilizes [Matryoshka Representation Learning](https://arxiv.org/abs/2205.13147) which gives developers the flexibility to trade off the embedding size for a negligible reduction in performance. |
|
|
|
|
|
|
|
|
| Name | SeqLen | Dimension | MTEB | |
|
|
| :-------------------------------:| :----- | :-------- | :------: | |
|
|
| nomic-embed-text-v1 | 8192 | 768 | **62.39** | |
|
|
| nomic-embed-text-v1.5 | 8192 | 768 | 62.28 | |
|
|
| nomic-embed-text-v1.5 | 8192 | 512 | 61.96 | |
|
|
| nomic-embed-text-v1.5 | 8192 | 256 | 61.04 | |
|
|
| nomic-embed-text-v1.5 | 8192 | 128 | 59.34 | |
|
|
| nomic-embed-text-v1.5 | 8192 | 64 | 56.10 | |
|
|
|
|
|
|
|
|
 |
|
|
|
|
|
## Training |
|
|
Click the Nomic Atlas map below to visualize a 5M sample of our contrastive pretraining data! |
|
|
|
|
|
[](https://atlas.nomic.ai/map/nomic-text-embed-v1-5m-sample) |
|
|
|
|
|
We train our embedder using a multi-stage training pipeline. Starting from a long-context [BERT model](https://huggingface.co/nomic-ai/nomic-bert-2048), |
|
|
the first unsupervised contrastive stage trains on a dataset generated from weakly related text pairs, such as question-answer pairs from forums like StackExchange and Quora, title-body pairs from Amazon reviews, and summarizations from news articles. |
|
|
|
|
|
In the second finetuning stage, higher quality labeled datasets such as search queries and answers from web searches are leveraged. Data curation and hard-example mining is crucial in this stage. |
|
|
|
|
|
For more details, see the Nomic Embed [Technical Report](https://static.nomic.ai/reports/2024_Nomic_Embed_Text_Technical_Report.pdf) and corresponding [blog post](https://blog.nomic.ai/posts/nomic-embed-matryoshka). |
|
|
|
|
|
Training data to train the models is released in its entirety. For more details, see the `contrastors` [repository](https://github.com/nomic-ai/contrastors) |
|
|
|
|
|
|
|
|
# Join the Nomic Community |
|
|
|
|
|
- Nomic: [https://nomic.ai](https://nomic.ai) |
|
|
- Discord: [https://discord.gg/myY5YDR8z8](https://discord.gg/myY5YDR8z8) |
|
|
- Twitter: [https://twitter.com/nomic_ai](https://twitter.com/nomic_ai) |
|
|
|
|
|
|
|
|
# Citation |
|
|
|
|
|
If you find the model, dataset, or training code useful, please cite our work |
|
|
|
|
|
```bibtex |
|
|
@misc{nussbaum2024nomic, |
|
|
title={Nomic Embed: Training a Reproducible Long Context Text Embedder}, |
|
|
author={Zach Nussbaum and John X. Morris and Brandon Duderstadt and Andriy Mulyar}, |
|
|
year={2024}, |
|
|
eprint={2402.01613}, |
|
|
archivePrefix={arXiv}, |
|
|
primaryClass={cs.CL} |
|
|
} |
|
|
``` |
