Better model with bs=256

Browse files

Files changed (8) hide show

README.md +2 -2
added_tokens.json +1 -1
config.json +1 -1
config_sentence_transformers.json +2 -2
eval/beir.json +276 -0
pytorch_model.bin +2 -2
tokenizer.json +0 -0
tokenizer_config.json +1 -1

README.md CHANGED Viewed

@@ -14,7 +14,7 @@ For usage instructions, refer to our codebase: https://github.com/Muennighoff/sg
 ## Evaluation Results
-For eval results, refer to our paper: https://arxiv.org/abs/2202.08904
 ## Training
@@ -70,4 +70,4 @@ SentenceTransformer(
   journal={arXiv preprint arXiv:2202.08904},
   year={2022}
 }
-```

 ## Evaluation Results
+For eval results, refer to the eval folder or our paper: https://arxiv.org/abs/2202.08904
 ## Training
   journal={arXiv preprint arXiv:2202.08904},
   year={2022}
 }
+```

added_tokens.json CHANGED Viewed

	@@ -1 +1 @@
1	- {"{SOS}": ~~50258~~, "[SOS]": ~~50257~~}


1	+ {"[SOS]": 50257, "{SOS}": 50258}

config.json CHANGED Viewed

@@ -47,7 +47,7 @@
   "summary_type": "cls_index",
   "summary_use_proj": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.11.3",
   "use_cache": true,
   "vocab_size": 50259,
   "window_size": 256

   "summary_type": "cls_index",
   "summary_use_proj": true,
   "torch_dtype": "float32",
+  "transformers_version": "4.20.0.dev0",
   "use_cache": true,
   "vocab_size": 50259,
   "window_size": 256

config_sentence_transformers.json CHANGED Viewed

@@ -1,7 +1,7 @@
 {
   "__version__": {
     "sentence_transformers": "2.1.0",
-    "transformers": "4.11.3",
-    "pytorch": "1.10.1"
   }
 }

 {
   "__version__": {
     "sentence_transformers": "2.1.0",
+    "transformers": "4.20.0.dev0",
+    "pytorch": "1.10.2"
   }
 }

eval/beir.json ADDED Viewed

	@@ -0,0 +1,276 @@

+{
+    "SGPT-125M-weightedmean-msmarco-specb-bitfit": {
+        "bioasq": {
+            "NDCG@1": 0.316,
+            "NDCG@3": 0.28125,
+            "NDCG@5": 0.27137,
+            "NDCG@10": 0.27215,
+            "NDCG@100": 0.3218,
+            "NDCG@1000": 0.35137
+        },
+        "nq": {
+            "NDCG@1": 0.17439,
+            "NDCG@3": 0.23938,
+            "NDCG@5": 0.26684,
+            "NDCG@10": 0.29726,
+            "NDCG@100": 0.34802,
+            "NDCG@1000": 0.36888
+        },
+        "dbpedia-entity": {
+            "NDCG@1": 0.31875,
+            "NDCG@3": 0.25912,
+            "NDCG@5": 0.24217,
+            "NDCG@10": 0.22673,
+            "NDCG@100": 0.24325,
+            "NDCG@1000": 0.29203
+        },
+        "hotpotqa": {
+            "NDCG@1": 0.51101,
+            "NDCG@3": 0.37672,
+            "NDCG@5": 0.39295,
+            "NDCG@10": 0.4089,
+            "NDCG@100": 0.43765,
+            "NDCG@1000": 0.45428
+        },
+        "fever": {
+            "NDCG@1": 0.4703,
+            "NDCG@3": 0.55269,
+            "NDCG@5": 0.58116,
+            "NDCG@10": 0.60497,
+            "NDCG@100": 0.63091,
+            "NDCG@1000": 0.63755
+        },
+        "cqadupstack_android": {
+            "NDCG@1": 0.22604,
+            "NDCG@3": 0.2575,
+            "NDCG@5": 0.2753,
+            "NDCG@10": 0.29861,
+            "NDCG@100": 0.34719,
+            "NDCG@1000": 0.38073
+        },
+        "cqadupstack_english": {
+            "NDCG@1": 0.23885,
+            "NDCG@3": 0.27032,
+            "NDCG@5": 0.28385,
+            "NDCG@10": 0.30398,
+            "NDCG@100": 0.34259,
+            "NDCG@1000": 0.36855
+        },
+        "cqadupstack_gaming": {
+            "NDCG@1": 0.25956,
+            "NDCG@3": 0.31655,
+            "NDCG@5": 0.3402,
+            "NDCG@10": 0.35956,
+            "NDCG@100": 0.40891,
+            "NDCG@1000": 0.43256
+        },
+        "cqadupstack_gis": {
+            "NDCG@1": 0.16949,
+            "NDCG@3": 0.1979,
+            "NDCG@5": 0.21682,
+            "NDCG@10": 0.23385,
+            "NDCG@100": 0.27691,
+            "NDCG@1000": 0.30686
+        },
+        "cqadupstack_mathematica": {
+            "NDCG@1": 0.10821,
+            "NDCG@3": 0.14519,
+            "NDCG@5": 0.16025,
+            "NDCG@10": 0.18252,
+            "NDCG@100": 0.23551,
+            "NDCG@1000": 0.27061
+        },
+        "cqadupstack_physics": {
+            "NDCG@1": 0.20789,
+            "NDCG@3": 0.24259,
+            "NDCG@5": 0.26274,
+            "NDCG@10": 0.28917,
+            "NDCG@100": 0.34858,
+            "NDCG@1000": 0.37641
+        },
+        "cqadupstack_programmers": {
+            "NDCG@1": 0.18721,
+            "NDCG@3": 0.22957,
+            "NDCG@5": 0.24378,
+            "NDCG@10": 0.26469,
+            "NDCG@100": 0.31762,
+            "NDCG@1000": 0.3495
+        },
+        "cqadupstack_stats": {
+            "NDCG@1": 0.16718,
+            "NDCG@3": 0.19468,
+            "NDCG@5": 0.20881,
+            "NDCG@10": 0.22403,
+            "NDCG@100": 0.26555,
+            "NDCG@1000": 0.29332
+        },
+        "cqadupstack_wordpress": {
+            "NDCG@1": 0.12754,
+            "NDCG@3": 0.16302,
+            "NDCG@5": 0.18037,
+            "NDCG@10": 0.19715,
+            "NDCG@100": 0.23863,
+            "NDCG@1000": 0.27293
+        },
+        "cqadupstack_webmasters": {
+            "NDCG@1": 0.18972,
+            "NDCG@3": 0.2142,
+            "NDCG@5": 0.23037,
+            "NDCG@10": 0.24997,
+            "NDCG@100": 0.30509,
+            "NDCG@1000": 0.34226
+        },
+        "cqadupstack_unix": {
+            "NDCG@1": 0.16325,
+            "NDCG@3": 0.19538,
+            "NDCG@5": 0.20865,
+            "NDCG@10": 0.22792,
+            "NDCG@100": 0.2762,
+            "NDCG@1000": 0.30613
+        },
+        "cqadupstack_tex": {
+            "NDCG@1": 0.10392,
+            "NDCG@3": 0.12618,
+            "NDCG@5": 0.13715,
+            "NDCG@10": 0.15087,
+            "NDCG@100": 0.19361,
+            "NDCG@1000": 0.22519
+        },
+        "cqadupstack": {
+            "NDCG@1": 0.17907166666666666,
+            "NDCG@3": 0.21275666666666665,
+            "NDCG@5": 0.22902416666666667,
+            "NDCG@10": 0.24852666666666667,
+            "NDCG@100": 0.2963658333333334,
+            "NDCG@1000": 0.3270875
+        },
+        "signal1m": {
+            "NDCG@1": 0.34021,
+            "NDCG@3": 0.30594,
+            "NDCG@5": 0.2707,
+            "NDCG@10": 0.23598,
+            "NDCG@100": 0.22928,
+            "NDCG@1000": 0.28209
+        },
+        "trec-news": {
+            "NDCG@1": 0.37135,
+            "NDCG@3": 0.36728,
+            "NDCG@5": 0.3443,
+            "NDCG@10": 0.31943,
+            "NDCG@100": 0.32159,
+            "NDCG@1000": 0.41124
+        },
+        "arguana": {
+            "NDCG@1": 0.22404,
+            "NDCG@3": 0.3505,
+            "NDCG@5": 0.401,
+            "NDCG@10": 0.45468,
+            "NDCG@100": 0.50377,
+            "NDCG@1000": 0.50896
+        },
+        "webis-touche2020": {
+            "NDCG@1": 0.31633,
+            "NDCG@3": 0.26398,
+            "NDCG@5": 0.24418,
+            "NDCG@10": 0.22969,
+            "NDCG@100": 0.34778,
+            "NDCG@1000": 0.45638
+        },
+        "quora": {
+            "NDCG@1": 0.6251,
+            "NDCG@3": 0.68147,
+            "NDCG@5": 0.70473,
+            "NDCG@10": 0.72977,
+            "NDCG@100": 0.76027,
+            "NDCG@1000": 0.76517
+        },
+        "scifact": {
+            "NDCG@1": 0.45333,
+            "NDCG@3": 0.52618,
+            "NDCG@5": 0.54557,
+            "NDCG@10": 0.56923,
+            "NDCG@100": 0.60721,
+            "NDCG@1000": 0.62065
+        },
+        "scidocs": {
+            "NDCG@1": 0.145,
+            "NDCG@3": 0.12064,
+            "NDCG@5": 0.1016,
+            "NDCG@10": 0.12118,
+            "NDCG@100": 0.17639,
+            "NDCG@1000": 0.22638
+        },
+        "nfcorpus": {
+            "NDCG@1": 0.31579,
+            "NDCG@3": 0.2681,
+            "NDCG@5": 0.25146,
+            "NDCG@10": 0.22776,
+            "NDCG@100": 0.21672,
+            "NDCG@1000": 0.30891
+        },
+        "robust04": {
+            "NDCG@1": 0.41365,
+            "NDCG@3": 0.38515,
+            "NDCG@5": 0.35751,
+            "NDCG@10": 0.31325,
+            "NDCG@100": 0.25065,
+            "NDCG@1000": 0.32678
+        },
+        "climate-fever": {
+            "NDCG@1": 0.19023,
+            "NDCG@3": 0.17124,
+            "NDCG@5": 0.18861,
+            "NDCG@10": 0.21839,
+            "NDCG@100": 0.28314,
+            "NDCG@1000": 0.3155
+        },
+        "trec-covid": {
+            "NDCG@1": 0.87,
+            "NDCG@3": 0.82704,
+            "NDCG@5": 0.79357,
+            "NDCG@10": 0.73818,
+            "NDCG@100": 0.48765,
+            "NDCG@1000": 0.39645
+        },
+        "fiqa": {
+            "NDCG@1": 0.19753,
+            "NDCG@3": 0.18319,
+            "NDCG@5": 0.19438,
+            "NDCG@10": 0.21115,
+            "NDCG@100": 0.2733,
+            "NDCG@1000": 0.31315
+        },
+        "average": {
+            "NDCG@1": 0.3573378703703704,
+            "NDCG@3": 0.35403481481481475,
+            "NDCG@5": 0.3545068981481481,
+            "NDCG@10": 0.35706814814814813,
+            "NDCG@100": 0.3742081018518519,
+            "NDCG@1000": 0.4090476388888889
+        },
+        "subaverage": {
+            "NDCG@1": 0.40840090909090904,
+            "NDCG@3": 0.40547545454545453,
+            "NDCG@5": 0.4127072727272727,
+            "NDCG@10": 0.41995,
+            "NDCG@100": 0.43560454545454547,
+            "NDCG@1000": 0.46082090909090917
+        },
+        "subsubaverage": {
+            "NDCG@1": 0.267138,
+            "NDCG@3": 0.289722,
+            "NDCG@5": 0.298802,
+            "NDCG@10": 0.31679999999999997,
+            "NDCG@100": 0.35547799999999996,
+            "NDCG@1000": 0.39561
+        },
+        "msmarco": {
+            "NDCG@1": 0.14785,
+            "NDCG@3": 0.2189,
+            "NDCG@5": 0.24875,
+            "NDCG@10": 0.27945,
+            "NDCG@100": 0.34107,
+            "NDCG@1000": 0.36339
+        }
+    }
+}

pytorch_model.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff8eb75758c05dbb0d40d5e4fea9ec8b707b5bb50d3e439d05273bb0b6b0c16b
-size 551196689

 version https://git-lfs.github.com/spec/v1
+oid sha256:f3f11c87e27372670924ad3b066f1159073753895cba56a30df56af485580521
+size 551189073

tokenizer.json CHANGED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json CHANGED Viewed

@@ -1 +1 @@

- {"unk_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "add_prefix_space": false, "errors": "replace", "model_max_length": 2048, "special_tokens_map_file": null, "name_or_path": "EleutherAI/gpt-neo-125M", "tokenizer_class": "GPT2Tokenizer"}

+ {"unk_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "add_prefix_space": false, "errors": "replace", "model_max_length": 2048, "special_tokens_map_file": null, "name_or_path": "EleutherAI/gpt-neo-125M", "pad_token": null, "add_bos_token": false, "tokenizer_class": "GPT2Tokenizer"}