Updated model with better training and evaluation. Test and val data included as pickle files. Older Legacy files were removed to avoid confusion.

.gitattributes

@@ -1,43 +1,5 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
-pytorch_model.bin filter=lfs diff=lfs merge=lfs -text
-.git/lfs/objects/50/3c/503cf29c02ef5e01d945058baabbeba308c59511cca769786fda5a6ee595a66c filter=lfs diff=lfs merge=lfs -text
 model.safetensors filter=lfs diff=lfs merge=lfs -text
 .git/lfs/objects/bd/17/bd176b01380930c75109b551bb5b5e3635b0a881683fc9be2cd1f0b1fb7da29c filter=lfs diff=lfs merge=lfs -text
-.git/lfs/objects/dd/ff/ddffe9c1334ca79b7140d04cef33760198594b6bb4ce06ea0dc12882f8cd2848 filter=lfs diff=lfs merge=lfs -text
-.git/lfs/objects/6e/be/6ebeec4dae5c7db97dea58a3964c3a7b86e8795effe472fa312fd1dc7245c6e2 filter=lfs diff=lfs merge=lfs -text
-.git/lfs/objects/52/8b/528be60f02c85078efa8d1b6ec884cb827d1dd33dfff57d1c1ff5b4daf42b2db filter=lfs diff=lfs merge=lfs -text
-.git/lfs/objects/26/89/2689c2c4bb9f0ae4eed9047269fc0c28ce91c50f3e91e29c8932c4a8b765af72 filter=lfs diff=lfs merge=lfs -text
+.git/lfs/objects/2a/12/2a1218c92154f764953c79f05c215b2b64682364ba9882d3bb51edabf3a1b827 filter=lfs diff=lfs merge=lfs -text
+test_data.pickle filter=lfs diff=lfs merge=lfs -text
+val_data.pickle filter=lfs diff=lfs merge=lfs -text

Information-Retrieval_evaluation_eval_results.csv

@@ -0,0 +1,10 @@
+epoch,steps,cos_sim-Accuracy@1,cos_sim-Accuracy@3,cos_sim-Accuracy@5,cos_sim-Accuracy@10,cos_sim-Precision@1,cos_sim-Recall@1,cos_sim-Precision@3,cos_sim-Recall@3,cos_sim-Precision@5,cos_sim-Recall@5,cos_sim-Precision@10,cos_sim-Recall@10,cos_sim-MRR@10,cos_sim-NDCG@10,cos_sim-MAP@100
+0,0,0.5724381625441696,0.8303886925795053,0.872791519434629,0.911660777385159,0.5724381625441696,0.5724381625441696,0.27679623085983507,0.8303886925795053,0.17455830388692575,0.872791519434629,0.09116607773851589,0.911660777385159,0.7069998317348142,0.757919293262377,0.7114810136344818
+0,0,0.5724381625441696,0.8303886925795053,0.872791519434629,0.911660777385159,0.5724381625441696,0.5724381625441696,0.27679623085983507,0.8303886925795053,0.17455830388692575,0.872791519434629,0.09116607773851589,0.911660777385159,0.7069998317348142,0.757919293262377,0.7114810136344818
+0,0,0.45889921372408865,0.629735525375268,0.6912080057183703,0.7919942816297355,0.45889921372408865,0.45889921372408865,0.20991184179175598,0.629735525375268,0.13824160114367404,0.6912080057183703,0.07919942816297354,0.7919942816297355,0.561284137195503,0.6163014164506035,0.5699982588123317
+0,0,0.45889921372408865,0.629735525375268,0.6912080057183703,0.7919942816297355,0.45889921372408865,0.45889921372408865,0.20991184179175598,0.629735525375268,0.13824160114367404,0.6912080057183703,0.07919942816297354,0.7919942816297355,0.561284137195503,0.6163014164506035,0.5699982588123317
+0,0,0.45889921372408865,0.629735525375268,0.6912080057183703,0.7919942816297355,0.45889921372408865,0.45889921372408865,0.20991184179175598,0.629735525375268,0.13824160114367404,0.6912080057183703,0.07919942816297354,0.7919942816297355,0.561284137195503,0.6163014164506035,0.5699982588123317
+0,0,0.45889921372408865,0.629735525375268,0.6912080057183703,0.7919942816297355,0.45889921372408865,0.45889921372408865,0.20991184179175598,0.629735525375268,0.13824160114367404,0.6912080057183703,0.07919942816297354,0.7919942816297355,0.561284137195503,0.6163014164506035,0.5699982588123317
+0,0,0.4753395282344532,0.6540385989992852,0.7326661901358112,0.8070050035739814,0.4753395282344532,0.4753395282344532,0.21801286633309508,0.6540385989992852,0.14653323802716223,0.7326661901358112,0.08070050035739812,0.8070050035739814,0.5827291602845585,0.6368435630051191,0.5905098783240487
+0,0,0.4753395282344532,0.6540385989992852,0.7326661901358112,0.8070050035739814,0.4753395282344532,0.4753395282344532,0.21801286633309508,0.6540385989992852,0.14653323802716223,0.7326661901358112,0.08070050035739812,0.8070050035739814,0.5827291602845585,0.6368435630051191,0.5905098783240487
+0,0,0.45889921372408865,0.629735525375268,0.6912080057183703,0.7919942816297355,0.45889921372408865,0.45889921372408865,0.20991184179175598,0.629735525375268,0.13824160114367404,0.6912080057183703,0.07919942816297354,0.7919942816297355,0.561284137195503,0.6163014164506035,0.5699982588123317

Information-Retrieval_evaluation_test_results.csv

@@ -0,0 +1,6 @@
+epoch,steps,cos_sim-Accuracy@1,cos_sim-Accuracy@3,cos_sim-Accuracy@5,cos_sim-Accuracy@10,cos_sim-Precision@1,cos_sim-Recall@1,cos_sim-Precision@3,cos_sim-Recall@3,cos_sim-Precision@5,cos_sim-Recall@5,cos_sim-Precision@10,cos_sim-Recall@10,cos_sim-MRR@10,cos_sim-NDCG@10,cos_sim-MAP@100
+0,0,0.9573333333333334,0.9866666666666667,0.9973333333333333,1.0,0.9573333333333334,0.9573333333333334,0.3288888888888889,0.9866666666666667,0.19946666666666663,0.9973333333333333,0.09999999999999998,1.0,0.9745555555555556,0.9809265830527836,0.9745555555555555
+0,0,0.9546666666666667,0.9893333333333333,0.9973333333333333,1.0,0.9546666666666667,0.9546666666666667,0.32977777777777784,0.9893333333333333,0.19946666666666663,0.9973333333333333,0.09999999999999998,1.0,0.9734444444444444,0.9801272582407785,0.9734444444444443
+0,0,0.8289902280130294,0.9543973941368078,0.9788273615635179,0.990228013029316,0.8289902280130294,0.8289902280130294,0.3181324647122693,0.9543973941368078,0.19576547231270355,0.9788273615635179,0.09902280130293159,0.990228013029316,0.8929153094462541,0.9172870124572965,0.8936588316677894
+0,0,0.8289902280130294,0.9560260586319218,0.9771986970684039,0.99185667752443,0.8289902280130294,0.8289902280130294,0.3186753528773073,0.9560260586319218,0.19543973941368073,0.9771986970684039,0.09918566775244299,0.99185667752443,0.8928293521534562,0.9174937140784137,0.8935326390945282
+0,0,0.8485342019543974,0.9560260586319218,0.9788273615635179,0.99185667752443,0.8485342019543974,0.8485342019543974,0.3186753528773073,0.9560260586319218,0.19576547231270355,0.9788273615635179,0.09918566775244299,0.99185667752443,0.9050533840028954,0.9266706066970336,0.9057234522625403

README.md

@@ -10,7 +10,7 @@ tags:
 
 ---
 
-# dell-research-harvard/lt-un-data-fine-coarse-en
+# {MODEL_NAME}
 
 This is a [LinkTransformer](https://linktransformer.github.io/) model. At its core this model this is a sentence transformer model [sentence-transformers](https://www.SBERT.net) model- it just wraps around the class. 
 It is designed for quick and easy record linkage (entity-matching) through the LinkTransformer package. The tasks include clustering, deduplication, linking, aggregation and more.

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "models/linkage_un_data_en_fine_coarse",
+  "_name_or_path": "sentence-transformers/multi-qa-mpnet-base-dot-v1",
   "architectures": [
     "MPNetModel"
   ],

special_tokens_map.json

@@ -9,7 +9,7 @@
   "cls_token": {
     "content": "<s>",
     "lstrip": false,
-    "normalized": false,
+    "normalized": true,
     "rstrip": false,
     "single_word": false
   },
@@ -37,7 +37,7 @@
   "sep_token": {
     "content": "</s>",
     "lstrip": false,
-    "normalized": false,
+    "normalized": true,
     "rstrip": false,
     "single_word": false
   },

pytorch_model.bin → test_data.pickle

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bd176b01380930c75109b551bb5b5e3635b0a881683fc9be2cd1f0b1fb7da29c
-size 438009257
+oid sha256:01ca36406ff9a66f7eb69641c5ff9bc16adb0f521bdd43510fc8971821846980
+size 71802

val_data.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:201eeb8bf97f7d77ce923b8cfccd5bad6023d15532393c3fc315de2a764f2766
+size 443286