Updated model with better training and evaluation. Test and val data included as pickle files. Older Legacy files were removed to avoid confusion.

.gitattributes

@@ -1,39 +1,5 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
 model.safetensors filter=lfs diff=lfs merge=lfs -text
-.git/lfs/objects/1a/76/1a76a58ef6d75109a94ea9fef24bd95c3e37b89337a9263dfd2e78be67d40fad filter=lfs diff=lfs merge=lfs -text
-.git/lfs/objects/d1/50/d150d5f1dc925180772426d6240cefc69cc62920fad2a06d088e0ce6ec787d61 filter=lfs diff=lfs merge=lfs -text
-.git/lfs/objects/d4/17/d417a0fe87c0361764cffc49b15f5bbe663a758233c22ab6d6a51a5ce008adfd filter=lfs diff=lfs merge=lfs -text
+.git/lfs/objects/bd/7a/bd7a72c763bbb2e770f97994bf96540aa1e424f50d331e9e7ceba8e214f5c49e filter=lfs diff=lfs merge=lfs -text
+test_data.pickle filter=lfs diff=lfs merge=lfs -text
+val_data.pickle filter=lfs diff=lfs merge=lfs -text
+sentencepiece.bpe.model filter=lfs diff=lfs merge=lfs -text

Information-Retrieval_evaluation_eval_results.csv

@@ -0,0 +1,5 @@
+epoch,steps,cos_sim-Accuracy@1,cos_sim-Accuracy@3,cos_sim-Accuracy@5,cos_sim-Accuracy@10,cos_sim-Precision@1,cos_sim-Recall@1,cos_sim-Precision@3,cos_sim-Recall@3,cos_sim-Precision@5,cos_sim-Recall@5,cos_sim-Precision@10,cos_sim-Recall@10,cos_sim-MRR@10,cos_sim-NDCG@10,cos_sim-MAP@100
+0,0,0.6184971098265896,0.8901734104046243,0.930635838150289,0.9710982658959537,0.6184971098265896,0.6184971098265896,0.29672447013487474,0.884393063583815,0.18612716763005774,0.9248554913294798,0.0982658959537572,0.9682080924855492,0.7596063859069642,0.8109979174972792,0.7595047697488971
+0,0,0.6184971098265896,0.8901734104046243,0.930635838150289,0.9710982658959537,0.6184971098265896,0.6184971098265896,0.29672447013487474,0.884393063583815,0.18612716763005774,0.9248554913294798,0.0982658959537572,0.9682080924855492,0.7596063859069642,0.8109979174972792,0.7595047697488971
+0,0,0.3956043956043956,0.5732600732600732,0.6556776556776557,0.7875457875457875,0.3956043956043956,0.3956043956043956,0.19108669108669107,0.5732600732600732,0.1311355311355311,0.6556776556776557,0.07875457875457875,0.7875457875457875,0.5070752660038369,0.5733445499232982,0.5169844475014571
+0,0,0.3956043956043956,0.5732600732600732,0.6556776556776557,0.7875457875457875,0.3956043956043956,0.3956043956043956,0.19108669108669107,0.5732600732600732,0.1311355311355311,0.6556776556776557,0.07875457875457875,0.7875457875457875,0.5070752660038369,0.5733445499232982,0.5169844475014571

Information-Retrieval_evaluation_test_results.csv

@@ -0,0 +1,5 @@
+epoch,steps,cos_sim-Accuracy@1,cos_sim-Accuracy@3,cos_sim-Accuracy@5,cos_sim-Accuracy@10,cos_sim-Precision@1,cos_sim-Recall@1,cos_sim-Precision@3,cos_sim-Recall@3,cos_sim-Precision@5,cos_sim-Recall@5,cos_sim-Precision@10,cos_sim-Recall@10,cos_sim-MRR@10,cos_sim-NDCG@10,cos_sim-MAP@100
+0,0,0.8766233766233766,0.9805194805194806,0.987012987012987,1.0,0.8766233766233766,0.8625541125541126,0.3354978354978355,0.9783549783549783,0.20259740259740258,0.9848484848484848,0.10324675324675324,1.0,0.9299242424242423,0.947124363967297,0.9286873840445271
+0,0,0.8896103896103896,1.0,1.0,1.0,0.8896103896103896,0.8755411255411256,0.341991341991342,0.9978354978354977,0.20519480519480518,0.9978354978354977,0.10324675324675324,1.0,0.9415584415584416,0.9562697249477613,0.9404761904761906
+0,0,0.7739463601532567,0.9233716475095786,0.9655172413793104,0.9808429118773946,0.7739463601532567,0.7739463601532567,0.30779054916985943,0.9233716475095786,0.19310344827586207,0.9655172413793104,0.09808429118773947,0.9808429118773946,0.85460834397616,0.8859652217228802,0.8558258273397413
+0,0,0.7739463601532567,0.9272030651340997,0.9578544061302682,0.9808429118773946,0.7739463601532567,0.7739463601532567,0.30906768837803317,0.9272030651340997,0.19157088122605365,0.9578544061302682,0.09808429118773947,0.9808429118773946,0.8563370431186524,0.8873142219416177,0.8576794640013031

README.md

@@ -10,7 +10,7 @@ tags:
 
 ---
 
-# dell-research-harvard/lt-un-data-fine-coarse-fr
+# {MODEL_NAME}
 
 This is a [LinkTransformer](https://linktransformer.github.io/) model. At its core this model this is a sentence transformer model [sentence-transformers](https://www.SBERT.net) model- it just wraps around the class. 
 It is designed for quick and easy record linkage (entity-matching) through the LinkTransformer package. The tasks include clustering, deduplication, linking, aggregation and more.

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "models/linkage_un_data_fr_fine_coarse",
+  "_name_or_path": "dangvantuan/sentence-camembert-large",
   "architectures": [
     "CamembertModel"
   ],

special_tokens_map.json

@@ -3,27 +3,9 @@
     "<s>NOTUSED",
     "</s>NOTUSED"
   ],
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "cls_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
   "mask_token": {
     "content": "<mask>",
     "lstrip": true,
@@ -31,25 +13,7 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": {
-    "content": "<pad>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "sep_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
 }

test_data.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43f227e16bf6c140dacf290f7318dca890a0dd320b11ba3149802313a47f23db
+size 32258

tokenizer_config.json

@@ -66,17 +66,10 @@
   "cls_token": "<s>",
   "eos_token": "</s>",
   "mask_token": "<mask>",
-  "max_length": 514,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_to_multiple_of": null,
   "pad_token": "<pad>",
-  "pad_token_type_id": 0,
-  "padding_side": "right",
   "sep_token": "</s>",
   "sp_model_kwargs": {},
-  "stride": 0,
   "tokenizer_class": "CamembertTokenizer",
-  "truncation_side": "right",
-  "truncation_strategy": "longest_first",
   "unk_token": "<unk>"
 }

val_data.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19c473d4402745303e49184b01708b83a2d0f6cd54e36eabefbcf150ee4731ba
+size 187515