model update

README.md

@@ -14,7 +14,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.8303968253968254
+      value: 0.780436507936508
   - task:
       name: Analogy Questions (SAT full)
       type: multiple-choice-qa
@@ -25,7 +25,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.7192513368983957
+      value: 0.5588235294117647
   - task:
       name: Analogy Questions (SAT)
       type: multiple-choice-qa
@@ -36,7 +36,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.7091988130563798
+      value: 0.5608308605341247
   - task:
       name: Analogy Questions (BATS)
       type: multiple-choice-qa
@@ -47,7 +47,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.8043357420789328
+      value: 0.6731517509727627
   - task:
       name: Analogy Questions (Google)
       type: multiple-choice-qa
@@ -58,7 +58,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.948
+      value: 0.856
   - task:
       name: Analogy Questions (U2)
       type: multiple-choice-qa
@@ -69,7 +69,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6798245614035088
+      value: 0.5570175438596491
   - task:
       name: Analogy Questions (U4)
       type: multiple-choice-qa
@@ -80,7 +80,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6643518518518519
+      value: 0.5439814814814815
   - task:
       name: Analogy Questions (ConceptNet Analogy)
       type: multiple-choice-qa
@@ -91,7 +91,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.4865771812080537
+      value: 0.30453020134228187
   - task:
       name: Analogy Questions (TREX Analogy)
       type: multiple-choice-qa
@@ -102,7 +102,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6338797814207651
+      value: 0.4644808743169399
   - task:
       name: Analogy Questions (NELL-ONE Analogy)
       type: multiple-choice-qa
@@ -113,7 +113,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6633333333333333
+      value: 0.635
   - task:
       name: Lexical Relation Classification (BLESS)
       type: classification
@@ -124,10 +124,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.9169805635076088
+      value: 0.9067349706192557
     - name: F1 (macro)
       type: f1_macro
-      value: 0.9133613159985977
+      value: 0.901083105005463
   - task:
       name: Lexical Relation Classification (CogALexV)
       type: classification
@@ -138,10 +138,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.8643192488262911
+      value: 0.8112676056338028
     - name: F1 (macro)
       type: f1_macro
-      value: 0.709680204738525
+      value: 0.6148092103324919
   - task:
       name: Lexical Relation Classification (EVALution)
       type: classification
@@ -152,10 +152,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.6782231852654388
+      value: 0.6305525460455038
     - name: F1 (macro)
       type: f1_macro
-      value: 0.665196173208286
+      value: 0.6268772505825797
   - task:
       name: Lexical Relation Classification (K&H+N)
       type: classification
@@ -166,10 +166,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.9568060095986646
+      value: 0.9433122348195033
     - name: F1 (macro)
       type: f1_macro
-      value: 0.8745909398702613
+      value: 0.8521358889073605
   - task:
       name: Lexical Relation Classification (ROOT09)
       type: classification
@@ -180,34 +180,34 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.9150736446255092
+      value: 0.895017235976183
     - name: F1 (macro)
       type: f1_macro
-      value: 0.9142555280970402
+      value: 0.8915314929167794
 
 ---
 # relbert/relbert-roberta-base-nce-semeval2012-2
 
-RelBERT based on [roberta-large](https://huggingface.co/roberta-large) fine-tuned on [relbert/semeval2012_relational_similarity](https://huggingface.co/datasets/relbert/semeval2012_relational_similarity) (see the [`relbert`](https://github.com/asahi417/relbert) for more detail of fine-tuning).
+RelBERT based on [roberta-base](https://huggingface.co/roberta-base) fine-tuned on [relbert/semeval2012_relational_similarity](https://huggingface.co/datasets/relbert/semeval2012_relational_similarity) (see the [`relbert`](https://github.com/asahi417/relbert) for more detail of fine-tuning).
 This model achieves the following results on the relation understanding tasks:
 - Analogy Question ([dataset](https://huggingface.co/datasets/relbert/analogy_questions), [full result](https://huggingface.co/relbert/relbert-roberta-base-nce-semeval2012-2/raw/main/analogy.forward.json)):
-    - Accuracy on SAT (full): 0.7192513368983957 
-    - Accuracy on SAT: 0.7091988130563798
-    - Accuracy on BATS: 0.8043357420789328
-    - Accuracy on U2: 0.6798245614035088
-    - Accuracy on U4: 0.6643518518518519
-    - Accuracy on Google: 0.948
-    - Accuracy on ConceptNet Analogy: 0.4865771812080537
-    - Accuracy on T-Rex Analogy: 0.6338797814207651
-    - Accuracy on NELL-ONE Analogy: 0.6633333333333333
+    - Accuracy on SAT (full): 0.5588235294117647 
+    - Accuracy on SAT: 0.5608308605341247
+    - Accuracy on BATS: 0.6731517509727627
+    - Accuracy on U2: 0.5570175438596491
+    - Accuracy on U4: 0.5439814814814815
+    - Accuracy on Google: 0.856
+    - Accuracy on ConceptNet Analogy: 0.30453020134228187
+    - Accuracy on T-Rex Analogy: 0.4644808743169399
+    - Accuracy on NELL-ONE Analogy: 0.635
 - Lexical Relation Classification ([dataset](https://huggingface.co/datasets/relbert/lexical_relation_classification), [full result](https://huggingface.co/relbert/relbert-roberta-base-nce-semeval2012-2/raw/main/classification.json)):
-    - Micro F1 score on BLESS: 0.9169805635076088
-    - Micro F1 score on CogALexV: 0.8643192488262911
-    - Micro F1 score on EVALution: 0.6782231852654388
-    - Micro F1 score on K&H+N: 0.9568060095986646
-    - Micro F1 score on ROOT09: 0.9150736446255092
+    - Micro F1 score on BLESS: 0.9067349706192557
+    - Micro F1 score on CogALexV: 0.8112676056338028
+    - Micro F1 score on EVALution: 0.6305525460455038
+    - Micro F1 score on K&H+N: 0.9433122348195033
+    - Micro F1 score on ROOT09: 0.895017235976183
 - Relation Mapping ([dataset](https://huggingface.co/datasets/relbert/relation_mapping), [full result](https://huggingface.co/relbert/relbert-roberta-base-nce-semeval2012-2/raw/main/relation_mapping.json)):
-    - Accuracy on Relation Mapping: 0.8303968253968254 
+    - Accuracy on Relation Mapping: 0.780436507936508 
 
 
 ### Usage
@@ -224,7 +224,7 @@ vector = model.get_embedding(['Tokyo', 'Japan'])  # shape of (n_dim, )
 
 ### Training hyperparameters
 
- - model: roberta-large
+ - model: roberta-base
  - max_length: 64
  - epoch: 10
  - batch: 32
@@ -239,7 +239,7 @@ vector = model.get_embedding(['Tokyo', 'Japan'])  # shape of (n_dim, )
  - split_valid: validation
  - loss_function: nce
  - classification_loss: False
- - loss_function_config: {'temperature': 0.05, 'num_negative': 100, 'num_positive': 10}
+ - loss_function_config: {'temperature': 0.05, 'num_negative': 400, 'num_positive': 10}
  - augment_negative_by_positive: True
 
 See the full configuration at [config file](https://huggingface.co/relbert/relbert-roberta-base-nce-semeval2012-2/raw/main/finetuning_config.json).

analogy.forward.json

@@ -1 +1 @@
-{"semeval2012_relational_similarity/validation": 0.7974683544303798, "scan/test": 0.2908415841584158, "sat_full/test": 0.7192513368983957, "sat/test": 0.7091988130563798, "u2/test": 0.6798245614035088, "u4/test": 0.6643518518518519, "google/test": 0.948, "bats/test": 0.8043357420789328, "t_rex_relational_similarity/test": 0.6338797814207651, "conceptnet_relational_similarity/test": 0.4865771812080537, "nell_relational_similarity/test": 0.6633333333333333}
+{"semeval2012_relational_similarity/validation": 0.6962025316455697, "scan/test": 0.2332920792079208, "sat_full/test": 0.5588235294117647, "sat/test": 0.5608308605341247, "u2/test": 0.5570175438596491, "u4/test": 0.5439814814814815, "google/test": 0.856, "bats/test": 0.6731517509727627, "t_rex_relational_similarity/test": 0.4644808743169399, "conceptnet_relational_similarity/test": 0.30453020134228187, "nell_relational_similarity/test": 0.635}

classification.json

@@ -1 +1 @@
-{"lexical_relation_classification/BLESS": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9169805635076088, "test/f1_macro": 0.9133613159985977, "test/f1_micro": 0.9169805635076088, "test/p_macro": 0.9065498330870753, "test/p_micro": 0.9169805635076088, "test/r_macro": 0.9208392163252331, "test/r_micro": 0.9169805635076088, "test/f1/attri": 0.9212207239176722, "test/p/attri": 0.9102384291725105, "test/r/attri": 0.9324712643678161, "test/f1/coord": 0.9544175576814856, "test/p/coord": 0.9474860335195531, "test/r/coord": 0.9614512471655329, "test/f1/event": 0.8538071065989848, "test/p/event": 0.8285714285714286, "test/r/event": 0.8806282722513089, "test/f1/hyper": 0.9296987087517934, "test/p/hyper": 0.9337175792507204, "test/r/hyper": 0.9257142857142857, "test/f1/mero": 0.8896146309601568, "test/p/mero": 0.867515923566879, "test/r/mero": 0.9128686327077749, "test/f1/random": 0.931409168081494, "test/p/random": 0.9517696044413602, "test/r/random": 0.9119015957446809}, "lexical_relation_classification/CogALexV": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8643192488262911, "test/f1_macro": 0.709680204738525, "test/f1_micro": 0.8643192488262911, "test/p_macro": 0.7459301562952307, "test/p_micro": 0.8643192488262911, "test/r_macro": 0.6794130847403439, "test/r_micro": 0.8643192488262911, "test/f1/ANT": 0.7822222222222223, "test/p/ANT": 0.8380952380952381, "test/r/ANT": 0.7333333333333333, "test/f1/HYPER": 0.6134453781512605, "test/p/HYPER": 0.6596385542168675, "test/r/HYPER": 0.5732984293193717, "test/f1/PART_OF": 0.7286063569682152, "test/p/PART_OF": 0.8054054054054054, "test/r/PART_OF": 0.6651785714285714, "test/f1/RANDOM": 0.937519923493784, "test/p/RANDOM": 0.9147744945567652, "test/r/RANDOM": 0.9614253023864008, "test/f1/SYN": 0.48660714285714285, "test/p/SYN": 0.5117370892018779, "test/r/SYN": 0.46382978723404256}, "lexical_relation_classification/EVALution": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.6782231852654388, "test/f1_macro": 0.665196173208286, "test/f1_micro": 0.6782231852654388, "test/p_macro": 0.6665905293786667, "test/p_micro": 0.6782231852654388, "test/r_macro": 0.6653300709395378, "test/r_micro": 0.6782231852654388, "test/f1/Antonym": 0.7931456548347613, "test/p/Antonym": 0.8059701492537313, "test/r/Antonym": 0.7807228915662651, "test/f1/HasA": 0.6541353383458647, "test/p/HasA": 0.7016129032258065, "test/r/HasA": 0.6126760563380281, "test/f1/HasProperty": 0.8126888217522659, "test/p/HasProperty": 0.7911764705882353, "test/r/HasProperty": 0.8354037267080745, "test/f1/IsA": 0.6145610278372591, "test/p/IsA": 0.6042105263157894, "test/r/IsA": 0.6252723311546841, "test/f1/MadeOf": 0.6363636363636364, "test/p/MadeOf": 0.6222222222222222, "test/r/MadeOf": 0.6511627906976745, "test/f1/PartOf": 0.6622073578595318, "test/p/PartOf": 0.6428571428571429, "test/r/PartOf": 0.6827586206896552, "test/f1/Synonym": 0.483271375464684, "test/p/Synonym": 0.49808429118773945, "test/r/Synonym": 0.4693140794223827}, "lexical_relation_classification/K&H+N": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9568060095986646, "test/f1_macro": 0.8745909398702613, "test/f1_micro": 0.9568060095986646, "test/p_macro": 0.8722419340096175, "test/p_micro": 0.9568060095986646, "test/r_macro": 0.8771975453603155, "test/r_micro": 0.9568060095986646, "test/f1/false": 0.9696655047096344, "test/p/false": 0.9703131957844738, "test/r/false": 0.9690186777349541, "test/f1/hypo": 0.9182389937106918, "test/p/hypo": 0.9258536585365854, "test/r/hypo": 0.9107485604606526, "test/f1/mero": 0.6490872210953347, "test/p/mero": 0.6324110671936759, "test/r/mero": 0.6666666666666666, "test/f1/sibl": 0.9613720399653843, "test/p/sibl": 0.9603898145237346, "test/r/sibl": 0.9623562765789888}, "lexical_relation_classification/ROOT09": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9150736446255092, "test/f1_macro": 0.9142555280970402, "test/f1_micro": 0.9150736446255092, "test/p_macro": 0.9123400631967442, "test/p_micro": 0.9150736446255092, "test/r_macro": 0.916280536276508, "test/r_micro": 0.9150736446255092, "test/f1/COORD": 0.9762050030506406, "test/p/COORD": 0.9720534629404617, "test/r/COORD": 0.9803921568627451, "test/f1/HYPER": 0.8489296636085627, "test/p/HYPER": 0.8401937046004843, "test/r/HYPER": 0.857849196538937, "test/f1/RANDOM": 0.9176319176319176, "test/p/RANDOM": 0.9247730220492867, "test/r/RANDOM": 0.9106002554278416}}
+{"lexical_relation_classification/BLESS": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9067349706192557, "test/f1_macro": 0.901083105005463, "test/f1_micro": 0.9067349706192557, "test/p_macro": 0.8997912343328448, "test/p_micro": 0.9067349706192557, "test/r_macro": 0.9030508070759272, "test/r_micro": 0.9067349706192557, "test/f1/attri": 0.9096005606166784, "test/p/attri": 0.8878248974008208, "test/r/attri": 0.9324712643678161, "test/f1/coord": 0.9407783417935703, "test/p/coord": 0.936026936026936, "test/r/coord": 0.9455782312925171, "test/f1/event": 0.8609341825902335, "test/p/event": 0.8729817007534983, "test/r/event": 0.8492146596858638, "test/f1/hyper": 0.9136163982430454, "test/p/hyper": 0.9369369369369369, "test/r/hyper": 0.8914285714285715, "test/f1/mero": 0.8599348534201954, "test/p/mero": 0.8365019011406845, "test/r/mero": 0.8847184986595175, "test/f1/random": 0.9216342933690557, "test/p/random": 0.9284750337381916, "test/r/random": 0.9148936170212766}, "lexical_relation_classification/CogALexV": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8112676056338028, "test/f1_macro": 0.6148092103324919, "test/f1_micro": 0.8112676056338028, "test/p_macro": 0.6304195397027895, "test/p_micro": 0.8112676056338028, "test/r_macro": 0.6012687546642258, "test/r_micro": 0.8112676056338028, "test/f1/ANT": 0.61731843575419, "test/p/ANT": 0.6207865168539326, "test/r/ANT": 0.6138888888888889, "test/f1/HYPER": 0.5638148667601683, "test/p/HYPER": 0.6072507552870091, "test/r/HYPER": 0.5261780104712042, "test/f1/PART_OF": 0.5849056603773586, "test/p/PART_OF": 0.62, "test/r/PART_OF": 0.5535714285714286, "test/f1/RANDOM": 0.9080070887707427, "test/p/RANDOM": 0.8951715374841169, "test/r/RANDOM": 0.9212160836874795, "test/f1/SYN": 0.39999999999999997, "test/p/SYN": 0.4088888888888889, "test/r/SYN": 0.39148936170212767}, "lexical_relation_classification/EVALution": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.6305525460455038, "test/f1_macro": 0.6268772505825797, "test/f1_micro": 0.6305525460455038, "test/p_macro": 0.637097321721357, "test/p_micro": 0.6305525460455038, "test/r_macro": 0.6212371311593516, "test/r_micro": 0.6305525460455038, "test/f1/Antonym": 0.711779448621554, "test/p/Antonym": 0.741514360313316, "test/r/Antonym": 0.6843373493975904, "test/f1/HasA": 0.6733333333333333, "test/p/HasA": 0.6392405063291139, "test/r/HasA": 0.7112676056338029, "test/f1/HasProperty": 0.796324655436447, "test/p/HasProperty": 0.7854984894259819, "test/r/HasProperty": 0.8074534161490683, "test/f1/IsA": 0.5868392664509169, "test/p/IsA": 0.5811965811965812, "test/r/IsA": 0.5925925925925926, "test/f1/MadeOf": 0.609271523178808, "test/p/MadeOf": 0.7076923076923077, "test/r/MadeOf": 0.5348837209302325, "test/f1/PartOf": 0.6223776223776224, "test/p/PartOf": 0.6312056737588653, "test/r/PartOf": 0.6137931034482759, "test/f1/Synonym": 0.3882149046793761, "test/p/Synonym": 0.37333333333333335, "test/r/Synonym": 0.4043321299638989}, "lexical_relation_classification/K&H+N": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9433122348195033, "test/f1_macro": 0.8521358889073605, "test/f1_micro": 0.9433122348195033, "test/p_macro": 0.8689695129753637, "test/p_micro": 0.9433122348195033, "test/r_macro": 0.8378283080134359, "test/r_micro": 0.9433122348195033, "test/f1/false": 0.955429255160036, "test/p/false": 0.9640808934500453, "test/r/false": 0.9469315149718351, "test/f1/hypo": 0.9023437499999999, "test/p/hypo": 0.9184890656063618, "test/r/hypo": 0.8867562380038387, "test/f1/mero": 0.6018099547511312, "test/p/mero": 0.6584158415841584, "test/r/mero": 0.5541666666666667, "test/f1/sibl": 0.948960595718275, "test/p/sibl": 0.9348922512608895, "test/r/sibl": 0.9634588124114034}, "lexical_relation_classification/ROOT09": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.895017235976183, "test/f1_macro": 0.8915314929167794, "test/f1_micro": 0.895017235976183, "test/p_macro": 0.8924056043823589, "test/p_micro": 0.895017235976183, "test/r_macro": 0.8910879079225317, "test/r_micro": 0.895017235976183, "test/f1/COORD": 0.9739866908650938, "test/p/COORD": 0.961768219832736, "test/r/COORD": 0.9865196078431373, "test/f1/HYPER": 0.7987341772151898, "test/p/HYPER": 0.8184176394293126, "test/r/HYPER": 0.7799752781211372, "test/f1/RANDOM": 0.901873610670054, "test/p/RANDOM": 0.8970309538850284, "test/r/RANDOM": 0.9067688378033205}}

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "roberta-large",
+  "_name_or_path": "roberta-base",
   "architectures": [
     "RobertaModel"
   ],
@@ -9,19 +9,19 @@
   "eos_token_id": 2,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
-  "hidden_size": 1024,
+  "hidden_size": 768,
   "initializer_range": 0.02,
-  "intermediate_size": 4096,
+  "intermediate_size": 3072,
   "layer_norm_eps": 1e-05,
   "max_position_embeddings": 514,
   "model_type": "roberta",
-  "num_attention_heads": 16,
-  "num_hidden_layers": 24,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
   "pad_token_id": 1,
   "position_embedding_type": "absolute",
   "relbert_config": {
     "aggregation_mode": "average_no_mask",
-    "template": "I wasn\u2019t aware of this relationship, but I just read in the encyclopedia that <subj> is the <mask> of <obj>"
+    "template": "I wasn\u2019t aware of this relationship, but I just read in the encyclopedia that <obj>  is <subj>\u2019s <mask>"
   },
   "torch_dtype": "float32",
   "transformers_version": "4.26.1",

finetuning_config.json

@@ -1,6 +1,6 @@
 {
-  "template": "I wasn\u2019t aware of this relationship, but I just read in the encyclopedia that <subj> is the <mask> of <obj>",
-  "model": "roberta-large",
+  "template": "I wasn\u2019t aware of this relationship, but I just read in the encyclopedia that <obj>  is <subj>\u2019s <mask>",
+  "model": "roberta-base",
   "max_length": 64,
   "epoch": 10,
   "batch": 32,
@@ -17,7 +17,7 @@
   "classification_loss": false,
   "loss_function_config": {
     "temperature": 0.05,
-    "num_negative": 100,
+    "num_negative": 400,
     "num_positive": 10
   },
   "augment_negative_by_positive": true

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2625a24982aa7546e5d7c5bb28293ef3adbfdf6393b31806150e3ba3272c8759
-size 1421575277
+oid sha256:b4c70aba115b9aea37f6758c47c99150b652bf727eee744bc8147b8054e24ae1
+size 498652017

tokenizer.json

@@ -53,8 +53,7 @@
   "pre_tokenizer": {
     "type": "ByteLevel",
     "add_prefix_space": false,
-    "trim_offsets": true,
-    "use_regex": true
+    "trim_offsets": true
   },
   "post_processor": {
     "type": "RobertaProcessing",
@@ -72,8 +71,7 @@
   "decoder": {
     "type": "ByteLevel",
     "add_prefix_space": true,
-    "trim_offsets": true,
-    "use_regex": true
+    "trim_offsets": true
   },
   "model": {
     "type": "BPE",

tokenizer_config.json

@@ -6,7 +6,7 @@
   "errors": "replace",
   "mask_token": "<mask>",
   "model_max_length": 512,
-  "name_or_path": "roberta-large",
+  "name_or_path": "roberta-base",
   "pad_token": "<pad>",
   "sep_token": "</s>",
   "special_tokens_map_file": null,