Update README.md

README.md

@@ -4,6 +4,489 @@ language:
 - ro
 base_model:
 - meta-llama/Meta-Llama-3-8B
+datasets:
+- OpenLLM-Ro/ro_sft_alpaca
+- OpenLLM-Ro/ro_sft_alpaca_gpt4
+- OpenLLM-Ro/ro_sft_dolly
+- OpenLLM-Ro/ro_sft_selfinstruct_gpt4
+- OpenLLM-Ro/ro_sft_norobots
+- OpenLLM-Ro/ro_sft_orca
+- OpenLLM-Ro/ro_sft_camel
+model-index:
+    - name: OpenLLM-Ro/RoLlama3-8b-Instruct
+      results:
+        - task:
+            type: text-generation
+          dataset:
+            name: RoMT-Bench
+            type: RoMT-Bench
+          metrics:
+            - name: Score
+              type: Score
+              value: 5.15
+        - task:
+            type: text-generation
+          dataset:
+            name: RoCulturaBench
+            type: RoCulturaBench
+          metrics:
+            - name: Score
+              type: Score
+              value: 3.71
+        - task:
+            type: text-generation
+          dataset:
+            name: Romanian_Academic_Benchmarks
+            type: Romanian_Academic_Benchmarks
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 50.56
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_arc_challenge
+            type: OpenLLM-Ro/ro_arc_challenge
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 44.70
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_mmlu
+            type: OpenLLM-Ro/ro_mmlu
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 52.19
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_winogrande
+            type: OpenLLM-Ro/ro_winogrande
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 67.23
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_hellaswag
+            type: OpenLLM-Ro/ro_hellaswag
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 57.69
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_gsm8k
+            type: OpenLLM-Ro/ro_gsm8k
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 30.23
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_truthfulqa
+            type: OpenLLM-Ro/ro_truthfulqa
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 51.34
+        - task:
+            type: text-generation
+          dataset:
+            name: LaRoSeDa_binary
+            type: LaRoSeDa_binary
+          metrics:
+            - name: Average macro-f1
+              type: macro-f1
+              value: 97.52
+        - task:
+            type: text-generation
+          dataset:
+            name: LaRoSeDa_multiclass
+            type: LaRoSeDa_multiclass
+          metrics:
+            - name: Average macro-f1
+              type: macro-f1
+              value: 67.41
+        - task:
+            type: text-generation
+          dataset:
+            name: LaRoSeDa_binary_finetuned
+            type: LaRoSeDa_binary_finetuned
+          metrics:
+            - name: Average macro-f1
+              type: macro-f1
+              value: 94.15
+        - task:
+            type: text-generation
+          dataset:
+            name: LaRoSeDa_multiclass_finetuned
+            type: LaRoSeDa_multiclass_finetuned
+          metrics:
+            - name: Average macro-f1
+              type: macro-f1
+              value: 87.13
+        - task:
+            type: text-generation
+          dataset:
+            name: WMT_EN-RO
+            type: WMT_EN-RO
+          metrics:
+            - name: Average bleu
+              type: bleu
+              value: 24.01
+        - task:
+            type: text-generation
+          dataset:
+            name: WMT_RO-EN
+            type: WMT_RO-EN
+          metrics:
+            - name: Average bleu
+              type: bleu
+              value: 27.36
+        - task:
+            type: text-generation
+          dataset:
+            name: WMT_EN-RO_finetuned
+            type: WMT_EN-RO_finetuned
+          metrics:
+            - name: Average bleu
+              type: bleu
+              value: 26.53
+        - task:
+            type: text-generation
+          dataset:
+            name: WMT_RO-EN_finetuned
+            type: WMT_RO-EN_finetuned
+          metrics:
+            - name: Average bleu
+              type: bleu
+              value: 40.36
+        - task:
+            type: text-generation
+          dataset:
+            name: XQuAD
+            type: XQuAD
+          metrics:
+            - name: Average exact_match
+              type: exact_match
+              value: 39.43
+        - task:
+            type: text-generation
+          dataset:
+            name: XQuAD
+            type: XQuAD
+          metrics:
+            - name: Average f1
+              type: f1
+              value: 59.50
+        - task:
+            type: text-generation
+          dataset:
+            name: XQuAD_finetuned
+            type: XQuAD_finetuned
+          metrics:
+            - name: Average exact_match
+              type: exact_match
+              value: 44.45
+        - task:
+            type: text-generation
+          dataset:
+            name: XQuAD_finetuned
+            type: XQuAD_finetuned
+          metrics:
+            - name: Average f1
+              type: f1
+              value: 59.76
+        - task:
+            type: text-generation
+          dataset:
+            name: STS
+            type: STS
+          metrics:
+            - name: Average spearman
+              type: spearman
+              value: 77.20
+        - task:
+            type: text-generation
+          dataset:
+            name: STS
+            type: STS
+          metrics:
+            - name: Average pearson
+              type: pearson
+              value: 77.87
+        - task:
+            type: text-generation
+          dataset:
+            name: STS_finetuned
+            type: STS_finetuned
+          metrics:
+            - name: Average spearman
+              type: spearman
+              value: 85.80
+        - task:
+            type: text-generation
+          dataset:
+            name: STS_finetuned
+            type: STS_finetuned
+          metrics:
+            - name: Average pearson
+              type: pearson
+              value: 86.05
+        - task:
+            type: text-generation
+          dataset:
+            name: RoMT-Bench
+            type: RoMT-Bench
+          metrics:
+            - name: First turn
+              type: Score
+              value: 6.03
+            - name: Second turn
+              type: Score
+              value: 4.28
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_arc_challenge
+            type: OpenLLM-Ro/ro_arc_challenge
+          metrics:
+            - name: 0-shot 
+              type: accuracy
+              value: 41.90
+            - name: 1-shot 
+              type: accuracy
+              value: 44.30
+            - name: 3-shot 
+              type: accuracy
+              value: 44.56
+            - name: 5-shot 
+              type: accuracy
+              value: 45.50
+            - name: 10-shot 
+              type: accuracy
+              value: 46.10
+            - name: 25-shot 
+              type: accuracy
+              value: 45.84
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_mmlu
+            type: OpenLLM-Ro/ro_mmlu
+          metrics:
+            - name: 0-shot 
+              type: accuracy
+              value: 50.85
+            - name: 1-shot 
+              type: accuracy
+              value: 51.24
+            - name: 3-shot 
+              type: accuracy
+              value: 53.30
+            - name: 5-shot 
+              type: accuracy
+              value: 53.39
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_winogrande
+            type: OpenLLM-Ro/ro_winogrande
+          metrics:
+            - name: 0-shot 
+              type: accuracy
+              value: 65.19
+            - name: 1-shot 
+              type: accuracy
+              value: 66.54
+            - name: 3-shot 
+              type: accuracy
+              value: 67.88
+            - name: 5-shot 
+              type: accuracy
+              value: 69.30
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_hellaswag
+            type: OpenLLM-Ro/ro_hellaswag
+          metrics:
+            - name: 0-shot 
+              type: accuracy
+              value: 56.12
+            - name: 1-shot 
+              type: accuracy
+              value: 57.37
+            - name: 3-shot 
+              type: accuracy
+              value: 57.92
+            - name: 5-shot 
+              type: accuracy
+              value: 58.18
+            - name: 10-shot 
+              type: accuracy
+              value: 58.85
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_gsm8k
+            type: OpenLLM-Ro/ro_gsm8k
+          metrics:
+            - name: 0-shot 
+              type: accuracy
+              value: 29.42
+            - name: 1-shot 
+              type: accuracy
+              value: 30.02
+            - name: 3-shot 
+              type: accuracy
+              value: 31.24
+        - task:
+            type: text-generation
+          dataset:
+            name: LaRoSeDa_binary
+            type: LaRoSeDa_binary
+          metrics:
+            - name: 0-shot 
+              type: macro-f1
+              value: 97.43
+            - name: 1-shot 
+              type: macro-f1
+              value: 96.60
+            - name: 3-shot 
+              type: macro-f1
+              value: 97.90
+            - name: 5-shot 
+              type: macro-f1
+              value: 98.13
+        - task:
+            type: text-generation
+          dataset:
+            name: LaRoSeDa_multiclass
+            type: LaRoSeDa_multiclass
+          metrics:
+            - name: 0-shot 
+              type: macro-f1
+              value: 63.77
+            - name: 1-shot 
+              type: macro-f1
+              value: 68.91
+            - name: 3-shot 
+              type: macro-f1
+              value: 66.36
+            - name: 5-shot 
+              type: macro-f1
+              value: 70.61
+        - task:
+            type: text-generation
+          dataset:
+            name: WMT_EN-RO
+            type: WMT_EN-RO
+          metrics:
+            - name: 0-shot 
+              type: bleu
+              value: 6.92
+            - name: 1-shot 
+              type: bleu
+              value: 29.33
+            - name: 3-shot 
+              type: bleu
+              value: 29.79
+            - name: 5-shot 
+              type: bleu
+              value: 30.02
+        - task:
+            type: text-generation
+          dataset:
+            name: WMT_RO-EN
+            type: WMT_RO-EN
+          metrics:
+            - name: 0-shot 
+              type: bleu
+              value: 4.50
+            - name: 1-shot 
+              type: bleu
+              value: 30.30
+            - name: 3-shot 
+              type: bleu
+              value: 36.96
+            - name: 5-shot 
+              type: bleu
+              value: 37.70
+        - task:
+            type: text-generation
+          dataset:
+            name: XQuAD_EM
+            type: XQuAD_EM
+          metrics:
+            - name: 0-shot 
+              type: exact_match
+              value: 4.45
+            - name: 1-shot 
+              type: exact_match
+              value: 48.24
+            - name: 3-shot 
+              type: exact_match
+              value: 52.03
+            - name: 5-shot 
+              type: exact_match
+              value: 53.03
+        - task:
+            type: text-generation
+          dataset:
+            name: XQuAD_F1
+            type: XQuAD_F1
+          metrics:
+            - name: 0-shot 
+              type: f1
+              value: 26.08
+            - name: 1-shot 
+              type: f1
+              value: 68.40
+            - name: 3-shot 
+              type: f1
+              value: 71.92
+            - name: 5-shot 
+              type: f1
+              value: 71.60
+        - task:
+            type: text-generation
+          dataset:
+            name: STS
+            type: STS
+          metrics:
+            - name: 0-shot 
+              type: spearman
+              value: 77.76
+            - name: 1-shot 
+              type: spearman
+              value: 76.72
+            - name: 3-shot 
+              type: spearman
+              value: 77.12
+        - task:
+            type: text-generation
+          dataset:
+            name: STS
+            type: STS
+          metrics:
+            - name: 0-shot 
+              type: pearson
+              value: 77.83
+            - name: 1-shot 
+              type: pearson
+              value: 77.64
+            - name: 3-shot 
+              type: pearson
+              value: 78.13
+
 ---
 
 # Model Card for Model ID
@@ -31,13 +514,14 @@ OpenLLM-Ro represents the first open-source effort to build a LLM specialized fo
 - **Language(s):** Romanian
 - **License:** cc-by-nc-4.0
 - **Finetuned from model:** [Meta-Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B)
+- **Trained using:** [RoAlpaca](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_alpaca), [RoAlpacaGPT4](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_alpaca_gpt4), [RoDolly](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_dolly), [RoSelfInstruct](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_selfinstruct_gpt4), [RoNoRobots](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_norobots), [RoOrca](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_orca), [RoCamel](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_camel)
 
 
 ### Model Sources
 
 <!-- Provide the basic links for the model. -->
 
-- **Repository:** https://github.com/OpenLLM-Ro/llama-recipes
+- **Repository:** https://github.com/OpenLLM-Ro/LLaMA-Factory
 - **Paper:** https://arxiv.org/abs/2406.18266
 
 ## Intended Use
@@ -76,32 +560,138 @@ outputs = model.generate(input_ids=inputs, max_new_tokens=128)
 print(tokenizer.decode(outputs[0]))
 ```
 
-## Benchmarks
-
-| Model              | Average  | ARC      | MMLU     |Winogrande|HellaSwag | GSM8k    |TruthfulQA|
-|--------------------|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|
-| Llama-3-8B-Instruct| 50.15     | 43.73     | 49.02    | 59.35    | 53.16    | **44.12**    | 51.52    |  
-| *RoLlama3-8b-Instruct*  | ***50.61***    | ***44.66***    | ***52.19***    | ***67.58***    | ***57.65***    | *30.20*     | ***51.39***   |  
-
-        
+## Academic Benchmarks
+
+<table>
+<tbody>
+<tr>
+<td><strong>Model</strong></td>
+<td><strong><center>Average</center></strong></td>
+<td><strong><center>ARC</center></strong></td>
+<td><strong><center>MMLU</center></strong></td>
+<td><strong><center>Winogrande</center></strong></td>
+<td><strong><center>Hellaswag</center></strong></td>
+<td><strong><center>GSM8k</center></strong></td>
+<td><strong><center>TruthfulQA</center></strong></td>
+</tr>
+<tr>
+<td>Llama-3-8B-Instruct</td><td><center><strong>50.62</strong></center></td><td><center>43.69</center></td><td><center>52.04</center></td><td><center>59.33</center></td><td><center>53.19</center></td><td><center><strong>43.87</strong></center></td><td><center><strong>51.59</strong></center></td>
+</tr>
+<tr>
+<td><em>RoLlama3-8b-Instruct</em></td><td><center><em>50.56</em></center></td><td><center><em><strong>44.70</strong></em></center></td><td><center><em><strong>52.20</strong></em></center></td><td><center><em><strong>67.23</strong></em></center></td><td><center><em><strong>57.69</strong></em></center></td><td><center><em>30.23</em></center></td><td><center><em>51.34</em></center></td>
+</tr>
+</tbody>
+</table>
+
+## Downstream tasks
+
+<table>
+<tbody>
+<tr>
+<td></td>
+<td colspan="4"><center><strong>LaRoSeDa</strong></center></td>
+<td colspan="4"><center><strong>WMT</strong></center></td>
+</tr>
+<tr>
+<td></td>
+<td colspan="2"><center><strong>Few-shot</strong></center></td>
+<td colspan="2"><center><strong>Finetuned</strong></center></td>
+<td colspan="2"><center><strong>Few-shot</strong></center></td>
+<td colspan="2"><center><strong>Finetuned</strong></center></td>
+</tr>
+<tr>
+<td><strong>Model</strong></td>
+<td><center><strong>Binary<br>(Macro F1)</strong></center></td>
+<td><center><strong>Multiclass<br>(Macro F1)</strong></center></td>
+<td><center><strong>Binary<br>(Macro F1)</strong></center></td>
+<td><center><strong>Multiclass<br>(Macro F1)</strong></center></td>
+<td><center><strong>EN-RO<br>(Bleu)</strong></center></td>
+<td><center><strong>RO-EN<br>(Bleu)</strong></center></td>
+<td><center><strong>EN-RO<br>(Bleu)</strong></center></td>
+<td><center><strong>RO-EN<br>(Bleu)</strong></center>
+</tr>
+<tr>
+<td>Llama-3-8B-Instruct</td><td><center>95.88</center></td><td><center>56.21</center></td><td><center><strong>98.53</strong></center></td><td><center>86.19</center></td><td><center>18.89</center></td><td><center><strong>30.98</strong></center></td><td><center><strong>28.02</strong></center></td><td><center>40.28</center></td>
+</tr>
+<tr>
+<td><em>RoLlama3-8b-Instruct</em></td><td><center><em><strong>97.52</strong></em></center></td><td><center><em><strong>67.41</strong></em></center></td><td><center><em>94.15</em></center></td><td><center><em><strong>87.13</strong></em></center></td><td><center><em><strong>24.02</strong></em></center></td><td><center><em>27.37</em></center></td><td><center><em>26.53</em></center></td><td><center><em><strong>40.37</strong></em></center></td>
+</tr>
+</tbody>
+</table>
+
+<table>
+<tbody>
+<tr>
+<td></td>
+<td colspan="4"><center><strong>XQuAD</strong></center></td>
+<td colspan="4"><center><strong>STS</strong></center></td>
+</tr>
+<tr>
+<td></td>
+<td colspan="2"><center><strong>Few-shot</strong></center></td>
+<td colspan="2"><center><strong>Finetuned</strong></center></td>
+<td colspan="2"><center><strong>Few-shot</strong></center></td>
+<td colspan="2"><center><strong>Finetuned</strong></center></td>
+</tr>
+<tr>
+<td><strong>Model</strong></td>
+<td><center><strong>(EM)</strong></center></td>
+<td><center><strong>(F1)</strong></center></td>
+<td><center><strong>(EM)</strong></center></td>
+<td><center><strong>(F1)</strong></center></td>
+<td><center><strong>(Spearman)</strong></center></td>
+<td><center><strong>(Pearson)</strong></center></td>
+<td><center><strong>(Spearman)</strong></center></td>
+<td><center><strong>(Pearson)</strong></center></td>
+</tr>
+<tr>
+<td>Llama-3-8B-Instruct</td><td><center><strong>39.48</strong></center></td><td><center>58.67</center></td><td><center><strong>67.65</strong></center></td><td><center><strong>82.77</strong></center></td><td><center>73.04</center></td><td><center>72.36</center></td><td><center>83.49</center></td><td><center>84.06</center></td>
+</tr>
+<tr>
+<td><em>RoLlama3-8b-Instruct</em></td><td><center><em>39.44</em></center></td><td><center><em><strong>59.50</strong></em></center></td><td><center><em>44.45</em></center></td><td><center><em>59.76</em></center></td><td><center><em><strong>77.20</strong></em></center></td><td><center><em><strong>77.87</strong></em></center></td><td><center><em><strong>85.80</strong></em></center></td><td><center><em><strong>86.05</strong></em></center></td>
+</tr>
+</tbody>
+</table>
 
 
 ## MT-Bench
 
-| Model              | Average  | 1st turn      | 2nd turn     |  Answers in Ro |
-|--------------------|:--------:|:--------:|:--------:|:--------:|
-| Llama-3-8B-Instruct    | **5.92**    | **6.36**    | **5.49**    | 158 / 160
-| *RoLlama3-8b-Instruct*| *5.28* |*6.10*| *4.45* | ***160 / 160*** |
-
+<table>
+<tbody>
+<tr>
+<td><strong>Model</strong></td>
+<td><strong><center>Average</center></strong></td>
+<td><strong><center>1st turn</center></strong></td>
+<td><strong><center>2nd turn</center></strong></td>
+<td><strong><center>Answers in Ro</center></strong></td>
+</tr>
+<tr>
+<td><em>Llama-3-8B-Instruct</em></td><td><center><em><strong>5.96</strong></em></center></td><td><center><em><strong>6.16</strong></em></center></td><td><center><em><strong>5.76</strong></em></center></td><td><center>158/160</center></td>
+</tr>
+<tr>
+<td><em>RoLlama3-8b-Instruct</em></td><td><center><em>5.15</em></center></td><td><center><em>6.03</em></center></td><td><center><em>4.28</em></center></td><td><center><em><strong>160/160</strong></em></center></td>
+</tr>
+</tbody>
+</table>
 
 
 ## RoCulturaBench
 
-| Model              | Score  | Answers in Ro|
-|--------------------|:--------:|:--------:|
-| Llama-3-8B-Instruct   | **4.61**    | **100 / 100**  |
-| *RoLlama3-8b-Instruct*| *3.83*| ***100 / 100*** |
-
+<table>
+<tbody>
+<tr>
+<td><strong>Model</strong></td>
+<td><strong><center>Average</center></strong></td>
+<td><strong><center>Answers in Ro</center></strong></td>
+</tr>
+<tr>
+<td><em>Llama-3-8B-Instruct</em></td><td><center><em><strong>4.62</strong></em></center></td><td><center><strong>100/100</strong></center></td>
+</tr>
+<tr>
+<td><em>RoLlama3-8b-Instruct</em></td><td><center><em>3.71</em></center></td><td><center><em><strong>100/100</strong></em></center></td>
+</tr>
+</tbody>
+</table>
 
 
 ## RoLlama3 Model Family