Update README.md

README.md

@@ -14,6 +14,481 @@ datasets:
 - OpenLLM-Ro/ro_sft_camel
 - OpenLLM-Ro/ro_sft_oasst
 - OpenLLM-Ro/ro_sft_ultrachat
+model-index:
+    - name: OpenLLM-Ro/RoLlama2-7b-Instruct-v2
+      results:
+        - task:
+            type: text-generation
+          dataset:
+            name: RoMT-Bench
+            type: RoMT-Bench
+          metrics:
+            - name: Score
+              type: Score
+              value: 4.43
+        - task:
+            type: text-generation
+          dataset:
+            name: RoCulturaBench
+            type: RoCulturaBench
+          metrics:
+            - name: Score
+              type: Score
+              value: 4.08
+        - task:
+            type: text-generation
+          dataset:
+            name: Romanian_Academic_Benchmarks
+            type: Romanian_Academic_Benchmarks
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 44.50
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_arc_challenge
+            type: OpenLLM-Ro/ro_arc_challenge
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 44.73
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_mmlu
+            type: OpenLLM-Ro/ro_mmlu
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 40.39
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_winogrande
+            type: OpenLLM-Ro/ro_winogrande
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 63.67
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_hellaswag
+            type: OpenLLM-Ro/ro_hellaswag
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 59.12
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_gsm8k
+            type: OpenLLM-Ro/ro_gsm8k
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 13.29
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_truthfulqa
+            type: OpenLLM-Ro/ro_truthfulqa
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 45.78
+        - task:
+            type: text-generation
+          dataset:
+            name: LaRoSeDa_binary
+            type: LaRoSeDa_binary
+          metrics:
+            - name: Average macro-f1
+              type: macro-f1
+              value: 97.66
+        - task:
+            type: text-generation
+          dataset:
+            name: LaRoSeDa_multiclass
+            type: LaRoSeDa_multiclass
+          metrics:
+            - name: Average macro-f1
+              type: macro-f1
+              value: 62.41
+        - task:
+            type: text-generation
+          dataset:
+            name: LaRoSeDa_binary_finetuned
+            type: LaRoSeDa_binary_finetuned
+          metrics:
+            - name: Average macro-f1
+              type: macro-f1
+              value: 97.97
+        - task:
+            type: text-generation
+          dataset:
+            name: LaRoSeDa_multiclass_finetuned
+            type: LaRoSeDa_multiclass_finetuned
+          metrics:
+            - name: Average macro-f1
+              type: macro-f1
+              value: 60.89
+        - task:
+            type: text-generation
+          dataset:
+            name: WMT_EN-RO
+            type: WMT_EN-RO
+          metrics:
+            - name: Average bleu
+              type: bleu
+              value: 27.13
+        - task:
+            type: text-generation
+          dataset:
+            name: WMT_RO-EN
+            type: WMT_RO-EN
+          metrics:
+            - name: Average bleu
+              type: bleu
+              value: 19.39
+        - task:
+            type: text-generation
+          dataset:
+            name: WMT_EN-RO_finetuned
+            type: WMT_EN-RO_finetuned
+          metrics:
+            - name: Average bleu
+              type: bleu
+              value: 27.63
+        - task:
+            type: text-generation
+          dataset:
+            name: WMT_RO-EN_finetuned
+            type: WMT_RO-EN_finetuned
+          metrics:
+            - name: Average bleu
+              type: bleu
+              value: 39.75
+        - task:
+            type: text-generation
+          dataset:
+            name: XQuAD
+            type: XQuAD
+          metrics:
+            - name: Average exact_match
+              type: exact_match
+              value: 45.71
+        - task:
+            type: text-generation
+          dataset:
+            name: XQuAD
+            type: XQuAD
+          metrics:
+            - name: Average f1
+              type: f1
+              value: 65.08
+        - task:
+            type: text-generation
+          dataset:
+            name: XQuAD_finetuned
+            type: XQuAD_finetuned
+          metrics:
+            - name: Average exact_match
+              type: exact_match
+              value: 59.24
+        - task:
+            type: text-generation
+          dataset:
+            name: XQuAD_finetuned
+            type: XQuAD_finetuned
+          metrics:
+            - name: Average f1
+              type: f1
+              value: 74.25
+        - task:
+            type: text-generation
+          dataset:
+            name: STS
+            type: STS
+          metrics:
+            - name: Average spearman
+              type: spearman
+              value: 59.69
+        - task:
+            type: text-generation
+          dataset:
+            name: STS
+            type: STS
+          metrics:
+            - name: Average pearson
+              type: pearson
+              value: 57.16
+        - task:
+            type: text-generation
+          dataset:
+            name: STS_finetuned
+            type: STS_finetuned
+          metrics:
+            - name: Average spearman
+              type: spearman
+              value: 84.66
+        - task:
+            type: text-generation
+          dataset:
+            name: STS_finetuned
+            type: STS_finetuned
+          metrics:
+            - name: Average pearson
+              type: pearson
+              value: 85.07
+        - task:
+            type: text-generation
+          dataset:
+            name: RoMT-Bench
+            type: RoMT-Bench
+          metrics:
+            - name: First turn
+              type: Score
+              value: 4.92
+            - name: Second turn
+              type: Score
+              value: 3.94
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_arc_challenge
+            type: OpenLLM-Ro/ro_arc_challenge
+          metrics:
+            - name: 0-shot 
+              type: accuracy
+              value: 42.67
+            - name: 1-shot 
+              type: accuracy
+              value: 44.64
+            - name: 3-shot 
+              type: accuracy
+              value: 44.90
+            - name: 5-shot 
+              type: accuracy
+              value: 45.16
+            - name: 10-shot 
+              type: accuracy
+              value: 45.67
+            - name: 25-shot 
+              type: accuracy
+              value: 45.33
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_mmlu
+            type: OpenLLM-Ro/ro_mmlu
+          metrics:
+            - name: 0-shot 
+              type: accuracy
+              value: 39.89
+            - name: 1-shot 
+              type: accuracy
+              value: 40.08
+            - name: 3-shot 
+              type: accuracy
+              value: 40.60
+            - name: 5-shot 
+              type: accuracy
+              value: 40.99
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_winogrande
+            type: OpenLLM-Ro/ro_winogrande
+          metrics:
+            - name: 0-shot 
+              type: accuracy
+              value: 63.06
+            - name: 1-shot 
+              type: accuracy
+              value: 62.98
+            - name: 3-shot 
+              type: accuracy
+              value: 65.19
+            - name: 5-shot 
+              type: accuracy
+              value: 63.46
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_hellaswag
+            type: OpenLLM-Ro/ro_hellaswag
+          metrics:
+            - name: 0-shot 
+              type: accuracy
+              value: 58.82
+            - name: 1-shot 
+              type: accuracy
+              value: 58.44
+            - name: 3-shot 
+              type: accuracy
+              value: 59.28
+            - name: 5-shot 
+              type: accuracy
+              value: 59.29
+            - name: 10-shot 
+              type: accuracy
+              value: 59.77
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_gsm8k
+            type: OpenLLM-Ro/ro_gsm8k
+          metrics:
+            - name: 0-shot 
+              type: accuracy
+              value: 6.14
+            - name: 1-shot 
+              type: accuracy
+              value: 15.01
+            - name: 3-shot 
+              type: accuracy
+              value: 18.72
+        - task:
+            type: text-generation
+          dataset:
+            name: LaRoSeDa_binary
+            type: LaRoSeDa_binary
+          metrics:
+            - name: 0-shot 
+              type: macro-f1
+              value: 98.20
+            - name: 1-shot 
+              type: macro-f1
+              value: 96.63
+            - name: 3-shot 
+              type: macro-f1
+              value: 97.67
+            - name: 5-shot 
+              type: macro-f1
+              value: 98.13
+        - task:
+            type: text-generation
+          dataset:
+            name: LaRoSeDa_multiclass
+            type: LaRoSeDa_multiclass
+          metrics:
+            - name: 0-shot 
+              type: macro-f1
+              value: 63.43
+            - name: 1-shot 
+              type: macro-f1
+              value: 53.58
+            - name: 3-shot 
+              type: macro-f1
+              value: 63.78
+            - name: 5-shot 
+              type: macro-f1
+              value: 68.85
+        - task:
+            type: text-generation
+          dataset:
+            name: WMT_EN-RO
+            type: WMT_EN-RO
+          metrics:
+            - name: 0-shot 
+              type: bleu
+              value: 20.57
+            - name: 1-shot 
+              type: bleu
+              value: 29.59
+            - name: 3-shot 
+              type: bleu
+              value: 29.50
+            - name: 5-shot 
+              type: bleu
+              value: 28.88
+        - task:
+            type: text-generation
+          dataset:
+            name: WMT_RO-EN
+            type: WMT_RO-EN
+          metrics:
+            - name: 0-shot 
+              type: bleu
+              value: 2.19
+            - name: 1-shot 
+              type: bleu
+              value: 9.97
+            - name: 3-shot 
+              type: bleu
+              value: 31.19
+            - name: 5-shot 
+              type: bleu
+              value: 34.23
+        - task:
+            type: text-generation
+          dataset:
+            name: XQuAD_EM
+            type: XQuAD_EM
+          metrics:
+            - name: 0-shot 
+              type: exact_match
+              value: 40.25
+            - name: 1-shot 
+              type: exact_match
+              value: 46.47
+            - name: 3-shot 
+              type: exact_match
+              value: 47.56
+            - name: 5-shot 
+              type: exact_match
+              value: 48.57
+        - task:
+            type: text-generation
+          dataset:
+            name: XQuAD_F1
+            type: XQuAD_F1
+          metrics:
+            - name: 0-shot 
+              type: f1
+              value: 62.24
+            - name: 1-shot 
+              type: f1
+              value: 65.33
+            - name: 3-shot 
+              type: f1
+              value: 65.89
+            - name: 5-shot 
+              type: f1
+              value: 66.86
+        - task:
+            type: text-generation
+          dataset:
+            name: STS
+            type: STS
+          metrics:
+            - name: 0-shot 
+              type: spearman
+              value: 55.44
+            - name: 1-shot 
+              type: spearman
+              value: 61.98
+            - name: 3-shot 
+              type: spearman
+              value: 61.65
+        - task:
+            type: text-generation
+          dataset:
+            name: STS
+            type: STS
+          metrics:
+            - name: 0-shot 
+              type: pearson
+              value: 56.18
+            - name: 1-shot 
+              type: pearson
+              value: 58.37
+            - name: 3-shot 
+              type: pearson
+              value: 56.94
+
 ---
 
 # Model Card for Model ID
@@ -37,7 +512,8 @@ OpenLLM represents the first open-source effort to build a LLM specialized for R
 - **Language(s):** Romanian
 - **License:** cc-by-nc-4.0
 - **Finetuned from model:** [RoLlama2-7b-Base](https://huggingface.co/OpenLLM-Ro/RoLlama2-7b-Base)
-- **Trained using:** [RoAlpaca](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_alpaca), [RoAlpacaGPT4](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_alpaca_gpt4), [RoDolly](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_dolly), [RoSelfInstruct](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_selfinstruct_gpt4), [RoNoRobots](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_norobots), [RoOrca](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_orca), [RoCamel](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_camel)
+- **Trained using:** [RoAlpaca](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_alpaca), [RoAlpacaGPT4](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_alpaca_gpt4), [RoDolly](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_dolly), [RoSelfInstruct](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_selfinstruct_gpt4), [RoNoRobots](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_norobots), [RoOrca](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_orca), [RoCamel](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_camel), [RoOpenAssistant](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_oasst), [RoUltraChat](https://huggingface.co/datasets/OpenLLM-Ro/ro_sft_ultrachat)
+
 
 ### Model Sources
 
@@ -101,12 +577,16 @@ print(tokenizer.decode(outputs[0]))
 <td>Llama-2-7b-chat</td><td><center>36.84</center></td><td><center>37.03</center></td><td><center>33.80</center></td><td><center>55.87</center></td><td><center>45.36</center></td><td><center>4.90</center></td><td><center>44.09</center></td>
 </tr>
 <tr>
-<td><em>RoLlama2-7b-Instruct</em></td><td><center><em><strong>45.71</strong></em></center></td><td><center><em><strong>43.66</strong></em></center></td><td><center><em><strong>39.70</strong></em></center></td><td><center><em><strong>70.34</strong></em></center></td><td><center><em><strong>57.36</strong></em></center></td><td><center><em><strong>18.78</strong></em></center></td><td><center><em><strong>44.44</strong></em></center></td>
+<td>RoLlama2-7b-Instruct</td><td><center><strong>45.71</strong></center></td><td><center>43.66</center></td><td><center>39.70</center></td><td><center><strong>70.34</strong></center></td><td><center>57.36</center></td><td><center><strong>18.78</strong></center></td><td><center>44.44</center></td>
+</tr>
+<tr>
+<td><em>RoLlama2-7b-Instruct-v2</em></td><td><center><em>44.50</em></center></td><td><center><em><strong>44.73</strong></em></center></td><td><center><em><strong>40.39</strong></em></center></td><td><center><em>63.67</em></center></td><td><center><em><strong>59.12</strong></em></center></td><td><center><em>13.29</em></center></td><td><center><em><strong>45.78</strong></em></center></td>
 </tr>
 </tbody>
 </table>
 
 
+
 ## Downstream tasks
 
                                   
@@ -139,7 +619,10 @@ print(tokenizer.decode(outputs[0]))
 <td>Llama-2-7b-chat</td><td><center>87.78</center></td><td><center>52.81</center></td><td><center>97.27</center></td><td><center>82.02</center></td><td><center>15.55</center></td><td><center><strong>28.53</strong></center></td><td><center>19.99</center></td><td><center>31.48</center></td>
 </tr>
 <tr>
-<td><em>RoLlama2-7b-Instruct</em></td><td><center><em><strong>97.48</strong></em></center></td><td><center><em><strong>65.26</strong></em></center></td><td><center><em><strong>98.83</strong></em></center></td><td><center><em><strong>87.28</strong></em></center></td><td><center><em><strong>27.38</strong></em></center></td><td><center><em>10.32</em></center></td><td><center><em><strong>27.59</strong></em></center></td><td><center><em><strong>40.13</strong></em></center></td>
+<td>RoLlama2-7b-Instruct</td><td><center>97.48</center></td><td><center><strong>65.26</strong></center></td><td><center><strong>98.83</strong></center></td><td><center><strong>87.28</strong></center></td><td><center><strong>27.38</strong></center></td><td><center>10.32</center></td><td><center>27.59</center></td><td><center><strong>40.13</strong></center></td>
+</tr>
+<tr>
+<td><em>RoLlama2-7b-Instruct-v2</em></td><td><center><em><strong>97.66</strong></em></center></td><td><center><em>62.41</em></center></td><td><center><em>97.97</em></center></td><td><center><em>60.89</em></center></td><td><center><em>27.13</em></center></td><td><center><em>19.39</em></center></td><td><center><em><strong>27.63</strong></em></center></td><td><center><em>39.75</em></center></td>
 </tr>
 </tbody>
 </table>
@@ -174,7 +657,10 @@ print(tokenizer.decode(outputs[0]))
 <td>Llama-2-7b-chat</td><td><center>32.35</center></td><td><center>54.00</center></td><td><center><strong>60.34</strong></center></td><td><center><strong>75.98</strong></center></td><td><center>32.56</center></td><td><center>31.99</center></td><td><center>74.08</center></td><td><center>72.64</center></td>
 </tr>
 <tr>
-<td><em>RoLlama2-7b-Instruct</em></td><td><center><em><strong>44.52</strong></em></center></td><td><center><em><strong>64.75</strong></em></center></td><td><center><em>54.96</em></center></td><td><center><em>70.20</em></center></td><td><center><em><strong>65.50</strong></em></center></td><td><center><em><strong>67.79</strong></em></center></td><td><center><em><strong>84.44</strong></em></center></td><td><center><em><strong>84.76</strong></em></center></td>
+<td>RoLlama2-7b-Instruct</td><td><center>44.52</center></td><td><center>64.75</center></td><td><center>54.96</center></td><td><center>70.20</center></td><td><center><strong>65.50</strong></center></td><td><center><strong>67.79</strong></center></td><td><center>84.44</center></td><td><center>84.76</center></td>
+</tr>
+<tr>
+<td><em>RoLlama2-7b-Instruct-v2</em></td><td><center><em><strong>45.71</strong></em></center></td><td><center><em><strong>65.08</strong></em></center></td><td><center><em>59.24</em></center></td><td><center><em>74.25</em></center></td><td><center><em>59.69</em></center></td><td><center><em>57.16</em></center></td><td><center><em><strong>84.66</strong></em></center></td><td><center><em><strong>85.07</strong></em></center></td>
 </tr>
 </tbody>
 </table>
@@ -194,12 +680,16 @@ print(tokenizer.decode(outputs[0]))
 <td>Llama-2-7b-chat</td><td><center>1.08</center></td><td><center>1.44</center></td><td><center>0.73</center></td><td><center>45/160</center></td>
 </tr>
 <tr>
-<td><em>RoLlama2-7b-Instruct</em></td><td><center><em><strong>3.86</strong></em></center></td><td><center><em><strong>4.67</strong></em></center></td><td><center><em><strong>3.04</strong></em></center></td><td><center><em><strong>160/160</strong></em></center></td>
+<td>RoLlama2-7b-Instruct</td><td><center>3.86</center></td><td><center>4.67</center></td><td><center>3.04</center></td><td><center><strong>160/160</strong></center></td>
+</tr>
+<tr>
+<td><em>RoLlama2-7b-Instruct-v2</em></td><td><center><em><strong>4.43</strong></em></center></td><td><center><em><strong>4.92</strong></em></center></td><td><center><em><strong>3.94</strong></em></center></td><td><center><em><strong>160/160</strong></em></center></td>
 </tr>
 </tbody>
 </table>
 
 
+
 ## RoCulturaBench
 
 
@@ -214,7 +704,10 @@ print(tokenizer.decode(outputs[0]))
 <td>Llama-2-7b-chat</td><td><center>1.21</center></td><td><center>33/100</center></td>
 </tr>
 <tr>
-<td><em>RoLlama2-7b-Instruct</em></td><td><center><em><strong>3.77</strong></em></center></td><td><center><em><strong>100/100</strong></em></center></td>
+<td>RoLlama2-7b-Instruct</td><td><center>3.77</center></td><td><center><strong>100/100</strong></center></td>
+</tr>
+<tr>
+<td><em>RoLlama2-7b-Instruct-v2</em></td><td><center><em><strong>4.08</strong></em></center></td><td><center><em><strong>100/100</strong></em></center></td>
 </tr>
 </tbody>
 </table>