Update README.md

README.md

@@ -132,22 +132,33 @@ test_dataset = test_dataset.map(speech_file_to_array_fn)
 # Preprocessing the datasets.
 # We need to read the audio files as arrays
 def evaluate(batch):
-	inputs = processor(batch["speech"], sampling_rate=16_000, return_tensors="pt", padding=True)
+\tinputs = processor(batch["speech"], sampling_rate=16_000, return_tensors="pt", padding=True)
 
-	with torch.no_grad():
-		logits = model(inputs.input_values.to(DEVICE), attention_mask=inputs.attention_mask.to(DEVICE)).logits
+\twith torch.no_grad():
+\t\tlogits = model(inputs.input_values.to(DEVICE), attention_mask=inputs.attention_mask.to(DEVICE)).logits
 
-	pred_ids = torch.argmax(logits, dim=-1)
-	batch["pred_strings"] = processor.batch_decode(pred_ids)
-	return batch
+\tpred_ids = torch.argmax(logits, dim=-1)
+\tbatch["pred_strings"] = processor.batch_decode(pred_ids)
+\treturn batch
 
 result = test_dataset.map(evaluate, batched=True, batch_size=8)
 
-print("WER: {:2f}".format(100 * wer.compute(predictions=result["pred_strings"], references=result["sentence"], chunk_size=1000)))
-print("CER: {:2f}".format(100 * cer.compute(predictions=result["pred_strings"], references=result["sentence"], chunk_size=1000)))
+predictions = [x.upper() for x in result["pred_strings"]]
+references = [x.upper() for x in result["sentence"]]
+
+print(f"WER: {wer.compute(predictions=predictions, references=references, chunk_size=1000) * 100}")
+print(f"CER: {cer.compute(predictions=predictions, references=references, chunk_size=1000) * 100}")
 ```
 
 **Test Result**:
 
-- WER: 11.85%
-- CER: 3.17%
+My model may report better scores than others because of some specificity of my evaluation script, so I ran the same evaluation script on other models (on 2021-04-22) to make a fairer comparison.
+
+| Model | WER | CER |
+| ------------- | ------------- | ------------- |
+| jonatasgrosman/wav2vec2-large-xlsr-53-german | **11.85%** | **3.17%** |
+| maxidl/wav2vec2-large-xlsr-german | 13.10% | 3.64% |
+| marcel/wav2vec2-large-xlsr-53-german | 15.97% | 4.37% |
+| flozi00/wav2vec-xlsr-german | 16.13% | 4.33% |
+| facebook/wav2vec2-large-xlsr-53-german | 17.15% | 5.79% |
+| MehdiHosseiniMoghadam/wav2vec2-large-xlsr-53-German | 19.31% | 5.41% |