Update README.md

README.md

@@ -45,12 +45,17 @@ test_dataset = load_dataset("common_voice", "br", split="test[:2%]")
 processor = Wav2Vec2Processor.from_pretrained("cahya/wav2vec2-large-xlsr-breton")
 model = Wav2Vec2ForCTC.from_pretrained("cahya/wav2vec2-large-xlsr-breton")
 
-resampler = torchaudio.transforms.Resample(48_000, 16_000)
+chars_to_ignore_regex = '[\\,\,\?\.\!\;\:\"\“\%\”\�\(\)\/\«\»\½\…]'
 
 # Preprocessing the datasets.
 # We need to read the aduio files as arrays
 def speech_file_to_array_fn(batch):
+    batch["sentence"] = re.sub(chars_to_ignore_regex, '', batch["sentence"]).lower() + " "
+    batch["sentence"] = batch["sentence"].replace("ʼ", "'")
+    batch["sentence"] = batch["sentence"].replace("’", "'")
+    batch["sentence"] = batch["sentence"].replace('‘', "'")
     speech_array, sampling_rate = torchaudio.load(batch["path"])
+    resampler = torchaudio.transforms.Resample(sampling_rate, 16_000)
     batch["speech"] = resampler(speech_array).squeeze().numpy()
     return batch
 
@@ -85,15 +90,17 @@ processor = Wav2Vec2Processor.from_pretrained("cahya/wav2vec2-large-xlsr-breton"
 model = Wav2Vec2ForCTC.from_pretrained("cahya/wav2vec2-large-xlsr-breton") 
 model.to("cuda")
 
-chars_to_ignore_regex = '[\,\?\.\!\-\;\:\"\“\%\‘\'\”\�]'
-
-resampler = torchaudio.transforms.Resample(48_000, 16_000)
+chars_to_ignore_regex = '[\\,\,\?\.\!\;\:\"\“\%\”\�\(\)\/\«\»\½\…]'
 
 # Preprocessing the datasets.
 # We need to read the aduio files as arrays
 def speech_file_to_array_fn(batch):
-    batch["sentence"] = re.sub(chars_to_ignore_regex, '', batch["sentence"]).lower()
+    batch["sentence"] = re.sub(chars_to_ignore_regex, '', batch["sentence"]).lower() + " "
+    batch["sentence"] = batch["sentence"].replace("ʼ", "'")
+    batch["sentence"] = batch["sentence"].replace("’", "'")
+    batch["sentence"] = batch["sentence"].replace('‘', "'")
     speech_array, sampling_rate = torchaudio.load(batch["path"])
+    resampler = torchaudio.transforms.Resample(sampling_rate, 16_000)
     batch["speech"] = resampler(speech_array).squeeze().numpy()
     return batch