diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..9f6087d61de13558f31c30a06cbada0547966010 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +*.arpa filter=lfs diff=lfs merge=lfs -text diff --git a/EnglishCV/common_voice_prepare.py b/EnglishCV/common_voice_prepare.py new file mode 100644 index 0000000000000000000000000000000000000000..09fa6971a0fdcfdd1d73df7aa4297f1ce7f9e0f4 --- /dev/null +++ b/EnglishCV/common_voice_prepare.py @@ -0,0 +1,410 @@ +""" +Data preparation. +Download: https://voice.mozilla.org/en/datasets +Author +------ +Titouan Parcollet +Luca Della Libera 2022 +Pooneh Mousavi 2022 +""" + +from dataclasses import dataclass +import os +import csv +import re +import logging +import torchaudio +from tqdm import tqdm +import unicodedata +import functools +torchaudio.set_audio_backend("soundfile") +from speechbrain.utils.parallel import parallel_map +from speechbrain.dataio.dataio import read_audio_info + +logger = logging.getLogger(__name__) + + +def prepare_common_voice( + data_folder, + save_folder, + train_tsv_file=None, + dev_tsv_file=None, + test_tsv_file=None, + accented_letters=False, + language="en", + skip_prep=False, +): + """ + Prepares the csv files for the Mozilla Common Voice dataset. + Download: https://voice.mozilla.org/en/datasets + Arguments + --------- + data_folder : str + Path to the folder where the original Common Voice dataset is stored. + This path should include the lang: /datasets/CommonVoice// + save_folder : str + The directory where to store the csv files. + train_tsv_file : str, optional + Path to the Train Common Voice .tsv file (cs) + dev_tsv_file : str, optional + Path to the Dev Common Voice .tsv file (cs) + test_tsv_file : str, optional + Path to the Test Common Voice .tsv file (cs) + accented_letters : bool, optional + Defines if accented letters will be kept as individual letters or + transformed to the closest non-accented letters. + language: str + Specify the language for text normalization. + skip_prep: bool + If True, skip data preparation. + Example + ------- + >>> from recipes.CommonVoice.common_voice_prepare import prepare_common_voice + >>> data_folder = '/datasets/CommonVoice/en' + >>> save_folder = 'exp/CommonVoice_exp' + >>> train_tsv_file = '/datasets/CommonVoice/en/train.tsv' + >>> dev_tsv_file = '/datasets/CommonVoice/en/dev.tsv' + >>> test_tsv_file = '/datasets/CommonVoice/en/test.tsv' + >>> accented_letters = False + >>> duration_threshold = 10 + >>> prepare_common_voice( \ + data_folder, \ + save_folder, \ + train_tsv_file, \ + dev_tsv_file, \ + test_tsv_file, \ + accented_letters, \ + language="en" \ + ) + """ + + if skip_prep: + return + + # If not specified point toward standard location w.r.t CommonVoice tree + if train_tsv_file is None: + train_tsv_file = data_folder + "/train.tsv" + else: + train_tsv_file = train_tsv_file + + if dev_tsv_file is None: + dev_tsv_file = data_folder + "/dev.tsv" + else: + dev_tsv_file = dev_tsv_file + + if test_tsv_file is None: + test_tsv_file = data_folder + "/test.tsv" + else: + test_tsv_file = test_tsv_file + + # Setting the save folder + if not os.path.exists(save_folder): + os.makedirs(save_folder) + + # Setting ouput files + save_csv_train = save_folder + "/train.csv" + save_csv_dev = save_folder + "/dev.csv" + save_csv_test = save_folder + "/test.csv" + + # If csv already exists, we skip the data preparation + if skip(save_csv_train, save_csv_dev, save_csv_test): + + msg = "%s already exists, skipping data preparation!" % (save_csv_train) + logger.info(msg) + + msg = "%s already exists, skipping data preparation!" % (save_csv_dev) + logger.info(msg) + + msg = "%s already exists, skipping data preparation!" % (save_csv_test) + logger.info(msg) + + return + + # Additional checks to make sure the data folder contains Common Voice + check_commonvoice_folders(data_folder) + # Creating csv files for {train, dev, test} data + file_pairs = zip( + [train_tsv_file, dev_tsv_file, test_tsv_file], + [save_csv_train, save_csv_dev, save_csv_test], + ) + for tsv_file, save_csv in file_pairs: + create_csv( + tsv_file, save_csv, data_folder, accented_letters, language, + ) + + +def skip(save_csv_train, save_csv_dev, save_csv_test): + """ + Detects if the Common Voice data preparation has been already done. + If the preparation has been done, we can skip it. + Returns + ------- + bool + if True, the preparation phase can be skipped. + if False, it must be done. + """ + + # Checking folders and save options + skip = False + + if ( + os.path.isfile(save_csv_train) + and os.path.isfile(save_csv_dev) + and os.path.isfile(save_csv_test) + ): + skip = True + + return skip + + +@dataclass +class CVRow: + snt_id: str + duration: float + mp3_path: str + spk_id: str + words: str + + +def process_line(line, data_folder, language, accented_letters): + # Path is at indice 1 in Common Voice tsv files. And .mp3 files + # are located in datasets/lang/clips/ + mp3_path = data_folder + "/clips/" + line.split("\t")[1] + file_name = mp3_path.split(".")[-2].split("/")[-1] + spk_id = line.split("\t")[0] + snt_id = file_name + + # Setting torchaudio backend to sox-io (needed to read mp3 files) + """ + if torchaudio.get_audio_backend() != "sox_io": + logger.warning("This recipe needs the sox-io backend of torchaudio") + logger.warning("The torchaudio backend is changed to sox_io") + torchaudio.set_audio_backend("sox_io") + """ + # Reading the signal (to retrieve duration in seconds) + if os.path.isfile(mp3_path): + info = read_audio_info(mp3_path) + else: + msg = "\tError loading: %s" % (str(len(file_name))) + logger.info(msg) + return None + + duration = info.num_frames / info.sample_rate + + # Getting transcript + words = line.split("\t")[2] + + # Unicode Normalization + words = unicode_normalisation(words) + + # !! Language specific cleaning !! + words = language_specific_preprocess(language, words) + + # Remove accents if specified + if not accented_letters: + words = strip_accents(words) + words = words.replace("'", " ") + words = words.replace("’", " ") + + # Remove multiple spaces + words = re.sub(" +", " ", words) + + # Remove spaces at the beginning and the end of the sentence + words = words.lstrip().rstrip() + + # Getting chars + chars = words.replace(" ", "_") + chars = " ".join([char for char in chars][:]) + + # Remove too short sentences (or empty): + if language in ["ja", "ch"]: + if len(chars) < 3: + return None + else: + if len(words.split(" ")) < 3: + return None + + # Composition of the csv_line + return CVRow(snt_id, duration, mp3_path, spk_id, words) + + +def create_csv( + orig_tsv_file, csv_file, data_folder, accented_letters=False, language="en" +): + """ + Creates the csv file given a list of wav files. + Arguments + --------- + orig_tsv_file : str + Path to the Common Voice tsv file (standard file). + data_folder : str + Path of the CommonVoice dataset. + accented_letters : bool, optional + Defines if accented letters will be kept as individual letters or + transformed to the closest non-accented letters. + Returns + ------- + None + """ + + # Check if the given files exists + if not os.path.isfile(orig_tsv_file): + msg = "\t%s doesn't exist, verify your dataset!" % (orig_tsv_file) + logger.info(msg) + raise FileNotFoundError(msg) + + # We load and skip the header + loaded_csv = open(orig_tsv_file, "r").readlines()[1:] + nb_samples = len(loaded_csv) + + msg = "Preparing CSV files for %s samples ..." % (str(nb_samples)) + logger.info(msg) + + # Adding some Prints + msg = "Creating csv lists in %s ..." % (csv_file) + logger.info(msg) + + # Process and write lines + total_duration = 0.0 + + line_processor = functools.partial( + process_line, + data_folder=data_folder, + language=language, + accented_letters=accented_letters, + ) + + # Stream into a .tmp file, and rename it to the real path at the end. + csv_file_tmp = csv_file + ".tmp" + + with open(csv_file_tmp, mode="w", encoding="utf-8") as csv_f: + csv_writer = csv.writer( + csv_f, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL + ) + + csv_writer.writerow(["ID", "duration", "wav", "spk_id", "wrd"]) + for line in tqdm(loaded_csv) : + + row = line_processor(line) + if row is not None : + total_duration += row.duration + csv_writer.writerow( + [ + row.snt_id, + str(row.duration), + row.mp3_path, + row.spk_id, + row.words, + ] + ) + + os.replace(csv_file_tmp, csv_file) + + # Final prints + msg = "%s successfully created!" % (csv_file) + logger.info(msg) + msg = "Number of samples: %s " % (str(len(loaded_csv))) + logger.info(msg) + msg = "Total duration: %s Hours" % (str(round(total_duration / 3600, 2))) + logger.info(msg) + + +def language_specific_preprocess(language, words): + # !! Language specific cleaning !! + # Important: feel free to specify the text normalization + # corresponding to your alphabet. + + if language in ["en", "fr", "it", "rw"]: + words = re.sub( + "[^’'A-Za-z0-9À-ÖØ-öø-ÿЀ-ӿéæœâçèàûî]+", " ", words + ).upper() + + if language == "de": + # this replacement helps preserve the case of ß + # (and helps retain solitary occurrences of SS) + # since python's upper() converts ß to SS. + words = words.replace("ß", "0000ß0000") + words = re.sub("[^’'A-Za-z0-9öÖäÄüÜß]+", " ", words).upper() + words = words.replace("'", " ") + words = words.replace("’", " ") + words = words.replace( + "0000SS0000", "ß" + ) # replace 0000SS0000 back to ß as its initial presence in the corpus + + if language == "fr": + # Replace J'y D'hui etc by J_ D_hui + words = words.replace("'", " ") + words = words.replace("’", " ") + + elif language == "ar": + HAMZA = "\u0621" + ALEF_MADDA = "\u0622" + ALEF_HAMZA_ABOVE = "\u0623" + letters = ( + "ابتةثجحخدذرزژشسصضطظعغفقكلمنهويىءآأؤإئ" + + HAMZA + + ALEF_MADDA + + ALEF_HAMZA_ABOVE + ) + words = re.sub("[^" + letters + " ]+", "", words).upper() + elif language == "fa": + HAMZA = "\u0621" + ALEF_MADDA = "\u0622" + ALEF_HAMZA_ABOVE = "\u0623" + letters = ( + "ابپتةثجحخچدذرزژسشصضطظعغفقگکلمنهویىءآأؤإئ" + + HAMZA + + ALEF_MADDA + + ALEF_HAMZA_ABOVE + ) + words = re.sub("[^" + letters + " ]+", "", words).upper() + elif language == "ga-IE": + # Irish lower() is complicated, but upper() is nondeterministic, so use lowercase + def pfxuc(a): + return len(a) >= 2 and a[0] in "tn" and a[1] in "AEIOUÁÉÍÓÚ" + + def galc(w): + return w.lower() if not pfxuc(w) else w[0] + "-" + w[1:].lower() + + words = re.sub("[^-A-Za-z'ÁÉÍÓÚáéíóú]+", " ", words) + words = " ".join(map(galc, words.split(" "))) + elif language == "es": + # Fix the following error in dataset large: + # KeyError: 'The item En noviembre lanzaron Queen Elizabeth , coproducida por Foreign Noi$e . requires replacements which were not supplied.' + words = words.replace("$", "s") + return words + + +def check_commonvoice_folders(data_folder): + """ + Check if the data folder actually contains the Common Voice dataset. + If not, raises an error. + Returns + ------- + None + Raises + ------ + FileNotFoundError + If data folder doesn't contain Common Voice dataset. + """ + files_str = "/clips" + # Checking clips + if not os.path.exists(data_folder + files_str): + err_msg = ( + "the folder %s does not exist (it is expected in " + "the Common Voice dataset)" % (data_folder + files_str) + ) + raise FileNotFoundError(err_msg) + + +def unicode_normalisation(text): + return str(text) + + +def strip_accents(text): + text = ( + unicodedata.normalize("NFD", text) + .encode("ascii", "ignore") + .decode("utf-8") + ) + return str(text) diff --git a/EnglishCV/results/final_cs/hyperparams.yaml b/EnglishCV/results/final_cs/hyperparams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51bb1fca03c6b555db1d3d2bae85bf67c03e7d75 --- /dev/null +++ b/EnglishCV/results/final_cs/hyperparams.yaml @@ -0,0 +1,144 @@ +# Generated 2023-09-08 from: +# /gpfsssd/scratch/rech/nou/uzn19yk/switched_data/stac.yaml +# yamllint disable +# Generated 2023-08-03 from: +# /home/salah/new_tunisian_model/hparams/train_tunisian_withwavlm.yaml +# yamllint disable +# ################################ +# Model: wav2vec2 + DNN + CTC +# Augmentation: SpecAugment +# Authors: Titouan Parcollet 2021 +# ################################ + +seed: 1994 +__set_seed: !!python/object/apply:torch.manual_seed [1234] +output_folder: results/non_semi_final_stac +wer_file: results/non_semi_final_stac/wer.txt +save_folder: results/non_semi_final_stac/save +train_log: results/non_semi_final_stac/train_log.txt + + + +# Data files +data_folder: junk # e.g, /localscratch/cv-corpus-5.1-2020-06-22/fr +train_tsv_file: junk/train.tsv # Standard CommonVoice .tsv files +dev_tsv_file: junk/dev.tsv # Standard CommonVoice .tsv files +test_tsv_file: junk/test.tsv # Standard CommonVoice .tsv files +accented_letters: true + +csv_folder: /gpfsscratch/rech/nou/uzn19yk/switched_data/extended_clean/ +train_csv: /gpfsscratch/rech/nou/uzn19yk/switched_data/extended_clean//train.csv +valid_csv: /gpfsscratch/rech/nou/uzn19yk/switched_data/extended_clean//dev.csv +test_csv: +- all_tests/cs_test.csv +- all_tests/stac_test.csv + +# We remove utterance slonger than 10s in the train/dev/test sets as +# longer sentences certainly correspond to "open microphones". +avoid_if_longer_than: 13.0 +avoid_if_shorter_than: 0.5 + +# Training parameters +number_of_epochs: 20 +lr: 0.0002 +lr_weights: 0.01 +sorting: ascending +auto_mix_prec: false +sample_rate: 16000 +language_modelling: true +ngram_lm_path: + /gpfsstore/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/arpas/pluslanguages_everything.arpa + +# With data_parallel batch_size is split into N jobs +# With DDP batch_size is multiplied by N jobs +# Must be 3 per GPU to fit 32GB of VRAM +batch_size: 3 +test_batch_size: 4 + +# Dataloader options +dataloader_options: + batch_size: 3 + num_workers: 6 + +test_dataloader_options: + batch_size: 4 + num_workers: 6 + +# Model parameters +activation: !name:torch.nn.Sigmoid +dnn_layers: 1 +dnn_neurons: 768 +freeze_encoder: true + +# Outputs +output_neurons: 76 # BPE size, index(blank/eos/bos) = 0 + +# Functions and classes +# +epoch_counter: &id006 !new:speechbrain.utils.epoch_loop.EpochCounter + limit: 20 + +encoder_dim: 3217 +enc: &id001 !new:speechbrain.nnet.RNN.LSTM + input_shape: [null, null, 3217] + num_layers: 2 + bidirectional: true + dropout: 0.2 + hidden_size: 1024 + +ctc_lin: &id002 !new:speechbrain.nnet.linear.Linear + + input_size: 2048 + n_neurons: 76 + +log_softmax: !new:speechbrain.nnet.activations.Softmax + apply_log: true + +ctc_cost: !name:speechbrain.nnet.losses.ctc_loss + blank_index: 0 + +modules: + enc: *id001 + ctc_lin: *id002 +model: &id003 !new:torch.nn.ModuleList +- [*id001, *id002] +model_opt_class: !name:torch.optim.Adam + lr: 0.0002 + +weights_opt_class: !name:torch.optim.Adam + lr: 0.01 + +lr_annealing_model: &id004 !new:speechbrain.nnet.schedulers.NewBobScheduler + initial_value: 0.0002 + improvement_threshold: 0.0025 + annealing_factor: 0.8 + patient: 0 + +lr_annealing_weights: &id005 !new:speechbrain.nnet.schedulers.NewBobScheduler + initial_value: 0.01 + improvement_threshold: 0.0025 + annealing_factor: 0.9 + patient: 0 + +label_encoder: &id007 !new:speechbrain.dataio.encoder.CTCTextEncoder + + +checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer + checkpoints_dir: results/non_semi_final_stac/save + recoverables: + model: *id003 + scheduler_model: *id004 + scheduler_encoder: *id005 + counter: *id006 + tokenizer: *id007 +blank_index: 0 +unk_index: 1 + + +train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger + save_file: results/non_semi_final_stac/train_log.txt + +error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats + +cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats + split_tokens: true diff --git a/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/CKPT.yaml b/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/CKPT.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b888496eecad2bd5d629fcf8869b08a724ec8e1 --- /dev/null +++ b/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/CKPT.yaml @@ -0,0 +1,4 @@ +# yamllint disable +WER: 51.292116454039906 +end-of-epoch: true +unixtime: 1694130018.9642384 diff --git a/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/brain.ckpt b/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/brain.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..257c53cab0548d9077f7ae97668d8fef1b0b1b3d --- /dev/null +++ b/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/brain.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5c026fe6fa51700406bd476e131950c797b0b3bacb3daae0854e85689bb4cf9 +size 50 diff --git a/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/counter.ckpt b/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/counter.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7dd349a9f2817e0adc9efe1492b7aa8992a52421 --- /dev/null +++ b/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/counter.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5ca38f748a1d6eaf726b8a42fb575c3c71f1864a8143301782de13da2d9202b +size 2 diff --git a/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/dataloader-TRAIN.ckpt b/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/dataloader-TRAIN.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b968e60328486cfa5ae986406f956ac0290aff90 --- /dev/null +++ b/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/dataloader-TRAIN.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7e1edcac43af8cea1439d222314af06354ae31da6a3d90b8cc6bcebc5c8e397 +size 4 diff --git a/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/model.ckpt b/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/model.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b66a0f410476fe28db087866a0aad965d85a5c6a --- /dev/null +++ b/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/model.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da683a8efa5709a06af9b258452c243da841780a0a7942c196c472a3e21e5010 +size 240389017 diff --git a/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/modelopt.ckpt b/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/modelopt.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c93f9f1b21d6c991f246a57a1800eadf1d140598 --- /dev/null +++ b/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/modelopt.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:416feb314443cf839f4425fc382e555dec90e3dea26fa52b75e4ac1b702c5078 +size 480787579 diff --git a/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/scheduler_encoder.ckpt b/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/scheduler_encoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..cd8641ac95a203186c7cc52e006fb11424905b88 --- /dev/null +++ b/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/scheduler_encoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e2efd50f0cf28a080e2625fdd8a1852c669841537cdc0a57fce60bc6c1eec11 +size 515 diff --git a/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/scheduler_model.ckpt b/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/scheduler_model.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..397681cc010e291f081703cf6f57ba010ec2c472 --- /dev/null +++ b/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/scheduler_model.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cec54cc9236fa7aa965b397675d24299b973675cc0c6345de038fc70e51629ab +size 703 diff --git a/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/tokenizer.ckpt b/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/tokenizer.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..74e4787a8d8b8d483c219099cf3fc729d86bc3de --- /dev/null +++ b/EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/tokenizer.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21080a140faeb4f39fad188aaf081914ec782be9c4320d6415e8822709e18017 +size 39 diff --git a/EnglishCV/results/final_cs/save/label_encoder.txt b/EnglishCV/results/final_cs/save/label_encoder.txt new file mode 100644 index 0000000000000000000000000000000000000000..a3a231ac6f609d8bf6376a0e1c83c079c5d19fe7 --- /dev/null +++ b/EnglishCV/results/final_cs/save/label_encoder.txt @@ -0,0 +1,80 @@ +'و' => 74 +'ي' => 1 +'ن' => 2 +' ' => 3 +'م' => 4 +'ش' => 5 +'ل' => 6 +'س' => 7 +'ت' => 8 +'ا' => 9 +'د' => 10 +'ر' => 11 +'ى' => 12 +'ب' => 13 +'ح' => 14 +'ط' => 15 +'ع' => 16 +'ك' => 17 +'ف' => 18 +'ق' => 19 +'ذ' => 20 +'ث' => 21 +'ج' => 22 +'ة' => 23 +'غ' => 24 +'o' => 25 +'k' => 26 +'b' => 27 +'n' => 28 +'خ' => 29 +'ه' => 30 +'v' => 31 +'i' => 32 +'l' => 33 +'à' => 34 +'ص' => 35 +'ض' => 36 +'a' => 37 +'u' => 38 +'t' => 39 +'m' => 40 +'q' => 41 +'e' => 42 +'d' => 43 +'c' => 44 +'p' => 45 +'r' => 46 +'أ' => 47 +'إ' => 48 +'s' => 49 +'j' => 50 +'ز' => 51 +'ء' => 52 +'h' => 53 +'f' => 54 +'آ' => 55 +'ئ' => 56 +'ؤ' => 57 +'ظ' => 58 +'y' => 59 +'é' => 60 +"'" => 61 +'z' => 62 +'x' => 63 +'w' => 64 +'g' => 65 +'è' => 66 +'û' => 67 +'ç' => 68 +'ê' => 69 +'ô' => 70 +'ù' => 71 +'î' => 72 +'â' => 73 +'' => 0 +1 => 75 +================ +'starting_index' => 0 +'unk_label' => 1 +'blank_label' => '' diff --git a/EnglishCV/results/final_cs/train_mixer.py b/EnglishCV/results/final_cs/train_mixer.py new file mode 100644 index 0000000000000000000000000000000000000000..4846ea6554104069ea6a2792607d883efb85c482 --- /dev/null +++ b/EnglishCV/results/final_cs/train_mixer.py @@ -0,0 +1,756 @@ +# -*- coding: utf-8 -*- + +import os +import sys +import torch +import logging +import speechbrain as sb +from speechbrain.utils.distributed import run_on_main +from hyperpyyaml import load_hyperpyyaml +from pathlib import Path +import torchaudio.transforms as T +from cv_train import ASRCV +import torchaudio +import numpy as np +import kenlm +from pyctcdecode import build_ctcdecoder +import re + +# Commented out IPython magic to ensure Python compatibility. +# %cd /content/drive/MyDrive/tunisian_corpora/tunisian_without_wavlm +#hparams_file, run_opts, overrides = sb.parse_arguments(["/gpfsstore/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/hparams/train_semi.yaml"]) +hparams_file, run_opts, overrides = sb.parse_arguments(["semi_supervised_test_tunisian.yaml"]) + +# If distributed_launch=True then +# create ddp_group with the right communication protocol +sb.utils.distributed.ddp_init_group(run_opts) + +with open(hparams_file) as fin: + hparams = load_hyperpyyaml(fin, overrides) + +# Create experiment directory +sb.create_experiment_directory( + experiment_directory=hparams["output_folder"], + hyperparams_to_save=hparams_file, + overrides=overrides, +) +# Dataset prep (parsing Librispeech) + +def dataio_prepare(hparams): + """This function prepares the datasets to be used in the brain class. + It also defines the data processing pipeline through user-defined functions.""" + + # 1. Define datasets + data_folder = hparams["data_folder"] + + train_data = sb.dataio.dataset.DynamicItemDataset.from_csv( + csv_path=hparams["train_csv"], replacements={"data_root": data_folder}, + ) + + if hparams["sorting"] == "ascending": + # we sort training data to speed up training and get better results. + train_data = train_data.filtered_sorted( + sort_key="duration", + key_max_value={"duration": hparams["avoid_if_longer_than"]}, + ) + # when sorting do not shuffle in dataloader ! otherwise is pointless + hparams["dataloader_options"]["shuffle"] = False + + elif hparams["sorting"] == "descending": + train_data = train_data.filtered_sorted( + sort_key="duration", + reverse=True, + key_max_value={"duration": hparams["avoid_if_longer_than"]}, + ) + # when sorting do not shuffle in dataloader ! otherwise is pointless + hparams["dataloader_options"]["shuffle"] = False + + elif hparams["sorting"] == "random": + pass + + else: + raise NotImplementedError( + "sorting must be random, ascending or descending" + ) + + valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv( + csv_path=hparams["valid_csv"], replacements={"data_root": data_folder}, + ) + # We also sort the validation data so it is faster to validate + valid_data = valid_data.filtered_sorted(sort_key="duration") + test_datasets = {} + for csv_file in hparams["test_csv"]: + name = Path(csv_file).stem + test_datasets[name] = sb.dataio.dataset.DynamicItemDataset.from_csv( + csv_path=csv_file, replacements={"data_root": data_folder} + ) + test_datasets[name] = test_datasets[name].filtered_sorted( + sort_key="duration" + ) + + datasets = [train_data, valid_data] + [i for k, i in test_datasets.items()] + + + # 2. Define audio pipeline: + @sb.utils.data_pipeline.takes("wav") + @sb.utils.data_pipeline.provides("sig") + def audio_pipeline(wav): + info = torchaudio.info(wav) + sig = sb.dataio.dataio.read_audio(wav) + if len(sig.shape)>1 : + sig = torch.mean(sig, dim=1) + resampled = torchaudio.transforms.Resample( + info.sample_rate, hparams["sample_rate"], + )(sig) + return resampled + + sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline) + label_encoder = sb.dataio.encoder.CTCTextEncoder() + + # 3. Define text pipeline: + @sb.utils.data_pipeline.takes("wrd") + @sb.utils.data_pipeline.provides( + "wrd", "char_list", "tokens_list", "tokens" + ) + def text_pipeline(wrd): + yield wrd + char_list = list(wrd) + yield char_list + tokens_list = label_encoder.encode_sequence(char_list) + yield tokens_list + tokens = torch.LongTensor(tokens_list) + yield tokens + + sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline) + lab_enc_file = os.path.join(hparams["save_folder"], "label_encoder.txt") + special_labels = { + "blank_label": hparams["blank_index"], + "unk_label": hparams["unk_index"] + } + label_encoder.load_or_create( + path=lab_enc_file, + from_didatasets=[train_data], + output_key="char_list", + special_labels=special_labels, + sequence_input=True, + ) + + # 4. Set output: + sb.dataio.dataset.set_output_keys( + datasets, ["id", "sig", "wrd", "char_list", "tokens"], + ) + return train_data, valid_data,test_datasets, label_encoder + +class ASR(sb.core.Brain): + def compute_forward(self, batch, stage): + """Forward computations from the waveform batches to the output probabilities.""" + + batch = batch.to(self.device) + wavs, wav_lens = batch.sig + wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device) + + if stage == sb.Stage.TRAIN: + if hasattr(self.hparams, "augmentation"): + wavs = self.hparams.augmentation(wavs, wav_lens) + + # Forward pass + feats = self.modules.wav2vec2(wavs, wav_lens) + x = self.modules.enc(feats) + logits = self.modules.ctc_lin(x) + p_ctc = self.hparams.log_softmax(logits) + + return p_ctc, wav_lens + + def custom_encode(self,wavs,wav_lens) : + wavs = wavs.to(self.device) + if(wav_lens is not None): wav_lens.to(self.device) + + feats = self.modules.wav2vec2(wavs, wav_lens) + x = self.modules.enc(feats) + logits = self.modules.ctc_lin(x) + p_ctc = self.hparams.log_softmax(logits) + + return feats,p_ctc + + + + def compute_objectives(self, predictions, batch, stage): + """Computes the loss (CTC) given predictions and targets.""" + + p_ctc, wav_lens = predictions + + ids = batch.id + tokens, tokens_lens = batch.tokens + + loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens) + + if stage != sb.Stage.TRAIN: + predicted_tokens = sb.decoders.ctc_greedy_decode( + p_ctc, wav_lens, blank_id=self.hparams.blank_index + ) + # Decode token terms to words + if self.hparams.use_language_modelling: + predicted_words = [] + for logs in p_ctc: + text = decoder.decode(logs.detach().cpu().numpy()) + predicted_words.append(text.split(" ")) + else: + predicted_words = [ + "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ") + for utt_seq in predicted_tokens + ] + # Convert indices to words + target_words = [wrd.split(" ") for wrd in batch.wrd] + + self.wer_metric.append(ids, predicted_words, target_words) + self.cer_metric.append(ids, predicted_words, target_words) + + return loss + + def fit_batch(self, batch): + """Train the parameters given a single batch in input""" + should_step = self.step % self.grad_accumulation_factor == 0 + # Managing automatic mixed precision + # TOFIX: CTC fine-tuning currently is unstable + # This is certainly due to CTC being done in fp16 instead of fp32 + if self.auto_mix_prec: + with torch.cuda.amp.autocast(): + with self.no_sync(): + outputs = self.compute_forward(batch, sb.Stage.TRAIN) + loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN) + with self.no_sync(not should_step): + self.scaler.scale( + loss / self.grad_accumulation_factor + ).backward() + if should_step: + + if not self.hparams.wav2vec2.freeze: + self.scaler.unscale_(self.wav2vec_optimizer) + self.scaler.unscale_(self.model_optimizer) + if self.check_gradients(loss): + if not self.hparams.wav2vec2.freeze: + if self.optimizer_step >= self.hparams.warmup_steps: + self.scaler.step(self.wav2vec_optimizer) + self.scaler.step(self.model_optimizer) + self.scaler.update() + self.zero_grad() + self.optimizer_step += 1 + else: + # This is mandatory because HF models have a weird behavior with DDP + # on the forward pass + with self.no_sync(): + outputs = self.compute_forward(batch, sb.Stage.TRAIN) + + loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN) + + with self.no_sync(not should_step): + (loss / self.grad_accumulation_factor).backward() + if should_step: + if self.check_gradients(loss): + if not self.hparams.wav2vec2.freeze: + if self.optimizer_step >= self.hparams.warmup_steps: + self.wav2vec_optimizer.step() + self.model_optimizer.step() + self.zero_grad() + self.optimizer_step += 1 + + self.on_fit_batch_end(batch, outputs, loss, should_step) + return loss.detach().cpu() + + def evaluate_batch(self, batch, stage): + """Computations needed for validation/test batches""" + predictions = self.compute_forward(batch, stage=stage) + with torch.no_grad(): + loss = self.compute_objectives(predictions, batch, stage=stage) + return loss.detach() + + def on_stage_start(self, stage, epoch): + """Gets called at the beginning of each epoch""" + if stage != sb.Stage.TRAIN: + self.cer_metric = self.hparams.cer_computer() + self.wer_metric = self.hparams.error_rate_computer() + + def on_stage_end(self, stage, stage_loss, epoch): + """Gets called at the end of an epoch.""" + # Compute/store important stats + stage_stats = {"loss": stage_loss} + if stage == sb.Stage.TRAIN: + self.train_stats = stage_stats + else: + stage_stats["CER"] = self.cer_metric.summarize("error_rate") + stage_stats["WER"] = self.wer_metric.summarize("error_rate") + + # Perform end-of-iteration things, like annealing, logging, etc. + if stage == sb.Stage.VALID: + old_lr_model, new_lr_model = self.hparams.lr_annealing_model( + stage_stats["loss"] + ) + old_lr_wav2vec, new_lr_wav2vec = self.hparams.lr_annealing_wav2vec( + stage_stats["loss"] + ) + sb.nnet.schedulers.update_learning_rate( + self.model_optimizer, new_lr_model + ) + if not self.hparams.wav2vec2.freeze: + sb.nnet.schedulers.update_learning_rate( + self.wav2vec_optimizer, new_lr_wav2vec + ) + self.hparams.train_logger.log_stats( + stats_meta={ + "epoch": epoch, + "lr_model": old_lr_model, + "lr_wav2vec": old_lr_wav2vec, + }, + train_stats=self.train_stats, + valid_stats=stage_stats, + ) + self.checkpointer.save_and_keep_only( + meta={"WER": stage_stats["WER"]}, min_keys=["WER"], + ) + elif stage == sb.Stage.TEST: + self.hparams.train_logger.log_stats( + stats_meta={"Epoch loaded": self.hparams.epoch_counter.current}, + test_stats=stage_stats, + ) + with open(self.hparams.wer_file, "w") as w: + self.wer_metric.write_stats(w) + + def init_optimizers(self): + "Initializes the wav2vec2 optimizer and model optimizer" + + # If the wav2vec encoder is unfrozen, we create the optimizer + if not self.hparams.wav2vec2.freeze: + self.wav2vec_optimizer = self.hparams.wav2vec_opt_class( + self.modules.wav2vec2.parameters() + ) + if self.checkpointer is not None: + self.checkpointer.add_recoverable( + "wav2vec_opt", self.wav2vec_optimizer + ) + + self.model_optimizer = self.hparams.model_opt_class( + self.hparams.model.parameters() + ) + + if self.checkpointer is not None: + self.checkpointer.add_recoverable("modelopt", self.model_optimizer) + + def zero_grad(self, set_to_none=False): + if not self.hparams.wav2vec2.freeze: + self.wav2vec_optimizer.zero_grad(set_to_none) + self.model_optimizer.zero_grad(set_to_none) + + +""" +label_encoder = sb.dataio.encoder.CTCTextEncoder() + +train_data, valid_data, test_datasets, label_encoder = dataio_prepare( + hparams + ) + + +# We dynamicaly add the tokenizer to our brain class. +# NB: This tokenizer corresponds to the one used for the LM!! +""" +from speechbrain.pretrained import EncoderASR,EncoderDecoderASR +french_asr_model = EncoderASR.from_hparams(source="speechbrain/asr-wav2vec2-commonvoice-fr", savedir="pretrained_models/asr-wav2vec2-commonvoice-fr").cuda() +#french_asr_model = "r" + +cvhparams_file, cvrun_opts, cvoverrides = sb.parse_arguments(["en_cv.yaml"]) +with open(cvhparams_file) as cvfin: + cvhparams = load_hyperpyyaml(cvfin, cvoverrides) +english_asr_model = ASRCV( + modules=cvhparams["modules"], + hparams=cvhparams, + run_opts=cvrun_opts, + checkpointer=cvhparams["checkpointer"], + ) +english_asr_model.checkpointer.recover_if_possible() +asr_brain = ASR( + modules=hparams["modules"], + hparams=hparams, + run_opts=run_opts, + checkpointer=hparams["checkpointer"], +) +asr_brain.checkpointer.recover_if_possible() +asr_brain.modules.eval() +english_asr_model.modules.eval() +french_asr_model.mods.eval() +""" +asr_brain.tokenizer = label_encoder + +# Testing +real = True +if real : + for k in test_datasets.keys(): # keys are test_clean, test_other etc + asr_brain.hparams.wer_file = os.path.join( + hparams["output_folder"], "wer_{}.txt".format(k) + ) + asr_brain.evaluate( + test_datasets[k], test_loader_kwargs=hparams["dataloader_options"] + ) +""" + +""" +from torch.nn.utils.rnn import pad_sequence +def load_paths(wavs_path): + waveforms = [] + for path in wavs_path : + waveform, _ = torchaudio.load(path) + waveforms.append(waveform.squeeze(0)) + # normalize array length to the bigger arrays by pading with 0's + padded_arrays = pad_sequence(waveforms, batch_first=True) + return torch.tensor(padded_arrays) + +waveform = load_paths(["/content/drive/MyDrive/tunisian_corpora/tunisian_without_wavlm/samples/Salah10.wav","/content/drive/MyDrive/tunisian_corpora/tunisian_without_wavlm/samples/Salah10.wav"]) +embeddings, posteriogram = asr_brain.custom_encode(waveform,None) +print(embeddings.shape) +print(posteriogram.shape) +""" + +from speechbrain.pretrained import EncoderASR,EncoderDecoderASR +import torchaudio +import speechbrain as sb +import torch +from torch.nn.utils.rnn import pad_sequence +import torch +import speechbrain as sb +import numpy as np +import torch.optim as optim +import torch.nn as nn + +# Commented out IPython magic to ensure Python compatibility. +# %ls + +#UTILS FUNCTIOJNS +def get_size_dimensions(arr): + size_dimensions = [] + while isinstance(arr, list): + size_dimensions.append(len(arr)) + arr = arr[0] + return size_dimensions + +def scale_array(batch,n): + scaled_batch = [] + + for array in batch: + if(n < len(array)): raise ValueError("Cannot scale Array down") + + repeat = round(n/len(array))+1 + scaled_length_array= [] + + for i in array: + for j in range(repeat) : + if(len(scaled_length_array) == n): break + scaled_length_array.append(i) + + scaled_batch.append(scaled_length_array) + + return torch.tensor(scaled_batch) + + +def load_paths(wavs_path): + waveforms = [] + for path in wavs_path : + waveform, _ = torchaudio.load(path) + waveforms.append(waveform.squeeze(0)) + # normalize array length to the bigger arrays by pading with 0's + padded_arrays = pad_sequence(waveforms, batch_first=True) + return torch.tensor(padded_arrays) + + + +def word_to_vec(input_string): + mapping= {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5, 'f': 6, 'g': 7, 'h': 8, 'i': 9, 'j': 10, 'k': 11, 'l': 12, 'm': 13, 'n': 14, 'o': 15, 'p': 16, 'q': 17, 'r': 18, 's': 19, 't': 20, 'u': 21, 'v': 22, 'w': 23, 'x': 24, 'y': 25, 'z': 26, 'ا': 27, 'ب': 28, 'ت': 29, 'ث': 30, 'ج': 31, 'ح': 32, 'خ': 33, 'د': 34, 'ذ': 35, 'ر': 36, 'ز': 37, 'س': 38, 'ش': 39, 'ص': 40, 'ض': 41, 'ط': 42, 'ظ': 43, 'ع': 44, 'غ': 45, 'ف': 46, 'ق': 47, 'ك': 48, 'ل': 49, 'م': 50, 'ن': 51, 'ه': 52, 'و': 53, 'ي': 54,' ':55} + + numbers = [mapping[word] for word in input_string if word in mapping] + return numbers + +device = 'cuda' +verbose = 0 +#FLOW LEVEL FUNCTIONS +def merge_strategy(embeddings1, embeddings2, embeddings3,post1, post2,post3): + + + post1 = post1.to(device) + post2 = post2.to(device) + post3 = post3.to(device) + embeddings1 = embeddings1.to(device) + embeddings2 = embeddings2.to(device) + embeddings3 = embeddings3.to(device) + + posteriograms_merged = torch.cat((post1,post2,post3),dim=2) + embeddings_merged = torch.cat((embeddings1,embeddings2,embeddings3),dim=2) + + if(verbose !=0): + print('MERGED POST ',posteriograms_merged.shape) + print('MERGED emb ',embeddings_merged.shape) + + return torch.cat((posteriograms_merged,embeddings_merged),dim=2).to(device) + +def decode(model,wavs,wav_lens): + + with torch.no_grad(): + wav_lens = wav_lens.to(model.device) + encoder_out = model.encode_batch(wavs, wav_lens) + predictions = model.decoding_function(encoder_out, wav_lens) + return predictions + +def middle_layer(batch, lens): + + tn_embeddings, tn_posteriogram = asr_brain.custom_encode(batch,None) + + fr_embeddings = french_asr_model.mods.encoder.wav2vec2(batch) + fr_posteriogram =french_asr_model.encode_batch(batch,lens) + en_embeddings = english_asr_model.modules.wav2vec2(batch, lens) + x = english_asr_model.modules.enc(en_embeddings) + en_posteriogram = english_asr_model.modules.ctc_lin(x) + #scores, en_posteriogram = english_asr_model.mods.decoder(en_embeddings ,lens) + if(verbose !=0): + print('[EMBEDDINGS] FR:',fr_embeddings.shape, "EN:",en_embeddings.shape, "TN:", tn_embeddings.shape) + print('[POSTERIOGRAM] FR:',fr_posteriogram.shape, "EN:",en_posteriogram.shape,"TN:",tn_posteriogram.shape) + + + bilangual_sample = merge_strategy(fr_embeddings,en_embeddings,tn_embeddings,fr_posteriogram,en_posteriogram,tn_posteriogram) + return bilangual_sample + +class Mixer(sb.core.Brain): + + def compute_forward(self, batch, stage): + """Forward computations from the waveform batches to the output probabilities.""" + wavs, wav_lens = batch.sig + wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device) + + if stage == sb.Stage.TRAIN: + if hasattr(self.hparams, "augmentation"): + wavs = self.hparams.augmentation(wavs, wav_lens) + + multi_langual_feats = middle_layer(wavs, wav_lens) + multi_langual_feats= multi_langual_feats.to(device) + feats, _ = self.modules.enc(multi_langual_feats) + logits = self.modules.ctc_lin(feats) + p_ctc = self.hparams.log_softmax(logits) + + if stage!= sb.Stage.TRAIN: + p_tokens = sb.decoders.ctc_greedy_decode( + p_ctc, wav_lens, blank_id=self.hparams.blank_index + ) + else : + p_tokens = None + return p_ctc, wav_lens, p_tokens + + def compute_objectives(self, predictions, batch, stage): + """Computes the loss (CTC) given predictions and targets.""" + + p_ctc, wav_lens , predicted_tokens= predictions + + ids = batch.id + tokens, tokens_lens = batch.tokens + + loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens) + + + if stage == sb.Stage.VALID: + predicted_words = [ + "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ") + for utt_seq in predicted_tokens + ] + target_words = [wrd.split(" ") for wrd in batch.wrd] + self.wer_metric.append(ids, predicted_words, target_words) + self.cer_metric.append(ids, predicted_words, target_words) + if stage ==sb.Stage.TEST : + if self.hparams.language_modelling: + predicted_words = [] + for logs in p_ctc: + text = decoder.decode(logs.detach().cpu().numpy()) + predicted_words.append(text.split(" ")) + else : + predicted_words = [ + "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ") + for utt_seq in predicted_tokens + ] + + target_words = [wrd.split(" ") for wrd in batch.wrd] + self.wer_metric.append(ids, predicted_words, target_words) + self.cer_metric.append(ids, predicted_words, target_words) + + return loss + + def fit_batch(self, batch): + """Train the parameters given a single batch in input""" + should_step = self.step % self.grad_accumulation_factor == 0 + # Managing automatic mixed precision + # TOFIX: CTC fine-tuning currently is unstable + # This is certainly due to CTC being done in fp16 instead of fp32 + if self.auto_mix_prec: + with torch.cuda.amp.autocast(): + with self.no_sync(): + outputs = self.compute_forward(batch, sb.Stage.TRAIN) + loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN) + with self.no_sync(not should_step): + self.scaler.scale( + loss / self.grad_accumulation_factor + ).backward() + if should_step: + + + self.scaler.unscale_(self.model_optimizer) + if self.check_gradients(loss): + self.scaler.step(self.model_optimizer) + self.scaler.update() + self.zero_grad() + self.optimizer_step += 1 + else: + # This is mandatory because HF models have a weird behavior with DDP + # on the forward pass + with self.no_sync(): + outputs = self.compute_forward(batch, sb.Stage.TRAIN) + + loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN) + + with self.no_sync(not should_step): + (loss / self.grad_accumulation_factor).backward() + if should_step: + if self.check_gradients(loss): + self.model_optimizer.step() + self.zero_grad() + self.optimizer_step += 1 + + self.on_fit_batch_end(batch, outputs, loss, should_step) + return loss.detach().cpu() + + def evaluate_batch(self, batch, stage): + """Computations needed for validation/test batches""" + predictions = self.compute_forward(batch, stage=stage) + with torch.no_grad(): + loss = self.compute_objectives(predictions, batch, stage=stage) + return loss.detach() + + def on_stage_start(self, stage, epoch): + """Gets called at the beginning of each epoch""" + if stage != sb.Stage.TRAIN: + self.cer_metric = self.hparams.cer_computer() + self.wer_metric = self.hparams.error_rate_computer() + + def on_stage_end(self, stage, stage_loss, epoch): + """Gets called at the end of an epoch.""" + # Compute/store important stats + stage_stats = {"loss": stage_loss} + if stage == sb.Stage.TRAIN: + self.train_stats = stage_stats + else: + stage_stats["CER"] = self.cer_metric.summarize("error_rate") + stage_stats["WER"] = self.wer_metric.summarize("error_rate") + + # Perform end-of-iteration things, like annealing, logging, etc. + if stage == sb.Stage.VALID: + old_lr_model, new_lr_model = self.hparams.lr_annealing_model( + stage_stats["loss"] + ) + sb.nnet.schedulers.update_learning_rate( + self.model_optimizer, new_lr_model + ) + self.hparams.train_logger.log_stats( + stats_meta={ + "epoch": epoch, + "lr_model": old_lr_model, + }, + train_stats=self.train_stats, + valid_stats=stage_stats, + ) + self.checkpointer.save_and_keep_only( + meta={"WER": stage_stats["WER"]}, min_keys=["WER"], + ) + elif stage == sb.Stage.TEST: + self.hparams.train_logger.log_stats( + stats_meta={"Epoch loaded": self.hparams.epoch_counter.current}, + test_stats=stage_stats, + ) + with open(self.hparams.wer_file, "w") as w: + self.wer_metric.write_stats(w) + + def init_optimizers(self): + + self.model_optimizer = self.hparams.model_opt_class( + self.hparams.model.parameters() + ) + + if self.checkpointer is not None: + self.checkpointer.add_recoverable("modelopt", self.model_optimizer) + + def zero_grad(self, set_to_none=False): + + self.model_optimizer.zero_grad(set_to_none) + + +hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:]) + +# If distributed_launch=True then +# create ddp_group with the right communication protocol +sb.utils.distributed.ddp_init_group(run_opts) + +with open(hparams_file) as fin: + hparams = load_hyperpyyaml(fin, overrides) + +# Create experiment directory +sb.create_experiment_directory( + experiment_directory=hparams["output_folder"], + hyperparams_to_save=hparams_file, + overrides=overrides, +) +def read_labels_file(labels_file): + with open(labels_file, "r",encoding="utf-8") as lf: + lines = lf.read().splitlines() + division = "===" + numbers = {} + for line in lines : + if division in line : + break + string, number = line.split("=>") + number = int(number) + string = string[1:-2] + numbers[number] = string + return [numbers[x] for x in range(len(numbers))] +train_data, valid_data, test_datasets, label_encoder = dataio_prepare( + hparams + ) + + +labels = read_labels_file(os.path.join(hparams["save_folder"], "label_encoder.txt")) +labels = [""] + labels[1:-1] + ["1"] +if hparams["language_modelling"]: + decoder = build_ctcdecoder( + labels, + kenlm_model_path=hparams["ngram_lm_path"], # either .arpa or .bin file + alpha=0.5, # tuned on a val set + beta=1, # tuned on a val set + ) + + + + +mixer = Mixer( + modules=hparams["modules"], + hparams=hparams, + run_opts=run_opts, + checkpointer=hparams["checkpointer"], +) +mixer.tokenizer = label_encoder + + +mixer.fit( + mixer.hparams.epoch_counter, + train_data, + valid_data, + train_loader_kwargs=hparams["dataloader_options"], + valid_loader_kwargs=hparams["test_dataloader_options"], +) +print(test_datasets.keys()) +for k in test_datasets.keys(): # keys are test_clean, test_other etc + mixer.hparams.wer_file = os.path.join( + hparams["output_folder"], "wer_{}.txt".format(k) + ) + mixer.evaluate( + test_datasets[k], test_loader_kwargs=hparams["test_dataloader_options"] + ) + diff --git a/EnglishCV/results/wav2vec2_ctc_en/1234/hyperparams.yaml b/EnglishCV/results/wav2vec2_ctc_en/1234/hyperparams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5823788972c12278383fd59997e0823ccd9916cc --- /dev/null +++ b/EnglishCV/results/wav2vec2_ctc_en/1234/hyperparams.yaml @@ -0,0 +1,190 @@ +# Generated 2023-09-06 from: +# /gpfsdswork/projects/rech/nou/uzn19yk/final_forke/speechbrain-3/recipes/CommonVoice/ASR/CTC/hparams/train_en_with_wav2vec.yaml +# yamllint disable +# ################################ +# Model: wav2vec2 + DNN + CTC +# Augmentation: SpecAugment +# Authors: Titouan Parcollet 2021 +# ################################ + +# Seed needs to be set at top of yaml, before objects with parameters are made +seed: 1234 +__set_seed: !!python/object/apply:torch.manual_seed [1234] +output_folder: results/wav2vec2_ctc_en/1234 +wer_file: results/wav2vec2_ctc_en/1234/wer.txt +save_folder: results/wav2vec2_ctc_en/1234/save +train_log: results/wav2vec2_ctc_en/1234/train_log.txt + +# URL for the biggest Fairseq english wav2vec2 model. +wav2vec2_hub: facebook/wav2vec2-large-lv60 +wav2vec2_folder: results/wav2vec2_ctc_en/1234/save/wav2vec2_checkpoint + +# Data files +data_folder: + /gpfsscratch/rech/nou/uzn19yk/download/cv-corpus-12.0-2022-12-07/en/cv-corpus-12.0-2022-12-07/en # e.g, /localscratch/cv-corpus-5.1-2020-06-22/fr +train_tsv_file: + /gpfsscratch/rech/nou/uzn19yk/download/cv-corpus-12.0-2022-12-07/en/cv-corpus-12.0-2022-12-07/en/train.tsv # Standard CommonVoice .tsv files +dev_tsv_file: + /gpfsscratch/rech/nou/uzn19yk/download/cv-corpus-12.0-2022-12-07/en/cv-corpus-12.0-2022-12-07/en/dev.tsv # Standard CommonVoice .tsv files +test_tsv_file: + /gpfsscratch/rech/nou/uzn19yk/download/cv-corpus-12.0-2022-12-07/en/cv-corpus-12.0-2022-12-07/en/test.tsv # Standard CommonVoice .tsv files +accented_letters: false +language: en # use 'it' for Italian, 'rw' for Kinyarwanda, 'en' for english +train_csv: results/wav2vec2_ctc_en/1234/save/train.csv +valid_csv: results/wav2vec2_ctc_en/1234/save/dev.csv +test_csv: results/wav2vec2_ctc_en/1234/save/test.csv +skip_prep: false # Skip data preparation + +# We remove utterance slonger than 10s in the train/dev/test sets as +# longer sentences certainly correspond to "open microphones". +avoid_if_longer_than: 10.0 + +# Training parameters +number_of_epochs: 10 +lr: 1.0 +lr_wav2vec: 0.0001 +sorting: ascending +auto_mix_prec: false +sample_rate: 16000 +ckpt_interval_minutes: 30 # save checkpoint every N min + +# With data_parallel batch_size is split into N jobs +# With DDP batch_size is multiplied by N jobs +# Must be 8 per GPU to fit 32GB of VRAM +batch_size: 8 +test_batch_size: 4 + +dataloader_options: + batch_size: 8 + num_workers: 6 +test_dataloader_options: + batch_size: 4 + num_workers: 6 + +# BPE parameters +token_type: char # ["unigram", "bpe", "char"] +character_coverage: 1.0 + +# Model parameters +# activation: !name:torch.nn.LeakyReLU +wav2vec_output_dim: 1024 +dnn_neurons: 1024 +freeze_wav2vec: false +freeze_feature_extractor: true +dropout: 0.15 +warmup_steps: 500 + +# Outputs +output_neurons: 29 # BPE size, index(blank/eos/bos) = 0 + +# Decoding parameters +# Be sure that the bos and eos index match with the BPEs ones +blank_index: 0 +bos_index: 1 +eos_index: 2 + +# +# Functions and classes +# +epoch_counter: &id007 !new:speechbrain.utils.epoch_loop.EpochCounter + + limit: 10 + +augmentation: !new:speechbrain.lobes.augment.TimeDomainSpecAugment + sample_rate: 16000 + speeds: [95, 100, 105] + +enc: &id002 !new:speechbrain.nnet.containers.Sequential + input_shape: [null, null, 1024] + linear1: !name:speechbrain.nnet.linear.Linear + n_neurons: 1024 + bias: true + bn1: !name:speechbrain.nnet.normalization.BatchNorm1d + activation: !new:torch.nn.LeakyReLU + drop: !new:torch.nn.Dropout + p: 0.15 + linear2: !name:speechbrain.nnet.linear.Linear + n_neurons: 1024 + bias: true + bn2: !name:speechbrain.nnet.normalization.BatchNorm1d + activation2: !new:torch.nn.LeakyReLU + drop2: !new:torch.nn.Dropout + p: 0.15 + linear3: !name:speechbrain.nnet.linear.Linear + n_neurons: 1024 + bias: true + bn3: !name:speechbrain.nnet.normalization.BatchNorm1d + activation3: !new:torch.nn.LeakyReLU + +wav2vec2: &id001 !new:speechbrain.lobes.models.huggingface_wav2vec.HuggingFaceWav2Vec2 + source: /gpfsscratch/rech/nou/uzn19yk/wav2vec2-large-lv60/ + output_norm: true + freeze: false + freeze_feature_extractor: true + save_path: results/wav2vec2_ctc_en/1234/save/wav2vec2_checkpoint + +##### +# Uncomment this block if you prefer to use a Fairseq pretrained model instead +# of a HuggingFace one. Here, we provide an URL that is obtained from the +# Fairseq github for the multilingual XLSR. +# +#wav2vec2_url: https://dl.fbaipublicfiles.com/fairseq/wav2vec/xlsr_53_56k.pt +#wav2vec2: !new:speechbrain.lobes.models.fairseq_wav2vec.FairseqWav2Vec2 +# pretrained_path: !ref +# output_norm: True +# freeze: False +# save_path: !ref /wav2vec2_checkpoint/model.pt +##### + +ctc_lin: &id003 !new:speechbrain.nnet.linear.Linear + + input_size: 1024 + n_neurons: 29 + +log_softmax: !new:speechbrain.nnet.activations.Softmax + apply_log: true + +ctc_cost: !name:speechbrain.nnet.losses.ctc_loss + blank_index: 0 + +modules: + wav2vec2: *id001 + enc: *id002 + ctc_lin: *id003 +model: &id004 !new:torch.nn.ModuleList +- [*id002, *id003] +model_opt_class: !name:torch.optim.Adadelta + lr: 1.0 + rho: 0.95 + eps: 1.e-8 + +wav2vec_opt_class: !name:torch.optim.Adam + lr: 0.0001 + +lr_annealing_model: &id005 !new:speechbrain.nnet.schedulers.NewBobScheduler + initial_value: 1.0 + improvement_threshold: 0.0025 + annealing_factor: 0.8 + patient: 0 + +lr_annealing_wav2vec: &id006 !new:speechbrain.nnet.schedulers.NewBobScheduler + initial_value: 0.0001 + improvement_threshold: 0.0025 + annealing_factor: 0.9 + patient: 0 + +checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer + checkpoints_dir: results/wav2vec2_ctc_en/1234/save + recoverables: + wav2vec2: *id001 + model: *id004 + scheduler_model: *id005 + scheduler_wav2vec: *id006 + counter: *id007 +train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger + save_file: results/wav2vec2_ctc_en/1234/train_log.txt + +error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats + +cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats + split_tokens: true diff --git a/EnglishCV/results/wav2vec2_ctc_en/1234/save/29_char.model b/EnglishCV/results/wav2vec2_ctc_en/1234/save/29_char.model new file mode 100644 index 0000000000000000000000000000000000000000..7d36281f2d5f3dcc6228ba6edc53530d7978c495 --- /dev/null +++ b/EnglishCV/results/wav2vec2_ctc_en/1234/save/29_char.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee4214a3ebba9461ca02ca61220a2338412bbf9ef5a5982f2bc40740c4ab91a8 +size 238011 diff --git a/EnglishCV/results/wav2vec2_ctc_en/1234/save/29_char.vocab b/EnglishCV/results/wav2vec2_ctc_en/1234/save/29_char.vocab new file mode 100644 index 0000000000000000000000000000000000000000..86b1ad0d585fd75de25634ff390c288698cd5b5c --- /dev/null +++ b/EnglishCV/results/wav2vec2_ctc_en/1234/save/29_char.vocab @@ -0,0 +1,28 @@ + 0 +▁ -1.786 +E -2.27261 +A -2.6326 +T -2.64317 +I -2.76341 +S -2.81519 +O -2.8189 +N -2.83568 +R -2.87568 +H -3.22802 +L -3.30075 +D -3.43047 +C -3.58554 +U -3.84445 +M -3.84732 +F -4.07023 +P -4.09107 +G -4.16259 +W -4.25412 +Y -4.30147 +B -4.36224 +V -4.71267 +K -5.1744 +X -6.46672 +J -6.5246 +Z -6.95828 +Q -7.12388 diff --git a/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/CKPT.yaml b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/CKPT.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c5173885d9fa6ca2966aef7e4012aecc7cc2b20 --- /dev/null +++ b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/CKPT.yaml @@ -0,0 +1,4 @@ +# yamllint disable +WER: 18.234978071545488 +end-of-epoch: true +unixtime: 1694033791.9455216 diff --git a/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/brain.ckpt b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/brain.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..fe4341d412eaba86adbbf377c9a80df25c6c172d --- /dev/null +++ b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/brain.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06617abf655f8550362b963062fc2a57bd819826ab70e63701676ea09d23618d +size 51 diff --git a/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/counter.ckpt b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/counter.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..252faa6bfe3ec2b69d80ccdd6607473fd4d85313 --- /dev/null +++ b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/counter.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35 +size 1 diff --git a/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/dataloader-TRAIN.ckpt b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/dataloader-TRAIN.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b49ae35814bed04596012d4a635942cdbeef6395 --- /dev/null +++ b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/dataloader-TRAIN.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f21c20a479fcc07663ec4255ad1c85466afb791f514f8f3baa174bd56edca2d4 +size 6 diff --git a/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/model.ckpt b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/model.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..129abdd503ba99ccdedf9a78eadc650fe47a47c5 --- /dev/null +++ b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/model.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:422a1d7a30720e846d2cb79ff510832fe96c1495f559f08fb37bdd118269ea7b +size 12769326 diff --git a/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/modelopt.ckpt b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/modelopt.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..cb5ca677f41201563e7642bebc858eacad4ec11c --- /dev/null +++ b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/modelopt.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65cda77e4403deb7c8cee3052ac687bfc3bf6e68264dcb0e297e8f88bccf0d66 +size 25485359 diff --git a/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/scheduler_model.ckpt b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/scheduler_model.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..309c42c4814b8ea00b72dc4c8967141be1447c96 --- /dev/null +++ b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/scheduler_model.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9c36e38dd81971c68387a9f921cf0d61adad21f5b3f6420b6f3015b0f9d20df +size 511 diff --git a/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/scheduler_wav2vec.ckpt b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/scheduler_wav2vec.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..0b45b969424e34d98d74d62c55ee3a11aa803233 --- /dev/null +++ b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/scheduler_wav2vec.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0293788921aad16c6e904d7ec0b7dba2dd4778fa3b7f1bfa04276b3965599999 +size 515 diff --git a/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/wav2vec2.ckpt b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/wav2vec2.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..56303af367ba7a4d0c577e8eab8b3488c148e15f --- /dev/null +++ b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/wav2vec2.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f7073aa70c88927f11cff4f2ba63a026c8ff6c119837391d84013feb229ad3e +size 1261924189 diff --git a/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/wav2vec_opt.ckpt b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/wav2vec_opt.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..771d857dff7a8da5e1325fa9d2a1528bbf84443d --- /dev/null +++ b/EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/wav2vec_opt.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42691a96ebaba3dd3baf7e2521763db7f79b37a6bde9b0ea9d1adc2cac5bdf5e +size 2490156402 diff --git a/EnglishCV/results/wav2vec2_ctc_en/1234/train_with_wav2vec.py b/EnglishCV/results/wav2vec2_ctc_en/1234/train_with_wav2vec.py new file mode 100644 index 0000000000000000000000000000000000000000..d462abf3d2e8244df5cdb183aa231082d7791a45 --- /dev/null +++ b/EnglishCV/results/wav2vec2_ctc_en/1234/train_with_wav2vec.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 +import sys +import torch +import logging +import speechbrain as sb +import torchaudio +from hyperpyyaml import load_hyperpyyaml +from speechbrain.tokenizers.SentencePiece import SentencePiece +from speechbrain.utils.data_utils import undo_padding +from speechbrain.utils.distributed import run_on_main + +"""Recipe for training a sequence-to-sequence ASR system with CommonVoice. +The system employs a wav2vec2 encoder and a CTC decoder. +Decoding is performed with greedy decoding (will be extended to beam search). + +To run this recipe, do the following: +> python train_with_wav2vec2.py hparams/train_with_wav2vec2.yaml + +With the default hyperparameters, the system employs a pretrained wav2vec2 encoder. +The wav2vec2 model is pretrained following the model given in the hprams file. +It may be dependent on the language. + +The neural network is trained with CTC on sub-word units estimated with +Byte Pairwise Encoding (BPE). + +The experiment file is flexible enough to support a large variety of +different systems. By properly changing the parameter files, you can try +different encoders, decoders, tokens (e.g, characters instead of BPE), +training languages (all CommonVoice languages), and many +other possible variations. + +Authors + * Titouan Parcollet 2021 +""" + +logger = logging.getLogger(__name__) + + +# Define training procedure +class ASR(sb.core.Brain): + def compute_forward(self, batch, stage): + """Forward computations from the waveform batches to the output probabilities.""" + + batch = batch.to(self.device) + wavs, wav_lens = batch.sig + tokens_bos, _ = batch.tokens_bos + wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device) + + if stage == sb.Stage.TRAIN: + if hasattr(self.hparams, "augmentation"): + wavs = self.hparams.augmentation(wavs, wav_lens) + + # Forward pass + feats = self.modules.wav2vec2(wavs, wav_lens) + x = self.modules.enc(feats) + logits = self.modules.ctc_lin(x) + p_ctc = self.hparams.log_softmax(logits) + + return p_ctc, wav_lens + + def compute_objectives(self, predictions, batch, stage): + """Computes the loss (CTC) given predictions and targets.""" + + p_ctc, wav_lens = predictions + + ids = batch.id + tokens_eos, tokens_eos_lens = batch.tokens_eos + tokens, tokens_lens = batch.tokens + + loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens) + + if stage != sb.Stage.TRAIN: + # Decode token terms to words + sequence = sb.decoders.ctc_greedy_decode( + p_ctc, wav_lens, blank_id=self.hparams.blank_index + ) + + predicted_words = self.tokenizer(sequence, task="decode_from_list") + + # Convert indices to words + target_words = undo_padding(tokens, tokens_lens) + target_words = self.tokenizer(target_words, task="decode_from_list") + + self.wer_metric.append(ids, predicted_words, target_words) + self.cer_metric.append(ids, predicted_words, target_words) + + return loss + + def fit_batch(self, batch): + """Train the parameters given a single batch in input""" + should_step = self.step % self.grad_accumulation_factor == 0 + # Managing automatic mixed precision + # TOFIX: CTC fine-tuning currently is unstable + # This is certainly due to CTC being done in fp16 instead of fp32 + if self.auto_mix_prec: + with torch.cuda.amp.autocast(): + with self.no_sync(): + outputs = self.compute_forward(batch, sb.Stage.TRAIN) + loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN) + with self.no_sync(not should_step): + self.scaler.scale( + loss / self.grad_accumulation_factor + ).backward() + if should_step: + + if not self.hparams.wav2vec2.freeze: + self.scaler.unscale_(self.wav2vec_optimizer) + self.scaler.unscale_(self.model_optimizer) + if self.check_gradients(loss): + if not self.hparams.wav2vec2.freeze: + if self.optimizer_step >= self.hparams.warmup_steps: + self.scaler.step(self.wav2vec_optimizer) + self.scaler.step(self.model_optimizer) + self.scaler.update() + self.zero_grad() + self.optimizer_step += 1 + else: + # This is mandatory because HF models have a weird behavior with DDP + # on the forward pass + with self.no_sync(): + outputs = self.compute_forward(batch, sb.Stage.TRAIN) + + loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN) + + with self.no_sync(not should_step): + (loss / self.grad_accumulation_factor).backward() + if should_step: + if self.check_gradients(loss): + if not self.hparams.wav2vec2.freeze: + if self.optimizer_step >= self.hparams.warmup_steps: + self.wav2vec_optimizer.step() + self.model_optimizer.step() + self.zero_grad() + self.optimizer_step += 1 + + self.on_fit_batch_end(batch, outputs, loss, should_step) + return loss.detach().cpu() + + def evaluate_batch(self, batch, stage): + """Computations needed for validation/test batches""" + predictions = self.compute_forward(batch, stage=stage) + with torch.no_grad(): + loss = self.compute_objectives(predictions, batch, stage=stage) + return loss.detach() + + def on_stage_start(self, stage, epoch): + """Gets called at the beginning of each epoch""" + if stage != sb.Stage.TRAIN: + self.cer_metric = self.hparams.cer_computer() + self.wer_metric = self.hparams.error_rate_computer() + + def on_stage_end(self, stage, stage_loss, epoch): + """Gets called at the end of an epoch.""" + # Compute/store important stats + stage_stats = {"loss": stage_loss} + if stage == sb.Stage.TRAIN: + self.train_stats = stage_stats + else: + stage_stats["CER"] = self.cer_metric.summarize("error_rate") + stage_stats["WER"] = self.wer_metric.summarize("error_rate") + + # Perform end-of-iteration things, like annealing, logging, etc. + if stage == sb.Stage.VALID: + old_lr_model, new_lr_model = self.hparams.lr_annealing_model( + stage_stats["loss"] + ) + old_lr_wav2vec, new_lr_wav2vec = self.hparams.lr_annealing_wav2vec( + stage_stats["loss"] + ) + sb.nnet.schedulers.update_learning_rate( + self.model_optimizer, new_lr_model + ) + if not self.hparams.wav2vec2.freeze: + sb.nnet.schedulers.update_learning_rate( + self.wav2vec_optimizer, new_lr_wav2vec + ) + self.hparams.train_logger.log_stats( + stats_meta={ + "epoch": epoch, + "lr_model": old_lr_model, + "lr_wav2vec": old_lr_wav2vec, + }, + train_stats=self.train_stats, + valid_stats=stage_stats, + ) + self.checkpointer.save_and_keep_only( + meta={"WER": stage_stats["WER"]}, min_keys=["WER"], + ) + elif stage == sb.Stage.TEST: + self.hparams.train_logger.log_stats( + stats_meta={"Epoch loaded": self.hparams.epoch_counter.current}, + test_stats=stage_stats, + ) + with open(self.hparams.wer_file, "w") as w: + self.wer_metric.write_stats(w) + + def init_optimizers(self): + "Initializes the wav2vec2 optimizer and model optimizer" + + # If the wav2vec encoder is unfrozen, we create the optimizer + if not self.hparams.wav2vec2.freeze: + self.wav2vec_optimizer = self.hparams.wav2vec_opt_class( + self.modules.wav2vec2.parameters() + ) + if self.checkpointer is not None: + self.checkpointer.add_recoverable( + "wav2vec_opt", self.wav2vec_optimizer + ) + + self.model_optimizer = self.hparams.model_opt_class( + self.hparams.model.parameters() + ) + + if self.checkpointer is not None: + self.checkpointer.add_recoverable("modelopt", self.model_optimizer) + + def zero_grad(self, set_to_none=False): + if not self.hparams.wav2vec2.freeze: + self.wav2vec_optimizer.zero_grad(set_to_none) + self.model_optimizer.zero_grad(set_to_none) + + +# Define custom data procedure +def dataio_prepare(hparams, tokenizer): + """This function prepares the datasets to be used in the brain class. + It also defines the data processing pipeline through user-defined functions.""" + + # 1. Define datasets + data_folder = hparams["data_folder"] + + train_data = sb.dataio.dataset.DynamicItemDataset.from_csv( + csv_path=hparams["train_csv"], replacements={"data_root": data_folder}, + ) + + if hparams["sorting"] == "ascending": + # we sort training data to speed up training and get better results. + train_data = train_data.filtered_sorted( + sort_key="duration", + key_max_value={"duration": hparams["avoid_if_longer_than"]}, + ) + # when sorting do not shuffle in dataloader ! otherwise is pointless + hparams["dataloader_options"]["shuffle"] = False + + elif hparams["sorting"] == "descending": + train_data = train_data.filtered_sorted( + sort_key="duration", + reverse=True, + key_max_value={"duration": hparams["avoid_if_longer_than"]}, + ) + # when sorting do not shuffle in dataloader ! otherwise is pointless + hparams["dataloader_options"]["shuffle"] = False + + elif hparams["sorting"] == "random": + pass + + else: + raise NotImplementedError( + "sorting must be random, ascending or descending" + ) + + valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv( + csv_path=hparams["valid_csv"], replacements={"data_root": data_folder}, + ) + # We also sort the validation data so it is faster to validate + valid_data = valid_data.filtered_sorted(sort_key="duration") + + test_data = sb.dataio.dataset.DynamicItemDataset.from_csv( + csv_path=hparams["test_csv"], replacements={"data_root": data_folder}, + ) + + # We also sort the validation data so it is faster to validate + test_data = test_data.filtered_sorted(sort_key="duration") + + datasets = [train_data, valid_data, test_data] + + # 2. Define audio pipeline: + @sb.utils.data_pipeline.takes("wav") + @sb.utils.data_pipeline.provides("sig") + def audio_pipeline(wav): + info = torchaudio.info(wav) + sig = sb.dataio.dataio.read_audio(wav) + resampled = torchaudio.transforms.Resample( + info.sample_rate, hparams["sample_rate"], + )(sig) + return resampled + + sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline) + + # 3. Define text pipeline: + @sb.utils.data_pipeline.takes("wrd") + @sb.utils.data_pipeline.provides( + "tokens_list", "tokens_bos", "tokens_eos", "tokens" + ) + def text_pipeline(wrd): + tokens_list = tokenizer.sp.encode_as_ids(wrd) + yield tokens_list + tokens_bos = torch.LongTensor([hparams["bos_index"]] + (tokens_list)) + yield tokens_bos + tokens_eos = torch.LongTensor(tokens_list + [hparams["eos_index"]]) + yield tokens_eos + tokens = torch.LongTensor(tokens_list) + yield tokens + + sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline) + + # 4. Set output: + sb.dataio.dataset.set_output_keys( + datasets, ["id", "sig", "tokens_bos", "tokens_eos", "tokens"], + ) + return train_data, valid_data, test_data + + +if __name__ == "__main__": + + # Load hyperparameters file with command-line overrides + hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:]) + with open(hparams_file) as fin: + hparams = load_hyperpyyaml(fin, overrides) + + # If --distributed_launch then + # create ddp_group with the right communication protocol + sb.utils.distributed.ddp_init_group(run_opts) + + # Dataset preparation (parsing CommonVoice) + from common_voice_prepare import prepare_common_voice # noqa + + # Create experiment directory + sb.create_experiment_directory( + experiment_directory=hparams["output_folder"], + hyperparams_to_save=hparams_file, + overrides=overrides, + ) + + # Due to DDP, we do the preparation ONLY on the main python process + run_on_main( + prepare_common_voice, + kwargs={ + "data_folder": hparams["data_folder"], + "save_folder": hparams["save_folder"], + "train_tsv_file": hparams["train_tsv_file"], + "dev_tsv_file": hparams["dev_tsv_file"], + "test_tsv_file": hparams["test_tsv_file"], + "accented_letters": hparams["accented_letters"], + "language": hparams["language"], + "skip_prep": hparams["skip_prep"], + }, + ) + + # Defining tokenizer and loading it + tokenizer = SentencePiece( + model_dir=hparams["save_folder"], + vocab_size=hparams["output_neurons"], + annotation_train=hparams["train_csv"], + annotation_read="wrd", + model_type=hparams["token_type"], + character_coverage=hparams["character_coverage"], + ) + + # Create the datasets objects as well as tokenization and encoding :-D + train_data, valid_data, test_data = dataio_prepare(hparams, tokenizer) + + # Trainer initialization + asr_brain = ASR( + modules=hparams["modules"], + hparams=hparams, + run_opts=run_opts, + checkpointer=hparams["checkpointer"], + ) + + # Adding objects to trainer. + asr_brain.tokenizer = tokenizer + + # Training + asr_brain.fit( + asr_brain.hparams.epoch_counter, + train_data, + valid_data, + train_loader_kwargs=hparams["dataloader_options"], + valid_loader_kwargs=hparams["test_dataloader_options"], + ) + + # Test + asr_brain.hparams.wer_file = hparams["output_folder"] + "/wer_test.txt" + asr_brain.evaluate( + test_data, + min_key="WER", + test_loader_kwargs=hparams["test_dataloader_options"], + ) diff --git a/EnglishCV/train_en_with_wav2vec.yaml b/EnglishCV/train_en_with_wav2vec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..576d870d0672a02f34ad4eaa98760af09a7e351b --- /dev/null +++ b/EnglishCV/train_en_with_wav2vec.yaml @@ -0,0 +1,184 @@ +# ################################ +# Model: wav2vec2 + DNN + CTC +# Augmentation: SpecAugment +# Authors: Titouan Parcollet 2021 +# ################################ + +# Seed needs to be set at top of yaml, before objects with parameters are made +seed: 1234 +__set_seed: !!python/object/apply:torch.manual_seed [!ref ] +output_folder: !ref results/wav2vec2_ctc_en/ +wer_file: !ref /wer.txt +save_folder: !ref /save +train_log: !ref /train_log.txt + +# URL for the biggest Fairseq english wav2vec2 model. +wav2vec2_hub: facebook/wav2vec2-large-lv60 +wav2vec2_folder: !ref /wav2vec2_checkpoint + +# Data files +data_folder: /gpfsscratch/rech/nou/uzn19yk/download/cv-corpus-12.0-2022-12-07/en/cv-corpus-12.0-2022-12-07/en # e.g, /localscratch/cv-corpus-5.1-2020-06-22/fr +train_tsv_file: !ref /train.tsv # Standard CommonVoice .tsv files +dev_tsv_file: !ref /dev.tsv # Standard CommonVoice .tsv files +test_tsv_file: !ref /test.tsv # Standard CommonVoice .tsv files +accented_letters: False +language: en # use 'it' for Italian, 'rw' for Kinyarwanda, 'en' for english +train_csv: !ref /train.csv +valid_csv: !ref /dev.csv +test_csv: !ref /test.csv +skip_prep: False # Skip data preparation + +# We remove utterance slonger than 10s in the train/dev/test sets as +# longer sentences certainly correspond to "open microphones". +avoid_if_longer_than: 10.0 + +# Training parameters +number_of_epochs: 10 +lr: 1.0 +lr_wav2vec: 0.0001 +sorting: ascending +auto_mix_prec: False +sample_rate: 16000 +ckpt_interval_minutes: 30 # save checkpoint every N min + +# With data_parallel batch_size is split into N jobs +# With DDP batch_size is multiplied by N jobs +# Must be 8 per GPU to fit 32GB of VRAM +batch_size: 8 +test_batch_size: 4 + +dataloader_options: + batch_size: !ref + num_workers: 6 +test_dataloader_options: + batch_size: !ref + num_workers: 6 + +# BPE parameters +token_type: char # ["unigram", "bpe", "char"] +character_coverage: 1.0 + +# Model parameters +# activation: !name:torch.nn.LeakyReLU +wav2vec_output_dim: 1024 +dnn_neurons: 1024 +freeze_wav2vec: False +freeze_feature_extractor: True +dropout: 0.15 +warmup_steps: 500 + +# Outputs +output_neurons: 29 # BPE size, index(blank/eos/bos) = 0 + +# Decoding parameters +# Be sure that the bos and eos index match with the BPEs ones +blank_index: 0 +bos_index: 1 +eos_index: 2 + +# +# Functions and classes +# +epoch_counter: !new:speechbrain.utils.epoch_loop.EpochCounter + limit: !ref + +augmentation: !new:speechbrain.lobes.augment.TimeDomainSpecAugment + sample_rate: !ref + speeds: [95, 100, 105] + +enc: !new:speechbrain.nnet.containers.Sequential + input_shape: [null, null, !ref ] + linear1: !name:speechbrain.nnet.linear.Linear + n_neurons: !ref + bias: True + bn1: !name:speechbrain.nnet.normalization.BatchNorm1d + activation: !new:torch.nn.LeakyReLU + drop: !new:torch.nn.Dropout + p: !ref + linear2: !name:speechbrain.nnet.linear.Linear + n_neurons: !ref + bias: True + bn2: !name:speechbrain.nnet.normalization.BatchNorm1d + activation2: !new:torch.nn.LeakyReLU + drop2: !new:torch.nn.Dropout + p: !ref + linear3: !name:speechbrain.nnet.linear.Linear + n_neurons: !ref + bias: True + bn3: !name:speechbrain.nnet.normalization.BatchNorm1d + activation3: !new:torch.nn.LeakyReLU + +wav2vec2: !new:speechbrain.lobes.models.huggingface_wav2vec.HuggingFaceWav2Vec2 + source: /gpfsscratch/rech/nou/uzn19yk/wav2vec2-large-lv60/ + output_norm: True + freeze: !ref + freeze_feature_extractor: !ref + save_path: !ref + +##### +# Uncomment this block if you prefer to use a Fairseq pretrained model instead +# of a HuggingFace one. Here, we provide an URL that is obtained from the +# Fairseq github for the multilingual XLSR. +# +#wav2vec2_url: https://dl.fbaipublicfiles.com/fairseq/wav2vec/xlsr_53_56k.pt +#wav2vec2: !new:speechbrain.lobes.models.fairseq_wav2vec.FairseqWav2Vec2 +# pretrained_path: !ref +# output_norm: True +# freeze: False +# save_path: !ref /wav2vec2_checkpoint/model.pt +##### + +ctc_lin: !new:speechbrain.nnet.linear.Linear + input_size: !ref + n_neurons: !ref + +log_softmax: !new:speechbrain.nnet.activations.Softmax + apply_log: True + +ctc_cost: !name:speechbrain.nnet.losses.ctc_loss + blank_index: !ref + +modules: + wav2vec2: !ref + enc: !ref + ctc_lin: !ref + +model: !new:torch.nn.ModuleList + - [!ref , !ref ] + +model_opt_class: !name:torch.optim.Adadelta + lr: !ref + rho: 0.95 + eps: 1.e-8 + +wav2vec_opt_class: !name:torch.optim.Adam + lr: !ref + +lr_annealing_model: !new:speechbrain.nnet.schedulers.NewBobScheduler + initial_value: !ref + improvement_threshold: 0.0025 + annealing_factor: 0.8 + patient: 0 + +lr_annealing_wav2vec: !new:speechbrain.nnet.schedulers.NewBobScheduler + initial_value: !ref + improvement_threshold: 0.0025 + annealing_factor: 0.9 + patient: 0 + +checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer + checkpoints_dir: !ref + recoverables: + wav2vec2: !ref + model: !ref + scheduler_model: !ref + scheduler_wav2vec: !ref + counter: !ref + +train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger + save_file: !ref + +error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats + +cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats + split_tokens: True diff --git a/EnglishCV/train_with_wav2vec.py b/EnglishCV/train_with_wav2vec.py new file mode 100644 index 0000000000000000000000000000000000000000..d462abf3d2e8244df5cdb183aa231082d7791a45 --- /dev/null +++ b/EnglishCV/train_with_wav2vec.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 +import sys +import torch +import logging +import speechbrain as sb +import torchaudio +from hyperpyyaml import load_hyperpyyaml +from speechbrain.tokenizers.SentencePiece import SentencePiece +from speechbrain.utils.data_utils import undo_padding +from speechbrain.utils.distributed import run_on_main + +"""Recipe for training a sequence-to-sequence ASR system with CommonVoice. +The system employs a wav2vec2 encoder and a CTC decoder. +Decoding is performed with greedy decoding (will be extended to beam search). + +To run this recipe, do the following: +> python train_with_wav2vec2.py hparams/train_with_wav2vec2.yaml + +With the default hyperparameters, the system employs a pretrained wav2vec2 encoder. +The wav2vec2 model is pretrained following the model given in the hprams file. +It may be dependent on the language. + +The neural network is trained with CTC on sub-word units estimated with +Byte Pairwise Encoding (BPE). + +The experiment file is flexible enough to support a large variety of +different systems. By properly changing the parameter files, you can try +different encoders, decoders, tokens (e.g, characters instead of BPE), +training languages (all CommonVoice languages), and many +other possible variations. + +Authors + * Titouan Parcollet 2021 +""" + +logger = logging.getLogger(__name__) + + +# Define training procedure +class ASR(sb.core.Brain): + def compute_forward(self, batch, stage): + """Forward computations from the waveform batches to the output probabilities.""" + + batch = batch.to(self.device) + wavs, wav_lens = batch.sig + tokens_bos, _ = batch.tokens_bos + wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device) + + if stage == sb.Stage.TRAIN: + if hasattr(self.hparams, "augmentation"): + wavs = self.hparams.augmentation(wavs, wav_lens) + + # Forward pass + feats = self.modules.wav2vec2(wavs, wav_lens) + x = self.modules.enc(feats) + logits = self.modules.ctc_lin(x) + p_ctc = self.hparams.log_softmax(logits) + + return p_ctc, wav_lens + + def compute_objectives(self, predictions, batch, stage): + """Computes the loss (CTC) given predictions and targets.""" + + p_ctc, wav_lens = predictions + + ids = batch.id + tokens_eos, tokens_eos_lens = batch.tokens_eos + tokens, tokens_lens = batch.tokens + + loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens) + + if stage != sb.Stage.TRAIN: + # Decode token terms to words + sequence = sb.decoders.ctc_greedy_decode( + p_ctc, wav_lens, blank_id=self.hparams.blank_index + ) + + predicted_words = self.tokenizer(sequence, task="decode_from_list") + + # Convert indices to words + target_words = undo_padding(tokens, tokens_lens) + target_words = self.tokenizer(target_words, task="decode_from_list") + + self.wer_metric.append(ids, predicted_words, target_words) + self.cer_metric.append(ids, predicted_words, target_words) + + return loss + + def fit_batch(self, batch): + """Train the parameters given a single batch in input""" + should_step = self.step % self.grad_accumulation_factor == 0 + # Managing automatic mixed precision + # TOFIX: CTC fine-tuning currently is unstable + # This is certainly due to CTC being done in fp16 instead of fp32 + if self.auto_mix_prec: + with torch.cuda.amp.autocast(): + with self.no_sync(): + outputs = self.compute_forward(batch, sb.Stage.TRAIN) + loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN) + with self.no_sync(not should_step): + self.scaler.scale( + loss / self.grad_accumulation_factor + ).backward() + if should_step: + + if not self.hparams.wav2vec2.freeze: + self.scaler.unscale_(self.wav2vec_optimizer) + self.scaler.unscale_(self.model_optimizer) + if self.check_gradients(loss): + if not self.hparams.wav2vec2.freeze: + if self.optimizer_step >= self.hparams.warmup_steps: + self.scaler.step(self.wav2vec_optimizer) + self.scaler.step(self.model_optimizer) + self.scaler.update() + self.zero_grad() + self.optimizer_step += 1 + else: + # This is mandatory because HF models have a weird behavior with DDP + # on the forward pass + with self.no_sync(): + outputs = self.compute_forward(batch, sb.Stage.TRAIN) + + loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN) + + with self.no_sync(not should_step): + (loss / self.grad_accumulation_factor).backward() + if should_step: + if self.check_gradients(loss): + if not self.hparams.wav2vec2.freeze: + if self.optimizer_step >= self.hparams.warmup_steps: + self.wav2vec_optimizer.step() + self.model_optimizer.step() + self.zero_grad() + self.optimizer_step += 1 + + self.on_fit_batch_end(batch, outputs, loss, should_step) + return loss.detach().cpu() + + def evaluate_batch(self, batch, stage): + """Computations needed for validation/test batches""" + predictions = self.compute_forward(batch, stage=stage) + with torch.no_grad(): + loss = self.compute_objectives(predictions, batch, stage=stage) + return loss.detach() + + def on_stage_start(self, stage, epoch): + """Gets called at the beginning of each epoch""" + if stage != sb.Stage.TRAIN: + self.cer_metric = self.hparams.cer_computer() + self.wer_metric = self.hparams.error_rate_computer() + + def on_stage_end(self, stage, stage_loss, epoch): + """Gets called at the end of an epoch.""" + # Compute/store important stats + stage_stats = {"loss": stage_loss} + if stage == sb.Stage.TRAIN: + self.train_stats = stage_stats + else: + stage_stats["CER"] = self.cer_metric.summarize("error_rate") + stage_stats["WER"] = self.wer_metric.summarize("error_rate") + + # Perform end-of-iteration things, like annealing, logging, etc. + if stage == sb.Stage.VALID: + old_lr_model, new_lr_model = self.hparams.lr_annealing_model( + stage_stats["loss"] + ) + old_lr_wav2vec, new_lr_wav2vec = self.hparams.lr_annealing_wav2vec( + stage_stats["loss"] + ) + sb.nnet.schedulers.update_learning_rate( + self.model_optimizer, new_lr_model + ) + if not self.hparams.wav2vec2.freeze: + sb.nnet.schedulers.update_learning_rate( + self.wav2vec_optimizer, new_lr_wav2vec + ) + self.hparams.train_logger.log_stats( + stats_meta={ + "epoch": epoch, + "lr_model": old_lr_model, + "lr_wav2vec": old_lr_wav2vec, + }, + train_stats=self.train_stats, + valid_stats=stage_stats, + ) + self.checkpointer.save_and_keep_only( + meta={"WER": stage_stats["WER"]}, min_keys=["WER"], + ) + elif stage == sb.Stage.TEST: + self.hparams.train_logger.log_stats( + stats_meta={"Epoch loaded": self.hparams.epoch_counter.current}, + test_stats=stage_stats, + ) + with open(self.hparams.wer_file, "w") as w: + self.wer_metric.write_stats(w) + + def init_optimizers(self): + "Initializes the wav2vec2 optimizer and model optimizer" + + # If the wav2vec encoder is unfrozen, we create the optimizer + if not self.hparams.wav2vec2.freeze: + self.wav2vec_optimizer = self.hparams.wav2vec_opt_class( + self.modules.wav2vec2.parameters() + ) + if self.checkpointer is not None: + self.checkpointer.add_recoverable( + "wav2vec_opt", self.wav2vec_optimizer + ) + + self.model_optimizer = self.hparams.model_opt_class( + self.hparams.model.parameters() + ) + + if self.checkpointer is not None: + self.checkpointer.add_recoverable("modelopt", self.model_optimizer) + + def zero_grad(self, set_to_none=False): + if not self.hparams.wav2vec2.freeze: + self.wav2vec_optimizer.zero_grad(set_to_none) + self.model_optimizer.zero_grad(set_to_none) + + +# Define custom data procedure +def dataio_prepare(hparams, tokenizer): + """This function prepares the datasets to be used in the brain class. + It also defines the data processing pipeline through user-defined functions.""" + + # 1. Define datasets + data_folder = hparams["data_folder"] + + train_data = sb.dataio.dataset.DynamicItemDataset.from_csv( + csv_path=hparams["train_csv"], replacements={"data_root": data_folder}, + ) + + if hparams["sorting"] == "ascending": + # we sort training data to speed up training and get better results. + train_data = train_data.filtered_sorted( + sort_key="duration", + key_max_value={"duration": hparams["avoid_if_longer_than"]}, + ) + # when sorting do not shuffle in dataloader ! otherwise is pointless + hparams["dataloader_options"]["shuffle"] = False + + elif hparams["sorting"] == "descending": + train_data = train_data.filtered_sorted( + sort_key="duration", + reverse=True, + key_max_value={"duration": hparams["avoid_if_longer_than"]}, + ) + # when sorting do not shuffle in dataloader ! otherwise is pointless + hparams["dataloader_options"]["shuffle"] = False + + elif hparams["sorting"] == "random": + pass + + else: + raise NotImplementedError( + "sorting must be random, ascending or descending" + ) + + valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv( + csv_path=hparams["valid_csv"], replacements={"data_root": data_folder}, + ) + # We also sort the validation data so it is faster to validate + valid_data = valid_data.filtered_sorted(sort_key="duration") + + test_data = sb.dataio.dataset.DynamicItemDataset.from_csv( + csv_path=hparams["test_csv"], replacements={"data_root": data_folder}, + ) + + # We also sort the validation data so it is faster to validate + test_data = test_data.filtered_sorted(sort_key="duration") + + datasets = [train_data, valid_data, test_data] + + # 2. Define audio pipeline: + @sb.utils.data_pipeline.takes("wav") + @sb.utils.data_pipeline.provides("sig") + def audio_pipeline(wav): + info = torchaudio.info(wav) + sig = sb.dataio.dataio.read_audio(wav) + resampled = torchaudio.transforms.Resample( + info.sample_rate, hparams["sample_rate"], + )(sig) + return resampled + + sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline) + + # 3. Define text pipeline: + @sb.utils.data_pipeline.takes("wrd") + @sb.utils.data_pipeline.provides( + "tokens_list", "tokens_bos", "tokens_eos", "tokens" + ) + def text_pipeline(wrd): + tokens_list = tokenizer.sp.encode_as_ids(wrd) + yield tokens_list + tokens_bos = torch.LongTensor([hparams["bos_index"]] + (tokens_list)) + yield tokens_bos + tokens_eos = torch.LongTensor(tokens_list + [hparams["eos_index"]]) + yield tokens_eos + tokens = torch.LongTensor(tokens_list) + yield tokens + + sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline) + + # 4. Set output: + sb.dataio.dataset.set_output_keys( + datasets, ["id", "sig", "tokens_bos", "tokens_eos", "tokens"], + ) + return train_data, valid_data, test_data + + +if __name__ == "__main__": + + # Load hyperparameters file with command-line overrides + hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:]) + with open(hparams_file) as fin: + hparams = load_hyperpyyaml(fin, overrides) + + # If --distributed_launch then + # create ddp_group with the right communication protocol + sb.utils.distributed.ddp_init_group(run_opts) + + # Dataset preparation (parsing CommonVoice) + from common_voice_prepare import prepare_common_voice # noqa + + # Create experiment directory + sb.create_experiment_directory( + experiment_directory=hparams["output_folder"], + hyperparams_to_save=hparams_file, + overrides=overrides, + ) + + # Due to DDP, we do the preparation ONLY on the main python process + run_on_main( + prepare_common_voice, + kwargs={ + "data_folder": hparams["data_folder"], + "save_folder": hparams["save_folder"], + "train_tsv_file": hparams["train_tsv_file"], + "dev_tsv_file": hparams["dev_tsv_file"], + "test_tsv_file": hparams["test_tsv_file"], + "accented_letters": hparams["accented_letters"], + "language": hparams["language"], + "skip_prep": hparams["skip_prep"], + }, + ) + + # Defining tokenizer and loading it + tokenizer = SentencePiece( + model_dir=hparams["save_folder"], + vocab_size=hparams["output_neurons"], + annotation_train=hparams["train_csv"], + annotation_read="wrd", + model_type=hparams["token_type"], + character_coverage=hparams["character_coverage"], + ) + + # Create the datasets objects as well as tokenization and encoding :-D + train_data, valid_data, test_data = dataio_prepare(hparams, tokenizer) + + # Trainer initialization + asr_brain = ASR( + modules=hparams["modules"], + hparams=hparams, + run_opts=run_opts, + checkpointer=hparams["checkpointer"], + ) + + # Adding objects to trainer. + asr_brain.tokenizer = tokenizer + + # Training + asr_brain.fit( + asr_brain.hparams.epoch_counter, + train_data, + valid_data, + train_loader_kwargs=hparams["dataloader_options"], + valid_loader_kwargs=hparams["test_dataloader_options"], + ) + + # Test + asr_brain.hparams.wer_file = hparams["output_folder"] + "/wer_test.txt" + asr_brain.evaluate( + test_data, + min_key="WER", + test_loader_kwargs=hparams["test_dataloader_options"], + ) diff --git a/README.md b/README.md index 154df8298fab5ecf322016157858e08cd1bccbe1..79e4305d04d46117886fcaa654f84ab798113a7e 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,21 @@ --- license: apache-2.0 --- +# Tunisian Arabic ASR Model with wav2vec2 and code switching +This repository provides all the necessary tools to perform automatic speech recognition from an end-to-end system pretrained on Tunisian arabic dialect. This model utilizes a code_switching approach and can process english , french and tunisian arabic +## Performance +the performance of the mode is : +| Release Version |WER (%) | CER (%) | +|-----------------|---------|---------| +| v1.0 |29.47 | 12.44 | +## Pipeline +The architecture comprises three components: + * French ASR pretrained with wav2vec2 on french corporas + * English ASR pretrained with wav2vec2 on english corporas + * Custom Tunisian ASR pretrained using wav2vec on a tunisian arabic corpora + All three models will process the audio data. Subsequently, the resulting posteriorgrams will be combined and utilized as input for the Mixer, which will produce the final posteriorgrams. +## Install +```python +pip install speechbrain transformers +``` + diff --git a/TunisianASR/README.md b/TunisianASR/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ee722217a29d8f656f945ba096eb38eebe58fc9 --- /dev/null +++ b/TunisianASR/README.md @@ -0,0 +1,21 @@ +# Tunisian Arabic ASR Model with wav2vec2 + +This repository provides all the necessary tools to perform automatic speech recognition from an end-to-end system pretrained on Tunisian arabic dialect + +## Performance +the performance of the mode is : +| Release Version | |WER (%) | CER (%) | +|-----------------|----|---------|---------| +| v1.0 | Without LM |11.82 | 6.33 | +## Dataset +This ASR model was trained on : +* TARIC : The corpus, named TARIC (Tunisian Arabic Railway Interaction Corpus) has a collection of audio recordings and transcriptions from dialogues in the Tunisian Railway Transport Network. - [Taric Corpus](https://aclanthology.org/L14-1385/) - +* STAC :A corpus of spoken Tunisian Arabic - [STAC Corpus](https://www.researchgate.net/publication/307583782_Spoken_Tunisian_Arabic_Corpus_STAC_Transcription_and_Annotation) +* IWSLT : A Tunisian conversational speech - [IWSLT Corpus](https://iwslt.org/2022/dialect)- +* Tunspeech : Our custom dataset + +## Install +```python +pip install speechbrain transformers +``` + diff --git a/TunisianASR/outdomain.arpa b/TunisianASR/outdomain.arpa new file mode 100644 index 0000000000000000000000000000000000000000..b9c323fdeb2bb915c5ee4f0bcb5b0a8b9b953004 --- /dev/null +++ b/TunisianASR/outdomain.arpa @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24654c1d236bb1bd367125131c847c4a734e69914eda71a6786964c20440d8fe +size 324243244 diff --git a/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/hyperparams.yaml b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/hyperparams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1e9433273644e8d155594d25be6a5469b7a82fa --- /dev/null +++ b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/hyperparams.yaml @@ -0,0 +1,194 @@ +# Generated 2023-09-08 from: +# /gpfsdsstore/projects/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/hparams/train_semi.yaml +# yamllint disable +# ################################ +# Model: wav2vec2 + DNN + CTC +# Augmentation: SpecAugment +# Authors: Titouan Parcollet 2021 +# ################################ + +# Seed needs to be set at top of yaml, before objects with parameters are made +seed: 1234 +__set_seed: !!python/object/apply:torch.manual_seed [1234] +output_folder: results/semi_wavlm_large_tunisian_ctc/1234 +wer_file: results/semi_wavlm_large_tunisian_ctc/1234/wer.txt +save_folder: results/semi_wavlm_large_tunisian_ctc/1234/save +train_log: results/semi_wavlm_large_tunisian_ctc/1234/train_log.txt + +# URL for the biggest LeBenchmark wav2vec french. +wav2vec2_folder: results/semi_wavlm_large_tunisian_ctc/1234/save/wav2vec2_checkpoint + +# Data files +data_folder: /gpfsscratch/rech/nou/uzn19yk/tunisian_junk # e.g, /localscratch/cv-corpus-5.1-2020-06-22/fr +train_tsv_file: /gpfsscratch/rech/nou/uzn19yk/tunisian_junk/train.tsv # Standard CommonVoice .tsv files +dev_tsv_file: /gpfsscratch/rech/nou/uzn19yk/tunisian_junk/dev.tsv # Standard CommonVoice .tsv files +test_tsv_file: /gpfsscratch/rech/nou/uzn19yk/tunisian_junk/test.tsv # Standard CommonVoice .tsv files +accented_letters: true +language: fr # use 'it' for Italian, 'rw' for Kinyarwanda, 'en' for english +train_csv: /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/good_final/train_enhanced.csv +valid_csv: /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/good_final/dev.csv +test_csv: +- /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/full_annotation_test.csv +- /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/good_final/iwslt_test.csv +- /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/good_final/taric_test.csv + +skip_prep: true # Skip data preparation + +use_language_modelling: true +ngram_lm_path: arpas/outdomain.arpa + +# We remove utterance slonger than 10s in the train/dev/test sets as +# longer sentences certainly correspond to "open microphones". +avoid_if_longer_than: 10.0 +avoid_if_shorter_than: 1.2 + + +# Training parameters +number_of_epochs: 12 +lr: 1.0 +lr_wav2vec: 0.0001 +sorting: ascending +auto_mix_prec: false +sample_rate: 16000 +ckpt_interval_minutes: 30 # save checkpoint every N min + +# With data_parallel batch_size is split into N jobs +# With DDP batch_size is multiplied by N jobs +# Must be 6 per GPU to fit 16GB of VRAM +batch_size: 10 +test_batch_size: 4 + +dataloader_options: + batch_size: 10 + num_workers: 6 +test_dataloader_options: + batch_size: 4 + num_workers: 6 + +# BPE parameters +token_type: char # ["unigram", "bpe", "char"] +character_coverage: 1.0 + +# Model parameters +# activation: !name:torch.nn.LeakyReLU +wav2vec_output_dim: 1024 +dnn_neurons: 1024 +freeze_wav2vec: false +freeze_feature_extractor: true +dropout: 0.15 +warmup_steps: 500 # The wav2vec 2 model isn't updated for this amount of steps + +# Outputs +output_neurons: 40 # BPE size, index(blank/eos/bos) = 0 + +# Decoding parameters +# Be sure that the bos and eos index match with the BPEs ones +blank_index: 0 +unk_index: 1 + +# +# Functions and classes +# +epoch_counter: &id007 !new:speechbrain.utils.epoch_loop.EpochCounter + + limit: 12 + +augmentation: !new:speechbrain.lobes.augment.TimeDomainSpecAugment + sample_rate: 16000 + speeds: [95, 100, 105] + +enc: &id002 !new:speechbrain.nnet.containers.Sequential + input_shape: [null, null, 1024] + linear1: !name:speechbrain.nnet.linear.Linear + n_neurons: 1024 + bias: true + bn1: !name:speechbrain.nnet.normalization.BatchNorm1d + activation: !new:torch.nn.LeakyReLU + drop: !new:torch.nn.Dropout + p: 0.15 + linear2: !name:speechbrain.nnet.linear.Linear + n_neurons: 1024 + bias: true + bn2: !name:speechbrain.nnet.normalization.BatchNorm1d + activation2: !new:torch.nn.LeakyReLU + drop2: !new:torch.nn.Dropout + p: 0.15 + linear3: !name:speechbrain.nnet.linear.Linear + n_neurons: 1024 + bias: true + bn3: !name:speechbrain.nnet.normalization.BatchNorm1d + activation3: !new:torch.nn.LeakyReLU + +wav2vec2: &id001 !new:speechbrain.lobes.models.huggingface_wav2vec.HuggingFaceWav2Vec2 + source: /gpfsstore/rech/nou/uzn19yk/wavlm/ + output_norm: false + freeze: false + freeze_feature_extractor: true + save_path: results/semi_wavlm_large_tunisian_ctc/1234/save/wav2vec2_checkpoint + +##### +# Uncomment this block if you prefer to use a Fairseq pretrained model instead +# of a HuggingFace one. Here, we provide an URL that is obtained from the +# Fairseq github for the multilingual XLSR. +# +#wav2vec2_url: https://dl.fbaipublicfiles.com/fairseq/wav2vec/xlsr_53_56k.pt +#wav2vec2: !new:speechbrain.lobes.models.fairseq_wav2vec.FairseqWav2Vec2 +# pretrained_path: !ref +# output_norm: True +# freeze: False +# save_path: !ref /wav2vec2_checkpoint/model.pt +##### + + +ctc_lin: &id003 !new:speechbrain.nnet.linear.Linear + + input_size: 1024 + n_neurons: 40 + +log_softmax: !new:speechbrain.nnet.activations.Softmax + apply_log: true + +ctc_cost: !name:speechbrain.nnet.losses.ctc_loss + blank_index: 0 + +modules: + wav2vec2: *id001 + enc: *id002 + ctc_lin: *id003 +model: &id004 !new:torch.nn.ModuleList +- [*id002, *id003] +model_opt_class: !name:torch.optim.Adadelta + lr: 1.0 + rho: 0.95 + eps: 1.e-8 + +wav2vec_opt_class: !name:torch.optim.Adam + lr: 0.0001 + +lr_annealing_model: &id005 !new:speechbrain.nnet.schedulers.NewBobScheduler + initial_value: 1.0 + improvement_threshold: 0.0025 + annealing_factor: 0.8 + patient: 0 + +lr_annealing_wav2vec: &id006 !new:speechbrain.nnet.schedulers.NewBobScheduler + initial_value: 0.0001 + improvement_threshold: 0.0025 + annealing_factor: 0.9 + patient: 0 + +checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer + checkpoints_dir: results/semi_wavlm_large_tunisian_ctc/1234/save + recoverables: + wav2vec2: *id001 + model: *id004 + scheduler_model: *id005 + scheduler_wav2vec: *id006 + counter: *id007 +train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger + save_file: results/semi_wavlm_large_tunisian_ctc/1234/train_log.txt + +error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats + +cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats + split_tokens: true diff --git a/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/CKPT.yaml b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/CKPT.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd397e0a3e0ccc45e6851c01e8504215b8dcc54b --- /dev/null +++ b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/CKPT.yaml @@ -0,0 +1,4 @@ +# yamllint disable +WER: 27.83210816487267 +end-of-epoch: true +unixtime: 1693868963.5220973 diff --git a/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/brain.ckpt b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/brain.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..701b1db3e66a840db866fb622fa5f3fd37e4bcd1 --- /dev/null +++ b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/brain.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3947a24e8dff5a14299b9cf2fe66ffb4d738cb88717de7f0cf7e8547a76e9776 +size 51 diff --git a/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/counter.ckpt b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/counter.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..57888a63b27e6c446fa67da3950dada56b034e8d --- /dev/null +++ b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/counter.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b51d431df5d7f141cbececcf79edf3dd861c3b4069f0b11661a3eefacbba918 +size 2 diff --git a/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/dataloader-TRAIN.ckpt b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/dataloader-TRAIN.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..33093232fcb90259d6ee328fe4b0286b9b50c479 --- /dev/null +++ b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/dataloader-TRAIN.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b363886c229e536bd3c84e0c3e89312d70e00422578e076a62df1b45c9390793 +size 5 diff --git a/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/model.ckpt b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/model.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..42e32175f209c8501ba9b202d2583d710d5ea110 --- /dev/null +++ b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/model.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc1dbeca1e1f1340b08d8ebea6e492f474708dddbbe8cabbcdde5ee9660704f2 +size 12814446 diff --git a/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/modelopt.ckpt b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/modelopt.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..332018da0edfb8ffa2d939a95982a9c22ad06ef1 --- /dev/null +++ b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/modelopt.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3af1791eb9a5bfbfc087d2c10b94634df24cad3ac503ce9ba280a3ecc4737781 +size 25575663 diff --git a/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/scheduler_model.ckpt b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/scheduler_model.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..84f2b41f86d47e2337b1a111ed84bb566365cbf5 --- /dev/null +++ b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/scheduler_model.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c275ab9245b440d1586f72058d9edaac1a2fb3e7a52712aa9a9ad022b99a1c0d +size 639 diff --git a/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/scheduler_wav2vec.ckpt b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/scheduler_wav2vec.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f045520e49875e906b2d8b1393045f89003afb18 --- /dev/null +++ b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/scheduler_wav2vec.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a88187f7882dc3e10c108f1b7abfbd819285b34bded4e88e91c4ff699c1bb5d2 +size 643 diff --git a/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/wav2vec2.ckpt b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/wav2vec2.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..1116f4f8f22dce9cbfbdebc0da4063c582ab43b3 --- /dev/null +++ b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/wav2vec2.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:788267bd25ef37623715fa21a975090e5e316fff05971375cd3f62e5160f0743 +size 1262005979 diff --git a/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/wav2vec_opt.ckpt b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/wav2vec_opt.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..8434648380dcd4a0949b117c13958181c02e1da8 --- /dev/null +++ b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/wav2vec_opt.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efa967fdd8067be7d88c18cd197980c9c91f344a3dff2b2518b8381c49f28b1e +size 2490361859 diff --git a/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/label_encoder.txt b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/label_encoder.txt new file mode 100644 index 0000000000000000000000000000000000000000..737f9151e65d7dd5cd4f0b5960d2caace176ac85 --- /dev/null +++ b/TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/label_encoder.txt @@ -0,0 +1,44 @@ +'ب' => 38 +'ا' => 1 +'ه' => 2 +'ي' => 3 +'و' => 4 +'ن' => 5 +'أ' => 6 +' ' => 7 +'م' => 8 +'ش' => 9 +'ل' => 10 +'س' => 11 +'ت' => 12 +'د' => 13 +'ر' => 14 +'ى' => 15 +'ح' => 16 +'ط' => 17 +'ع' => 18 +'ك' => 19 +'ف' => 20 +'ق' => 21 +'آ' => 22 +'ة' => 23 +'ج' => 24 +'ض' => 25 +'ز' => 26 +'ص' => 27 +'إ' => 28 +'ث' => 29 +'خ' => 30 +'ڨ' => 31 +'ذ' => 32 +'ظ' => 33 +'ء' => 34 +'غ' => 35 +'ئ' => 36 +'ؤ' => 37 +'' => 0 +1 => 39 +================ +'starting_index' => 0 +'unk_label' => 1 +'blank_label' => '' diff --git a/TunisianASR/train_semi.yaml b/TunisianASR/train_semi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0902d0658b1d210ba1dfa81267f9ee64d4c4a6ee --- /dev/null +++ b/TunisianASR/train_semi.yaml @@ -0,0 +1,175 @@ +# ################################ +# Model: wav2vec2 + DNN + CTC +# Augmentation: SpecAugment +# Authors: Titouan Parcollet 2021 +# ################################ + +# Seed needs to be set at top of yaml, before objects with parameters are made +seed: 1234 +__set_seed: !!python/object/apply:torch.manual_seed [!ref ] +output_folder: !ref semi_wavlm_large_tunisian_ctc/ +wer_file: !ref /wer.txt +save_folder: !ref /save +train_log: !ref /train_log.txt + +# URL for the biggest LeBenchmark wav2vec french. +wav2vec2_folder: !ref /wav2vec2_checkpoint + +# Data files +data_folder: /path/to/data # e.g, /localscratch/cv-corpus-5.1-2020-06-22/fr +train_tsv_file: !ref /train.tsv # Standard CommonVoice .tsv files +dev_tsv_file: !ref /dev.tsv # Standard CommonVoice .tsv files +test_tsv_file: !ref /test.tsv # Standard CommonVoice .tsv files +accented_letters: True +language: fr # use 'it' for Italian, 'rw' for Kinyarwanda, 'en' for english +test_csv: + - /path/to/test_data + +skip_prep: True # Skip data preparation + +use_language_modelling: True +ngram_lm_path: outdomain.arpa + +# We remove utterance slonger than 10s in the train/dev/test sets as +# longer sentences certainly correspond to "open microphones". +avoid_if_longer_than: 10.0 +avoid_if_shorter_than: 1.2 + + +# Training parameters +number_of_epochs: 12 +lr: 1.0 +lr_wav2vec: 0.0001 +sorting: ascending +auto_mix_prec: False +sample_rate: 16000 +ckpt_interval_minutes: 30 # save checkpoint every N min + +# With data_parallel batch_size is split into N jobs +# With DDP batch_size is multiplied by N jobs +# Must be 6 per GPU to fit 16GB of VRAM +batch_size: 10 +test_batch_size: 4 + +dataloader_options: + batch_size: !ref + num_workers: 6 +test_dataloader_options: + batch_size: !ref + num_workers: 6 + +# BPE parameters +token_type: char # ["unigram", "bpe", "char"] +character_coverage: 1.0 + +# Model parameters +# activation: !name:torch.nn.LeakyReLU +wav2vec_output_dim: 1024 +dnn_neurons: 1024 +freeze_wav2vec: False +freeze_feature_extractor: True +dropout: 0.15 +warmup_steps: 500 # The wav2vec 2 model isn't updated for this amount of steps + +# Outputs +output_neurons: 40 # BPE size, index(blank/eos/bos) = 0 + +# Decoding parameters +# Be sure that the bos and eos index match with the BPEs ones +blank_index: 0 +unk_index: 1 + +# +# Functions and classes +# +epoch_counter: !new:speechbrain.utils.epoch_loop.EpochCounter + limit: !ref + +augmentation: !new:speechbrain.lobes.augment.TimeDomainSpecAugment + sample_rate: !ref + speeds: [95, 100, 105] + +enc: !new:speechbrain.nnet.containers.Sequential + input_shape: [null, null, !ref ] + linear1: !name:speechbrain.nnet.linear.Linear + n_neurons: !ref + bias: True + bn1: !name:speechbrain.nnet.normalization.BatchNorm1d + activation: !new:torch.nn.LeakyReLU + drop: !new:torch.nn.Dropout + p: !ref + linear2: !name:speechbrain.nnet.linear.Linear + n_neurons: !ref + bias: True + bn2: !name:speechbrain.nnet.normalization.BatchNorm1d + activation2: !new:torch.nn.LeakyReLU + drop2: !new:torch.nn.Dropout + p: !ref + linear3: !name:speechbrain.nnet.linear.Linear + n_neurons: !ref + bias: True + bn3: !name:speechbrain.nnet.normalization.BatchNorm1d + activation3: !new:torch.nn.LeakyReLU + +wav2vec2: !new:speechbrain.lobes.models.huggingface_wav2vec.HuggingFaceWav2Vec2 + source: /gpfsstore/rech/nou/uzn19yk/wavlm/ + output_norm: False + freeze: !ref + freeze_feature_extractor: !ref + save_path: !ref + + +ctc_lin: !new:speechbrain.nnet.linear.Linear + input_size: !ref + n_neurons: !ref + +log_softmax: !new:speechbrain.nnet.activations.Softmax + apply_log: True + +ctc_cost: !name:speechbrain.nnet.losses.ctc_loss + blank_index: !ref + +modules: + wav2vec2: !ref + enc: !ref + ctc_lin: !ref + +model: !new:torch.nn.ModuleList + - [!ref , !ref ] + +model_opt_class: !name:torch.optim.Adadelta + lr: !ref + rho: 0.95 + eps: 1.e-8 + +wav2vec_opt_class: !name:torch.optim.Adam + lr: !ref + +lr_annealing_model: !new:speechbrain.nnet.schedulers.NewBobScheduler + initial_value: !ref + improvement_threshold: 0.0025 + annealing_factor: 0.8 + patient: 0 + +lr_annealing_wav2vec: !new:speechbrain.nnet.schedulers.NewBobScheduler + initial_value: !ref + improvement_threshold: 0.0025 + annealing_factor: 0.9 + patient: 0 + +checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer + checkpoints_dir: !ref + recoverables: + wav2vec2: !ref + model: !ref + scheduler_model: !ref + scheduler_wav2vec: !ref + counter: !ref + +train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger + save_file: !ref + +error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats + +cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats + split_tokens: True diff --git a/TunisianASR/train_with_wavlm.py b/TunisianASR/train_with_wavlm.py new file mode 100644 index 0000000000000000000000000000000000000000..5d6ca4c5a378583fd297e1202522b9dc9c2368de --- /dev/null +++ b/TunisianASR/train_with_wavlm.py @@ -0,0 +1,399 @@ +#!/usr/bin/env python3 +import sys +import torch +import logging +import speechbrain as sb +from pathlib import Path +import os +import torchaudio +from hyperpyyaml import load_hyperpyyaml +from speechbrain.tokenizers.SentencePiece import SentencePiece +from speechbrain.utils.data_utils import undo_padding +from speechbrain.utils.distributed import run_on_main + +"""Recipe for training a sequence-to-sequence ASR system with CommonVoice. +The system employs a wav2vec2 encoder and a CTC decoder. +Decoding is performed with greedy decoding (will be extended to beam search). + +To run this recipe, do the following: +> python train_with_wav2vec2.py hparams/train_with_wav2vec2.yaml + +With the default hyperparameters, the system employs a pretrained wav2vec2 encoder. +The wav2vec2 model is pretrained following the model given in the hprams file. +It may be dependent on the language. + +The neural network is trained with CTC on sub-word units estimated with +Byte Pairwise Encoding (BPE). + +The experiment file is flexible enough to support a large variety of +different systems. By properly changing the parameter files, you can try +different encoders, decoders, tokens (e.g, characters instead of BPE), +training languages (all CommonVoice languages), and many +other possible variations. + +Authors + * Titouan Parcollet 2021 +""" + +logger = logging.getLogger(__name__) + + +# Define training procedure +class ASR(sb.core.Brain): + def compute_forward(self, batch, stage): + """Forward computations from the waveform batches to the output probabilities.""" + + batch = batch.to(self.device) + wavs, wav_lens = batch.sig + wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device) + if stage == sb.Stage.TRAIN: + if hasattr(self.hparams, "augmentation"): + wavs = self.hparams.augmentation(wavs, wav_lens) + + # Forward pass + feats = self.modules.wav2vec2(wavs, wav_lens) + x = self.modules.enc(feats) + logits = self.modules.ctc_lin(x) + p_ctc = self.hparams.log_softmax(logits) + + return p_ctc, wav_lens + + def compute_objectives(self, predictions, batch, stage): + """Computes the loss (CTC) given predictions and targets.""" + + p_ctc, wav_lens = predictions + + ids = batch.id + tokens, tokens_lens = batch.tokens + + loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens) + + if stage != sb.Stage.TRAIN: + predicted_tokens = sb.decoders.ctc_greedy_decode( + p_ctc, wav_lens, blank_id=self.hparams.blank_index + ) + # Decode token terms to words + if self.hparams.use_language_modelling: + predicted_words = [] + for logs in p_ctc: + text = decoder.decode(logs.detach().cpu().numpy()) + predicted_words.append(text.split(" ")) + else: + predicted_words = [ + "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ") + for utt_seq in predicted_tokens + ] + # Convert indices to words + target_words = [wrd.split(" ") for wrd in batch.wrd] + + self.wer_metric.append(ids, predicted_words, target_words) + self.cer_metric.append(ids, predicted_words, target_words) + + return loss + + def fit_batch(self, batch): + """Train the parameters given a single batch in input""" + should_step = self.step % self.grad_accumulation_factor == 0 + # Managing automatic mixed precision + # TOFIX: CTC fine-tuning currently is unstable + # This is certainly due to CTC being done in fp16 instead of fp32 + if self.auto_mix_prec: + with torch.cuda.amp.autocast(): + with self.no_sync(): + outputs = self.compute_forward(batch, sb.Stage.TRAIN) + loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN) + with self.no_sync(not should_step): + self.scaler.scale( + loss / self.grad_accumulation_factor + ).backward() + if should_step: + + if not self.hparams.wav2vec2.freeze: + self.scaler.unscale_(self.wav2vec_optimizer) + self.scaler.unscale_(self.model_optimizer) + if self.check_gradients(loss): + if not self.hparams.wav2vec2.freeze: + if self.optimizer_step >= self.hparams.warmup_steps: + self.scaler.step(self.wav2vec_optimizer) + self.scaler.step(self.model_optimizer) + self.scaler.update() + self.zero_grad() + self.optimizer_step += 1 + else: + # This is mandatory because HF models have a weird behavior with DDP + # on the forward pass + with self.no_sync(): + outputs = self.compute_forward(batch, sb.Stage.TRAIN) + + loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN) + + with self.no_sync(not should_step): + (loss / self.grad_accumulation_factor).backward() + if should_step: + if self.check_gradients(loss): + if not self.hparams.wav2vec2.freeze: + if self.optimizer_step >= self.hparams.warmup_steps: + self.wav2vec_optimizer.step() + self.model_optimizer.step() + self.zero_grad() + self.optimizer_step += 1 + + self.on_fit_batch_end(batch, outputs, loss, should_step) + return loss.detach().cpu() + + def evaluate_batch(self, batch, stage): + """Computations needed for validation/test batches""" + predictions = self.compute_forward(batch, stage=stage) + with torch.no_grad(): + loss = self.compute_objectives(predictions, batch, stage=stage) + return loss.detach() + + def on_stage_start(self, stage, epoch): + """Gets called at the beginning of each epoch""" + if stage != sb.Stage.TRAIN: + self.cer_metric = self.hparams.cer_computer() + self.wer_metric = self.hparams.error_rate_computer() + + def on_stage_end(self, stage, stage_loss, epoch): + """Gets called at the end of an epoch.""" + # Compute/store important stats + stage_stats = {"loss": stage_loss} + if stage == sb.Stage.TRAIN: + self.train_stats = stage_stats + else: + stage_stats["CER"] = self.cer_metric.summarize("error_rate") + stage_stats["WER"] = self.wer_metric.summarize("error_rate") + + # Perform end-of-iteration things, like annealing, logging, etc. + if stage == sb.Stage.VALID: + old_lr_model, new_lr_model = self.hparams.lr_annealing_model( + stage_stats["loss"] + ) + old_lr_wav2vec, new_lr_wav2vec = self.hparams.lr_annealing_wav2vec( + stage_stats["loss"] + ) + sb.nnet.schedulers.update_learning_rate( + self.model_optimizer, new_lr_model + ) + if not self.hparams.wav2vec2.freeze: + sb.nnet.schedulers.update_learning_rate( + self.wav2vec_optimizer, new_lr_wav2vec + ) + self.hparams.train_logger.log_stats( + stats_meta={ + "epoch": epoch, + "lr_model": old_lr_model, + "lr_wav2vec": old_lr_wav2vec, + }, + train_stats=self.train_stats, + valid_stats=stage_stats, + ) + self.checkpointer.save_and_keep_only( + meta={"WER": stage_stats["WER"]}, min_keys=["WER"], + ) + elif stage == sb.Stage.TEST: + self.hparams.train_logger.log_stats( + stats_meta={"Epoch loaded": self.hparams.epoch_counter.current}, + test_stats=stage_stats, + ) + with open(self.hparams.wer_file, "w") as w: + self.wer_metric.write_stats(w) + + def init_optimizers(self): + "Initializes the wav2vec2 optimizer and model optimizer" + + # If the wav2vec encoder is unfrozen, we create the optimizer + if not self.hparams.wav2vec2.freeze: + self.wav2vec_optimizer = self.hparams.wav2vec_opt_class( + self.modules.wav2vec2.parameters() + ) + if self.checkpointer is not None: + self.checkpointer.add_recoverable( + "wav2vec_opt", self.wav2vec_optimizer + ) + + self.model_optimizer = self.hparams.model_opt_class( + self.hparams.model.parameters() + ) + + if self.checkpointer is not None: + self.checkpointer.add_recoverable("modelopt", self.model_optimizer) + + def zero_grad(self, set_to_none=False): + if not self.hparams.wav2vec2.freeze: + self.wav2vec_optimizer.zero_grad(set_to_none) + self.model_optimizer.zero_grad(set_to_none) + + +# Define custom data procedure +def dataio_prepare(hparams): + """This function prepares the datasets to be used in the brain class. + It also defines the data processing pipeline through user-defined functions.""" + + # 1. Define datasets + data_folder = hparams["data_folder"] + + train_data = sb.dataio.dataset.DynamicItemDataset.from_csv( + csv_path=hparams["train_csv"], replacements={"data_root": data_folder}, + ) + + if hparams["sorting"] == "ascending": + # we sort training data to speed up training and get better results. + train_data = train_data.filtered_sorted( + sort_key="duration", + key_max_value={"duration": hparams["avoid_if_longer_than"]}, + ) + # when sorting do not shuffle in dataloader ! otherwise is pointless + hparams["dataloader_options"]["shuffle"] = False + + elif hparams["sorting"] == "descending": + train_data = train_data.filtered_sorted( + sort_key="duration", + reverse=True, + key_max_value={"duration": hparams["avoid_if_longer_than"]}, + ) + # when sorting do not shuffle in dataloader ! otherwise is pointless + hparams["dataloader_options"]["shuffle"] = False + + elif hparams["sorting"] == "random": + pass + + else: + raise NotImplementedError( + "sorting must be random, ascending or descending" + ) + + valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv( + csv_path=hparams["valid_csv"], replacements={"data_root": data_folder}, + ) + # We also sort the validation data so it is faster to validate + valid_data = valid_data.filtered_sorted(sort_key="duration") + test_datasets = {} + for csv_file in hparams["test_csv"]: + name = Path(csv_file).stem + test_datasets[name] = sb.dataio.dataset.DynamicItemDataset.from_csv( + csv_path=csv_file, replacements={"data_root": data_folder} + ) + test_datasets[name] = test_datasets[name].filtered_sorted( + sort_key="duration" + ) + + datasets = [train_data, valid_data] + [i for k, i in test_datasets.items()] + + + # 2. Define audio pipeline: + @sb.utils.data_pipeline.takes("wav") + @sb.utils.data_pipeline.provides("sig") + def audio_pipeline(wav): + info = torchaudio.info(wav) + sig = sb.dataio.dataio.read_audio(wav) + resampled = torchaudio.transforms.Resample( + info.sample_rate, hparams["sample_rate"], + )(sig) + return resampled + + sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline) + label_encoder = sb.dataio.encoder.CTCTextEncoder() + + # 3. Define text pipeline: + @sb.utils.data_pipeline.takes("wrd") + @sb.utils.data_pipeline.provides( + "wrd", "char_list", "tokens_list", "tokens" + ) + def text_pipeline(wrd): + yield wrd + char_list = list(wrd) + yield char_list + tokens_list = label_encoder.encode_sequence(char_list) + yield tokens_list + tokens = torch.LongTensor(tokens_list) + yield tokens + + sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline) + lab_enc_file = os.path.join(hparams["save_folder"], "label_encoder.txt") + special_labels = { + "blank_label": hparams["blank_index"], + "unk_label": hparams["unk_index"] + } + label_encoder.load_or_create( + path=lab_enc_file, + from_didatasets=[train_data], + output_key="char_list", + special_labels=special_labels, + sequence_input=True, + ) + + # 4. Set output: + sb.dataio.dataset.set_output_keys( + datasets, ["id", "sig", "wrd", "char_list", "tokens"], + ) + return train_data, valid_data,test_datasets, label_encoder + + +if __name__ == "__main__": + + # Load hyperparameters file with command-line overrides + hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:]) + with open(hparams_file) as fin: + hparams = load_hyperpyyaml(fin, overrides) + + # If --distributed_launch then + # create ddp_group with the right communication protocol + sb.utils.distributed.ddp_init_group(run_opts) + + + # Create experiment directory + sb.create_experiment_directory( + experiment_directory=hparams["output_folder"], + hyperparams_to_save=hparams_file, + overrides=overrides, + ) + + # Due to DDP, we do the preparation ONLY on the main python process + # Defining tokenizer and loading it + # Create the datasets objects as well as tokenization and encoding :-D + train_data, valid_data, test_datasets, label_encoder = dataio_prepare(hparams) + if hparams["use_language_modelling"]: + print("using langauge_modeeling") + from pyctcdecode import build_ctcdecoder + ind2lab = label_encoder.ind2lab + print(ind2lab) + labels = [ind2lab[x] for x in range(len(ind2lab))] + labels = [""] + labels[1:-1] + ["1"] + # Replace the token with a blank character, needed for PyCTCdecode + print(labels) + decoder = build_ctcdecoder( + labels, + kenlm_model_path=hparams["ngram_lm_path"], # .arpa or .bin + alpha=0.5, # Default by KenLM + beta=1.0, # Default by KenLM + ) + # Trainer initialization + asr_brain = ASR( + modules=hparams["modules"], + hparams=hparams, + run_opts=run_opts, + checkpointer=hparams["checkpointer"], + ) + + # Adding objects to trainer. + asr_brain.tokenizer = label_encoder + + # Training + asr_brain.fit( + asr_brain.hparams.epoch_counter, + train_data, + valid_data, + train_loader_kwargs=hparams["dataloader_options"], + valid_loader_kwargs=hparams["test_dataloader_options"], + ) + + # Test + for k in test_datasets.keys(): # keys are test_clean, test_other etc + asr_brain.hparams.wer_file = os.path.join( + hparams["output_folder"], "wer_{}.txt".format(k) + ) + asr_brain.evaluate( + test_datasets[k], test_loader_kwargs=hparams["test_dataloader_options"] + ) + diff --git a/arpas/everything.arpa b/arpas/everything.arpa new file mode 100644 index 0000000000000000000000000000000000000000..709c25e7e6f7ddb0a0ae621cddc54ec54aaab154 --- /dev/null +++ b/arpas/everything.arpa @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bada7d41f63b1e5fd661ba66bccdfa93c3e5c391038ac6e52615a42ec0e0174 +size 345991397 diff --git a/asr-wav2vec2-commonvoice-fr/README.md b/asr-wav2vec2-commonvoice-fr/README.md new file mode 100644 index 0000000000000000000000000000000000000000..190108e53a978dbfd19c845e39ab14fa0f6229ee --- /dev/null +++ b/asr-wav2vec2-commonvoice-fr/README.md @@ -0,0 +1,130 @@ +--- +language: +- fr +thumbnail: null +pipeline_tag: automatic-speech-recognition +tags: +- CTC +- pytorch +- speechbrain +- Transformer +- hf-asr-leaderboard +license: apache-2.0 +datasets: +- commonvoice +metrics: +- wer +- cer +model-index: +- name: asr-wav2vec2-commonvoice-fr + results: + - task: + name: Automatic Speech Recognition + type: automatic-speech-recognition + dataset: + name: CommonVoice 6.1 (French) + type: mozilla-foundation/common_voice_6_1 + config: fr + split: test + args: + language: fr + metrics: + - name: Test WER + type: wer + value: '9.96' +--- + + +

+ +# wav2vec 2.0 with CTC/Attention trained on CommonVoice French (No LM) + +This repository provides all the necessary tools to perform automatic speech +recognition from an end-to-end system pretrained on CommonVoice (French Language) within +SpeechBrain. For a better experience, we encourage you to learn more about +[SpeechBrain](https://speechbrain.github.io). + +The performance of the model is the following: + +| Release | Test CER | Test WER | GPUs | +|:-------------:|:--------------:|:--------------:| :--------:| +| 24-08-21 | 3.19 | 9.96 | 2xV100 32GB | + +## Pipeline description + +This ASR system is composed of 2 different but linked blocks: +- Tokenizer (unigram) that transforms words into subword units and trained with +the train transcriptions (train.tsv) of CommonVoice (FR). +- Acoustic model (wav2vec2.0 + CTC). A pretrained wav2vec 2.0 model ([LeBenchmark/wav2vec2-FR-7K-large](https://huggingface.co/LeBenchmark/wav2vec2-FR-7K-large)) is combined with two DNN layers and finetuned on CommonVoice FR. +The obtained final acoustic representation is given to the CTC greedy decoder. + +The system is trained with recordings sampled at 16kHz (single channel). +The code will automatically normalize your audio (i.e., resampling + mono channel selection) when calling *transcribe_file* if needed. + +## Install SpeechBrain + +First of all, please install tranformers and SpeechBrain with the following command: + +``` +pip install speechbrain transformers +``` + +Please notice that we encourage you to read our tutorials and learn more about +[SpeechBrain](https://speechbrain.github.io). + +### Transcribing your own audio files (in French) + +```python +from speechbrain.pretrained import EncoderASR + +asr_model = EncoderASR.from_hparams(source="speechbrain/asr-wav2vec2-commonvoice-fr", savedir="pretrained_models/asr-wav2vec2-commonvoice-fr") +asr_model.transcribe_file('speechbrain/asr-wav2vec2-commonvoice-fr/example-fr.wav') + +``` +### Inference on GPU +To perform inference on the GPU, add `run_opts={"device":"cuda"}` when calling the `from_hparams` method. + +### Training +The model was trained with SpeechBrain. +To train it from scratch follow these steps: +1. Clone SpeechBrain: +```bash +git clone https://github.com/speechbrain/speechbrain/ +``` +2. Install it: +```bash +cd speechbrain +pip install -r requirements.txt +pip install -e . +``` + +3. Run Training: +```bash +cd recipes/CommonVoice/ASR/CTC/ +python train_with_wav2vec.py hparams/train_fr_with_wav2vec.yaml --data_folder=your_data_folder +``` + +You can find our training results (models, logs, etc) [here](https://drive.google.com/drive/folders/1T9DfdZwcNI9CURxhLCi8GA5JVz8adiY8?usp=sharing). + +### Limitations +The SpeechBrain team does not provide any warranty on the performance achieved by this model when used on other datasets. + +#### Referencing SpeechBrain + +``` +@misc{SB2021, + author = {Ravanelli, Mirco and Parcollet, Titouan and Rouhe, Aku and Plantinga, Peter and Rastorgueva, Elena and Lugosch, Loren and Dawalatabad, Nauman and Ju-Chieh, Chou and Heba, Abdel and Grondin, Francois and Aris, William and Liao, Chien-Feng and Cornell, Samuele and Yeh, Sung-Lin and Na, Hwidong and Gao, Yan and Fu, Szu-Wei and Subakan, Cem and De Mori, Renato and Bengio, Yoshua }, + title = {SpeechBrain}, + year = {2021}, + publisher = {GitHub}, + journal = {GitHub repository}, + howpublished = {\\\\url{https://github.com/speechbrain/speechbrain}}, + } +``` + +#### About SpeechBrain +SpeechBrain is an open-source and all-in-one speech toolkit. It is designed to be simple, extremely flexible, and user-friendly. Competitive or state-of-the-art performance is obtained in various domains. + +Website: https://speechbrain.github.io/ + +GitHub: https://github.com/speechbrain/speechbrain diff --git a/asr-wav2vec2-commonvoice-fr/asr.ckpt b/asr-wav2vec2-commonvoice-fr/asr.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..cbe4d503e40e0ab6e1635e09ce94dffb9a5d1482 --- /dev/null +++ b/asr-wav2vec2-commonvoice-fr/asr.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64ba475ed7be735d4ac054c2d537f22251b80f6ecb65cb04217eb0d1ed50a143 +size 12963902 diff --git a/asr-wav2vec2-commonvoice-fr/config.json b/asr-wav2vec2-commonvoice-fr/config.json new file mode 100644 index 0000000000000000000000000000000000000000..7149b1f7ff0596ec873378ea9ccbf9811f16cd5b --- /dev/null +++ b/asr-wav2vec2-commonvoice-fr/config.json @@ -0,0 +1,76 @@ +{ + "speechbrain_interface": "EncoderASR", + "activation_dropout": 0.0, + "apply_spec_augment": true, + "architectures": [ + "Wav2Vec2Model" + ], + "attention_dropout": 0.1, + "bos_token_id": 1, + "conv_bias": true, + "conv_dim": [ + 512, + 512, + 512, + 512, + 512, + 512, + 512 + ], + "conv_kernel": [ + 10, + 3, + 3, + 3, + 3, + 2, + 2 + ], + "conv_stride": [ + 5, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "ctc_loss_reduction": "sum", + "ctc_zero_infinity": false, + "do_stable_layer_norm": true, + "eos_token_id": 2, + "feat_extract_activation": "gelu", + "feat_extract_dropout": 0.0, + "feat_extract_norm": "layer", + "feat_proj_dropout": 0.1, + "final_dropout": 0.0, + "gradient_checkpointing": false, + "hidden_act": "gelu", + "hidden_dropout": 0.1, + "hidden_size": 1024, + "initializer_range": 0.02, + "intermediate_size": 4096, + "layer_norm_eps": 1e-05, + "layerdrop": 0.1, + "mask_channel_length": 10, + "mask_channel_min_space": 1, + "mask_channel_other": 0.0, + "mask_channel_prob": 0.0, + "mask_channel_selection": "static", + "mask_feature_length": 10, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_space": 1, + "mask_time_other": 0.0, + "mask_time_prob": 0.075, + "mask_time_selection": "static", + "model_type": "wav2vec2", + "num_attention_heads": 16, + "num_conv_pos_embedding_groups": 16, + "num_conv_pos_embeddings": 128, + "num_feat_extract_layers": 7, + "num_hidden_layers": 24, + "pad_token_id": 0, + "transformers_version": "4.5.1", + "vocab_size": 32 +} diff --git a/asr-wav2vec2-commonvoice-fr/example-fr.wav b/asr-wav2vec2-commonvoice-fr/example-fr.wav new file mode 100644 index 0000000000000000000000000000000000000000..709c2426101b93ce09d033eac48a56efe1a79e99 Binary files /dev/null and b/asr-wav2vec2-commonvoice-fr/example-fr.wav differ diff --git a/asr-wav2vec2-commonvoice-fr/example.wav b/asr-wav2vec2-commonvoice-fr/example.wav new file mode 100644 index 0000000000000000000000000000000000000000..709c2426101b93ce09d033eac48a56efe1a79e99 Binary files /dev/null and b/asr-wav2vec2-commonvoice-fr/example.wav differ diff --git a/asr-wav2vec2-commonvoice-fr/hyperparams.yaml b/asr-wav2vec2-commonvoice-fr/hyperparams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6b7d4a4b0725ec7bc240e80de26916b4737e75d --- /dev/null +++ b/asr-wav2vec2-commonvoice-fr/hyperparams.yaml @@ -0,0 +1,96 @@ +# ################################ +# Model: wav2vec2 + DNN + CTC/Attention +# Augmentation: SpecAugment +# Authors: Titouan Parcollet 2021 +# ################################ + +sample_rate: 16000 +wav2vec2_hub: /gpfsscratch/rech/nou/uzn19yk/wav2vec2-FR-7K-large + +# BPE parameters +token_type: unigram # ["unigram", "bpe", "char"] +character_coverage: 1.0 + +# Model parameters +activation: !name:torch.nn.LeakyReLU +dnn_layers: 2 +dnn_neurons: 1024 +emb_size: 128 +dec_neurons: 1024 + +# Outputs +output_neurons: 76 # BPE size, index(blank/eos/bos) = 0 + +# Decoding parameters +# Be sure that the bos and eos index match with the BPEs ones +blank_index: 0 +bos_index: 1 +eos_index: 2 +min_decode_ratio: 0.0 +max_decode_ratio: 1.0 +beam_size: 80 +eos_threshold: 1.5 +using_max_attn_shift: True +max_attn_shift: 140 +ctc_weight_decode: 0.0 +temperature: 1.50 + +enc: !new:speechbrain.nnet.containers.Sequential + input_shape: [null, null, 1024] + linear1: !name:speechbrain.nnet.linear.Linear + n_neurons: 1024 + bias: True + bn1: !name:speechbrain.nnet.normalization.BatchNorm1d + activation: !new:torch.nn.LeakyReLU + drop: !new:torch.nn.Dropout + p: 0.15 + linear2: !name:speechbrain.nnet.linear.Linear + n_neurons: 1024 + bias: True + bn2: !name:speechbrain.nnet.normalization.BatchNorm1d + activation2: !new:torch.nn.LeakyReLU + drop2: !new:torch.nn.Dropout + p: 0.15 + linear3: !name:speechbrain.nnet.linear.Linear + n_neurons: 1024 + bias: True + bn3: !name:speechbrain.nnet.normalization.BatchNorm1d + activation3: !new:torch.nn.LeakyReLU + +wav2vec2: !new:speechbrain.lobes.models.huggingface_wav2vec.HuggingFaceWav2Vec2 + source: !ref + output_norm: True + freeze: True + save_path: model_checkpoints + +ctc_lin: !new:speechbrain.nnet.linear.Linear + input_size: !ref + n_neurons: !ref + +log_softmax: !new:speechbrain.nnet.activations.Softmax + apply_log: True + +ctc_cost: !name:speechbrain.nnet.losses.ctc_loss + blank_index: !ref + +asr_model: !new:torch.nn.ModuleList + - [!ref , !ref ] + +tokenizer: !new:sentencepiece.SentencePieceProcessor + +encoder: !new:speechbrain.nnet.containers.LengthsCapableSequential + wav2vec2: !ref + enc: !ref + ctc_lin: !ref + +decoding_function: !name:speechbrain.decoders.ctc_greedy_decode + blank_id: !ref + +modules: + encoder: !ref + +pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer + loadables: + wav2vec2: !ref + asr: !ref + tokenizer: !ref diff --git a/asr-wav2vec2-commonvoice-fr/preprocessor_config.json b/asr-wav2vec2-commonvoice-fr/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..0886a48276922a77013d8aa4681192138ae90d90 --- /dev/null +++ b/asr-wav2vec2-commonvoice-fr/preprocessor_config.json @@ -0,0 +1,8 @@ +{ + "do_normalize": true, + "feature_size": 1, + "padding_side": "right", + "padding_value": 0.0, + "return_attention_mask": true, + "sampling_rate": 16000 +} diff --git a/asr-wav2vec2-commonvoice-fr/tokenizer.ckpt b/asr-wav2vec2-commonvoice-fr/tokenizer.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..2dbbdb6556765e96ffed7b9c38a6f3bfb5113b8a --- /dev/null +++ b/asr-wav2vec2-commonvoice-fr/tokenizer.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e39b467b62ca9e9fd3b4a108bb45a9d164a896709756e300bbe57d7fea073f5a +size 238487 diff --git a/asr-wav2vec2-commonvoice-fr/wav2vec2.ckpt b/asr-wav2vec2-commonvoice-fr/wav2vec2.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b5ec86eda90dfef744b5953c7765463125dd8d3a --- /dev/null +++ b/asr-wav2vec2-commonvoice-fr/wav2vec2.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b2d9f900fd7a57a30bdc6220606f1ccf37f582f07aab7a5b75213ac46c38204 +size 1261930757 diff --git a/cs.yaml b/cs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e9b5880c47834836b35921f7a11d5f7944f9829 --- /dev/null +++ b/cs.yaml @@ -0,0 +1,141 @@ +# Generated 2023-08-03 from: +# /home/salah/new_tunisian_model/hparams/train_tunisian_withwavlm.yaml +# yamllint disable +# ################################ +# Model: wav2vec2 + DNN + CTC +# Augmentation: SpecAugment +# Authors: Titouan Parcollet 2021 +# ################################ + +seed: 1994 +__set_seed: !!python/object/apply:torch.manual_seed [1234] +output_folder: results/non_semi_final_stac +wer_file: !ref /wer.txt +save_folder: !ref /save +train_log: !ref /train_log.txt + + + +# Data files +data_folder: junk # e.g, /localscratch/cv-corpus-5.1-2020-06-22/fr +train_tsv_file: junk/train.tsv # Standard CommonVoice .tsv files +dev_tsv_file: junk/dev.tsv # Standard CommonVoice .tsv files +test_tsv_file: junk/test.tsv # Standard CommonVoice .tsv files +accented_letters: true + +csv_folder: /gpfsscratch/rech/nou/uzn19yk/switched_data/extended_clean/ +train_csv: !ref /train.csv +valid_csv: !ref /dev.csv +test_csv: + - all_tests/cs_test.csv + - all_tests/stac_test.csv + +# We remove utterance slonger than 10s in the train/dev/test sets as +# longer sentences certainly correspond to "open microphones". +avoid_if_longer_than: 13.0 +avoid_if_shorter_than: 0.5 + +# Training parameters +number_of_epochs: 20 +lr: 0.0002 +lr_weights: 0.01 +sorting: ascending +auto_mix_prec: False +sample_rate: 16000 +language_modelling: True +ngram_lm_path: /gpfsstore/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/arpas/pluslanguages_everything.arpa + +# With data_parallel batch_size is split into N jobs +# With DDP batch_size is multiplied by N jobs +# Must be 3 per GPU to fit 32GB of VRAM +batch_size: 3 +test_batch_size: 4 + +# Dataloader options +dataloader_options: + batch_size: !ref + num_workers: 6 + +test_dataloader_options: + batch_size: !ref + num_workers: 6 + +# Model parameters +activation: !name:torch.nn.Sigmoid +dnn_layers: 1 +dnn_neurons: 768 +freeze_encoder: True + +# Outputs +output_neurons: 76 # BPE size, index(blank/eos/bos) = 0 + +# Functions and classes +# +epoch_counter: !new:speechbrain.utils.epoch_loop.EpochCounter + limit: !ref + +encoder_dim: 3217 +enc: !new:speechbrain.nnet.RNN.LSTM + input_shape: [Null, Null, !ref ] + num_layers: 2 + bidirectional: True + dropout: 0.2 + hidden_size: 1024 + +ctc_lin: !new:speechbrain.nnet.linear.Linear + input_size: 2048 + n_neurons: !ref + +log_softmax: !new:speechbrain.nnet.activations.Softmax + apply_log: True + +ctc_cost: !name:speechbrain.nnet.losses.ctc_loss + blank_index: !ref + +modules: + enc: !ref + ctc_lin: !ref + +model: !new:torch.nn.ModuleList + - [!ref , !ref ] + +model_opt_class: !name:torch.optim.Adam + lr: !ref + +weights_opt_class: !name:torch.optim.Adam + lr: !ref + +lr_annealing_model: !new:speechbrain.nnet.schedulers.NewBobScheduler + initial_value: !ref + improvement_threshold: 0.0025 + annealing_factor: 0.8 + patient: 0 + +lr_annealing_weights: !new:speechbrain.nnet.schedulers.NewBobScheduler + initial_value: !ref + improvement_threshold: 0.0025 + annealing_factor: 0.9 + patient: 0 + +label_encoder: !new:speechbrain.dataio.encoder.CTCTextEncoder + +checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer + checkpoints_dir: !ref + recoverables: + model: !ref + scheduler_model: !ref + scheduler_encoder: !ref + counter: !ref + tokenizer: !ref + +blank_index: 0 +unk_index: 1 + + +train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger + save_file: !ref + +error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats + +cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats + split_tokens: True diff --git a/train_mixer.py b/train_mixer.py new file mode 100644 index 0000000000000000000000000000000000000000..acac2a1e16daad18c2c182751872998cbe2c468b --- /dev/null +++ b/train_mixer.py @@ -0,0 +1,698 @@ +# -*- coding: utf-8 -*- + +import os +import sys +import torch +import logging +import speechbrain as sb +from speechbrain.utils.distributed import run_on_main +from hyperpyyaml import load_hyperpyyaml +from pathlib import Path +import torchaudio.transforms as T +from cv_train import ASRCV +import torchaudio +import numpy as np +import kenlm +from pyctcdecode import build_ctcdecoder +import re +from torch.nn.utils.rnn import pad_sequence +import torch.optim as optim +import torch.nn as nn + + +# Commented out IPython magic to ensure Python compatibility. +hparams_file, run_opts, overrides = sb.parse_arguments(["hparams/train_semi.yaml"]) + +# If distributed_launch=True then +# create ddp_group with the right communication protocol +sb.utils.distributed.ddp_init_group(run_opts) + +with open(hparams_file) as fin: + hparams = load_hyperpyyaml(fin, overrides) + +# Create experiment directory +sb.create_experiment_directory( + experiment_directory=hparams["output_folder"], + hyperparams_to_save=hparams_file, + overrides=overrides, +) +# Dataset prep (parsing Librispeech) + +def dataio_prepare(hparams): + """This function prepares the datasets to be used in the brain class. + It also defines the data processing pipeline through user-defined functions.""" + + # 1. Define datasets + data_folder = hparams["data_folder"] + + train_data = sb.dataio.dataset.DynamicItemDataset.from_csv( + csv_path=hparams["train_csv"], replacements={"data_root": data_folder}, + ) + + if hparams["sorting"] == "ascending": + # we sort training data to speed up training and get better results. + train_data = train_data.filtered_sorted( + sort_key="duration", + key_max_value={"duration": hparams["avoid_if_longer_than"]}, + ) + # when sorting do not shuffle in dataloader ! otherwise is pointless + hparams["dataloader_options"]["shuffle"] = False + + elif hparams["sorting"] == "descending": + train_data = train_data.filtered_sorted( + sort_key="duration", + reverse=True, + key_max_value={"duration": hparams["avoid_if_longer_than"]}, + ) + # when sorting do not shuffle in dataloader ! otherwise is pointless + hparams["dataloader_options"]["shuffle"] = False + + elif hparams["sorting"] == "random": + pass + + else: + raise NotImplementedError( + "sorting must be random, ascending or descending" + ) + + valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv( + csv_path=hparams["valid_csv"], replacements={"data_root": data_folder}, + ) + # We also sort the validation data so it is faster to validate + valid_data = valid_data.filtered_sorted(sort_key="duration") + test_datasets = {} + for csv_file in hparams["test_csv"]: + name = Path(csv_file).stem + test_datasets[name] = sb.dataio.dataset.DynamicItemDataset.from_csv( + csv_path=csv_file, replacements={"data_root": data_folder} + ) + test_datasets[name] = test_datasets[name].filtered_sorted( + sort_key="duration" + ) + + datasets = [train_data, valid_data] + [i for k, i in test_datasets.items()] + + + # 2. Define audio pipeline: + @sb.utils.data_pipeline.takes("wav") + @sb.utils.data_pipeline.provides("sig") + def audio_pipeline(wav): + info = torchaudio.info(wav) + sig = sb.dataio.dataio.read_audio(wav) + if len(sig.shape)>1 : + sig = torch.mean(sig, dim=1) + resampled = torchaudio.transforms.Resample( + info.sample_rate, hparams["sample_rate"], + )(sig) + return resampled + + sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline) + label_encoder = sb.dataio.encoder.CTCTextEncoder() + + # 3. Define text pipeline: + @sb.utils.data_pipeline.takes("wrd") + @sb.utils.data_pipeline.provides( + "wrd", "char_list", "tokens_list", "tokens" + ) + def text_pipeline(wrd): + yield wrd + char_list = list(wrd) + yield char_list + tokens_list = label_encoder.encode_sequence(char_list) + yield tokens_list + tokens = torch.LongTensor(tokens_list) + yield tokens + + sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline) + lab_enc_file = os.path.join(hparams["save_folder"], "label_encoder.txt") + special_labels = { + "blank_label": hparams["blank_index"], + "unk_label": hparams["unk_index"] + } + label_encoder.load_or_create( + path=lab_enc_file, + from_didatasets=[train_data], + output_key="char_list", + special_labels=special_labels, + sequence_input=True, + ) + + # 4. Set output: + sb.dataio.dataset.set_output_keys( + datasets, ["id", "sig", "wrd", "char_list", "tokens"], + ) + return train_data, valid_data,test_datasets, label_encoder + +class ASR(sb.core.Brain): + def compute_forward(self, batch, stage): + """Forward computations from the waveform batches to the output probabilities.""" + + batch = batch.to(self.device) + wavs, wav_lens = batch.sig + wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device) + + if stage == sb.Stage.TRAIN: + if hasattr(self.hparams, "augmentation"): + wavs = self.hparams.augmentation(wavs, wav_lens) + + # Forward pass + feats = self.modules.wav2vec2(wavs, wav_lens) + x = self.modules.enc(feats) + logits = self.modules.ctc_lin(x) + p_ctc = self.hparams.log_softmax(logits) + + return p_ctc, wav_lens + + def custom_encode(self,wavs,wav_lens) : + wavs = wavs.to(self.device) + if(wav_lens is not None): wav_lens.to(self.device) + + feats = self.modules.wav2vec2(wavs, wav_lens) + x = self.modules.enc(feats) + logits = self.modules.ctc_lin(x) + p_ctc = self.hparams.log_softmax(logits) + + return feats,p_ctc + + + + def compute_objectives(self, predictions, batch, stage): + """Computes the loss (CTC) given predictions and targets.""" + + p_ctc, wav_lens = predictions + + ids = batch.id + tokens, tokens_lens = batch.tokens + + loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens) + + if stage != sb.Stage.TRAIN: + predicted_tokens = sb.decoders.ctc_greedy_decode( + p_ctc, wav_lens, blank_id=self.hparams.blank_index + ) + # Decode token terms to words + if self.hparams.use_language_modelling: + predicted_words = [] + for logs in p_ctc: + text = decoder.decode(logs.detach().cpu().numpy()) + predicted_words.append(text.split(" ")) + else: + predicted_words = [ + "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ") + for utt_seq in predicted_tokens + ] + # Convert indices to words + target_words = [wrd.split(" ") for wrd in batch.wrd] + + self.wer_metric.append(ids, predicted_words, target_words) + self.cer_metric.append(ids, predicted_words, target_words) + + return loss + + def fit_batch(self, batch): + """Train the parameters given a single batch in input""" + should_step = self.step % self.grad_accumulation_factor == 0 + # Managing automatic mixed precision + # TOFIX: CTC fine-tuning currently is unstable + # This is certainly due to CTC being done in fp16 instead of fp32 + if self.auto_mix_prec: + with torch.cuda.amp.autocast(): + with self.no_sync(): + outputs = self.compute_forward(batch, sb.Stage.TRAIN) + loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN) + with self.no_sync(not should_step): + self.scaler.scale( + loss / self.grad_accumulation_factor + ).backward() + if should_step: + + if not self.hparams.wav2vec2.freeze: + self.scaler.unscale_(self.wav2vec_optimizer) + self.scaler.unscale_(self.model_optimizer) + if self.check_gradients(loss): + if not self.hparams.wav2vec2.freeze: + if self.optimizer_step >= self.hparams.warmup_steps: + self.scaler.step(self.wav2vec_optimizer) + self.scaler.step(self.model_optimizer) + self.scaler.update() + self.zero_grad() + self.optimizer_step += 1 + else: + # This is mandatory because HF models have a weird behavior with DDP + # on the forward pass + with self.no_sync(): + outputs = self.compute_forward(batch, sb.Stage.TRAIN) + + loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN) + + with self.no_sync(not should_step): + (loss / self.grad_accumulation_factor).backward() + if should_step: + if self.check_gradients(loss): + if not self.hparams.wav2vec2.freeze: + if self.optimizer_step >= self.hparams.warmup_steps: + self.wav2vec_optimizer.step() + self.model_optimizer.step() + self.zero_grad() + self.optimizer_step += 1 + + self.on_fit_batch_end(batch, outputs, loss, should_step) + return loss.detach().cpu() + + def evaluate_batch(self, batch, stage): + """Computations needed for validation/test batches""" + predictions = self.compute_forward(batch, stage=stage) + with torch.no_grad(): + loss = self.compute_objectives(predictions, batch, stage=stage) + return loss.detach() + + def on_stage_start(self, stage, epoch): + """Gets called at the beginning of each epoch""" + if stage != sb.Stage.TRAIN: + self.cer_metric = self.hparams.cer_computer() + self.wer_metric = self.hparams.error_rate_computer() + + def on_stage_end(self, stage, stage_loss, epoch): + """Gets called at the end of an epoch.""" + # Compute/store important stats + stage_stats = {"loss": stage_loss} + if stage == sb.Stage.TRAIN: + self.train_stats = stage_stats + else: + stage_stats["CER"] = self.cer_metric.summarize("error_rate") + stage_stats["WER"] = self.wer_metric.summarize("error_rate") + + # Perform end-of-iteration things, like annealing, logging, etc. + if stage == sb.Stage.VALID: + old_lr_model, new_lr_model = self.hparams.lr_annealing_model( + stage_stats["loss"] + ) + old_lr_wav2vec, new_lr_wav2vec = self.hparams.lr_annealing_wav2vec( + stage_stats["loss"] + ) + sb.nnet.schedulers.update_learning_rate( + self.model_optimizer, new_lr_model + ) + if not self.hparams.wav2vec2.freeze: + sb.nnet.schedulers.update_learning_rate( + self.wav2vec_optimizer, new_lr_wav2vec + ) + self.hparams.train_logger.log_stats( + stats_meta={ + "epoch": epoch, + "lr_model": old_lr_model, + "lr_wav2vec": old_lr_wav2vec, + }, + train_stats=self.train_stats, + valid_stats=stage_stats, + ) + self.checkpointer.save_and_keep_only( + meta={"WER": stage_stats["WER"]}, min_keys=["WER"], + ) + elif stage == sb.Stage.TEST: + self.hparams.train_logger.log_stats( + stats_meta={"Epoch loaded": self.hparams.epoch_counter.current}, + test_stats=stage_stats, + ) + with open(self.hparams.wer_file, "w") as w: + self.wer_metric.write_stats(w) + + def init_optimizers(self): + "Initializes the wav2vec2 optimizer and model optimizer" + + # If the wav2vec encoder is unfrozen, we create the optimizer + if not self.hparams.wav2vec2.freeze: + self.wav2vec_optimizer = self.hparams.wav2vec_opt_class( + self.modules.wav2vec2.parameters() + ) + if self.checkpointer is not None: + self.checkpointer.add_recoverable( + "wav2vec_opt", self.wav2vec_optimizer + ) + + self.model_optimizer = self.hparams.model_opt_class( + self.hparams.model.parameters() + ) + + if self.checkpointer is not None: + self.checkpointer.add_recoverable("modelopt", self.model_optimizer) + + def zero_grad(self, set_to_none=False): + if not self.hparams.wav2vec2.freeze: + self.wav2vec_optimizer.zero_grad(set_to_none) + self.model_optimizer.zero_grad(set_to_none) + + +from speechbrain.pretrained import EncoderASR,EncoderDecoderASR +french_asr_model = EncoderASR.from_hparams(source="speechbrain/asr-wav2vec2-commonvoice-fr", savedir="pretrained_models/asr-wav2vec2-commonvoice-fr").cuda() + +cvhparams_file, cvrun_opts, cvoverrides = sb.parse_arguments(["en_cv.yaml"]) +with open(cvhparams_file) as cvfin: + cvhparams = load_hyperpyyaml(cvfin, cvoverrides) +english_asr_model = ASRCV( + modules=cvhparams["modules"], + hparams=cvhparams, + run_opts=cvrun_opts, + checkpointer=cvhparams["checkpointer"], + ) +english_asr_model.checkpointer.recover_if_possible() +asr_brain = ASR( + modules=hparams["modules"], + hparams=hparams, + run_opts=run_opts, + checkpointer=hparams["checkpointer"], +) +asr_brain.checkpointer.recover_if_possible() +asr_brain.modules.eval() +english_asr_model.modules.eval() +french_asr_model.mods.eval() + +# Commented out IPython magic to ensure Python compatibility. +# %ls + +#UTILS FUNCTIOJNS +def get_size_dimensions(arr): + size_dimensions = [] + while isinstance(arr, list): + size_dimensions.append(len(arr)) + arr = arr[0] + return size_dimensions + +def scale_array(batch,n): + scaled_batch = [] + + for array in batch: + if(n < len(array)): raise ValueError("Cannot scale Array down") + + repeat = round(n/len(array))+1 + scaled_length_array= [] + + for i in array: + for j in range(repeat) : + if(len(scaled_length_array) == n): break + scaled_length_array.append(i) + + scaled_batch.append(scaled_length_array) + + return torch.tensor(scaled_batch) + + +def load_paths(wavs_path): + waveforms = [] + for path in wavs_path : + waveform, _ = torchaudio.load(path) + waveforms.append(waveform.squeeze(0)) + # normalize array length to the bigger arrays by pading with 0's + padded_arrays = pad_sequence(waveforms, batch_first=True) + return torch.tensor(padded_arrays) + + + +device = 'cuda' +verbose = 0 +#FLOW LEVEL FUNCTIONS +def merge_strategy(embeddings1, embeddings2, embeddings3,post1, post2,post3): + + + post1 = post1.to(device) + post2 = post2.to(device) + post3 = post3.to(device) + embeddings1 = embeddings1.to(device) + embeddings2 = embeddings2.to(device) + embeddings3 = embeddings3.to(device) + + posteriograms_merged = torch.cat((post1,post2,post3),dim=2) + embeddings_merged = torch.cat((embeddings1,embeddings2,embeddings3),dim=2) + + if(verbose !=0): + print('MERGED POST ',posteriograms_merged.shape) + print('MERGED emb ',embeddings_merged.shape) + + return torch.cat((posteriograms_merged,embeddings_merged),dim=2).to(device) + +def decode(model,wavs,wav_lens): + + with torch.no_grad(): + wav_lens = wav_lens.to(model.device) + encoder_out = model.encode_batch(wavs, wav_lens) + predictions = model.decoding_function(encoder_out, wav_lens) + return predictions + +def middle_layer(batch, lens): + + tn_embeddings, tn_posteriogram = asr_brain.custom_encode(batch,None) + + fr_embeddings = french_asr_model.mods.encoder.wav2vec2(batch) + fr_posteriogram =french_asr_model.encode_batch(batch,lens) + en_embeddings = english_asr_model.modules.wav2vec2(batch, lens) + x = english_asr_model.modules.enc(en_embeddings) + en_posteriogram = english_asr_model.modules.ctc_lin(x) + #scores, en_posteriogram = english_asr_model.mods.decoder(en_embeddings ,lens) + if(verbose !=0): + print('[EMBEDDINGS] FR:',fr_embeddings.shape, "EN:",en_embeddings.shape, "TN:", tn_embeddings.shape) + print('[POSTERIOGRAM] FR:',fr_posteriogram.shape, "EN:",en_posteriogram.shape,"TN:",tn_posteriogram.shape) + + + bilangual_sample = merge_strategy(fr_embeddings,en_embeddings,tn_embeddings,fr_posteriogram,en_posteriogram,tn_posteriogram) + return bilangual_sample + +class Mixer(sb.core.Brain): + + def compute_forward(self, batch, stage): + """Forward computations from the waveform batches to the output probabilities.""" + wavs, wav_lens = batch.sig + wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device) + + if stage == sb.Stage.TRAIN: + if hasattr(self.hparams, "augmentation"): + wavs = self.hparams.augmentation(wavs, wav_lens) + + multi_langual_feats = middle_layer(wavs, wav_lens) + multi_langual_feats= multi_langual_feats.to(device) + feats, _ = self.modules.enc(multi_langual_feats) + logits = self.modules.ctc_lin(feats) + p_ctc = self.hparams.log_softmax(logits) + + if stage!= sb.Stage.TRAIN: + p_tokens = sb.decoders.ctc_greedy_decode( + p_ctc, wav_lens, blank_id=self.hparams.blank_index + ) + else : + p_tokens = None + return p_ctc, wav_lens, p_tokens + + def compute_objectives(self, predictions, batch, stage): + """Computes the loss (CTC) given predictions and targets.""" + + p_ctc, wav_lens , predicted_tokens= predictions + + ids = batch.id + tokens, tokens_lens = batch.tokens + + loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens) + + + if stage == sb.Stage.VALID: + predicted_words = [ + "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ") + for utt_seq in predicted_tokens + ] + target_words = [wrd.split(" ") for wrd in batch.wrd] + self.wer_metric.append(ids, predicted_words, target_words) + self.cer_metric.append(ids, predicted_words, target_words) + if stage ==sb.Stage.TEST : + if self.hparams.language_modelling: + predicted_words = [] + for logs in p_ctc: + text = decoder.decode(logs.detach().cpu().numpy()) + predicted_words.append(text.split(" ")) + else : + predicted_words = [ + "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ") + for utt_seq in predicted_tokens + ] + + target_words = [wrd.split(" ") for wrd in batch.wrd] + self.wer_metric.append(ids, predicted_words, target_words) + self.cer_metric.append(ids, predicted_words, target_words) + + return loss + + def fit_batch(self, batch): + """Train the parameters given a single batch in input""" + should_step = self.step % self.grad_accumulation_factor == 0 + # Managing automatic mixed precision + # TOFIX: CTC fine-tuning currently is unstable + # This is certainly due to CTC being done in fp16 instead of fp32 + if self.auto_mix_prec: + with torch.cuda.amp.autocast(): + with self.no_sync(): + outputs = self.compute_forward(batch, sb.Stage.TRAIN) + loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN) + with self.no_sync(not should_step): + self.scaler.scale( + loss / self.grad_accumulation_factor + ).backward() + if should_step: + + + self.scaler.unscale_(self.model_optimizer) + if self.check_gradients(loss): + self.scaler.step(self.model_optimizer) + self.scaler.update() + self.zero_grad() + self.optimizer_step += 1 + else: + # This is mandatory because HF models have a weird behavior with DDP + # on the forward pass + with self.no_sync(): + outputs = self.compute_forward(batch, sb.Stage.TRAIN) + + loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN) + + with self.no_sync(not should_step): + (loss / self.grad_accumulation_factor).backward() + if should_step: + if self.check_gradients(loss): + self.model_optimizer.step() + self.zero_grad() + self.optimizer_step += 1 + + self.on_fit_batch_end(batch, outputs, loss, should_step) + return loss.detach().cpu() + + def evaluate_batch(self, batch, stage): + """Computations needed for validation/test batches""" + predictions = self.compute_forward(batch, stage=stage) + with torch.no_grad(): + loss = self.compute_objectives(predictions, batch, stage=stage) + return loss.detach() + + def on_stage_start(self, stage, epoch): + """Gets called at the beginning of each epoch""" + if stage != sb.Stage.TRAIN: + self.cer_metric = self.hparams.cer_computer() + self.wer_metric = self.hparams.error_rate_computer() + + def on_stage_end(self, stage, stage_loss, epoch): + """Gets called at the end of an epoch.""" + # Compute/store important stats + stage_stats = {"loss": stage_loss} + if stage == sb.Stage.TRAIN: + self.train_stats = stage_stats + else: + stage_stats["CER"] = self.cer_metric.summarize("error_rate") + stage_stats["WER"] = self.wer_metric.summarize("error_rate") + + # Perform end-of-iteration things, like annealing, logging, etc. + if stage == sb.Stage.VALID: + old_lr_model, new_lr_model = self.hparams.lr_annealing_model( + stage_stats["loss"] + ) + sb.nnet.schedulers.update_learning_rate( + self.model_optimizer, new_lr_model + ) + self.hparams.train_logger.log_stats( + stats_meta={ + "epoch": epoch, + "lr_model": old_lr_model, + }, + train_stats=self.train_stats, + valid_stats=stage_stats, + ) + self.checkpointer.save_and_keep_only( + meta={"WER": stage_stats["WER"]}, min_keys=["WER"], + ) + elif stage == sb.Stage.TEST: + self.hparams.train_logger.log_stats( + stats_meta={"Epoch loaded": self.hparams.epoch_counter.current}, + test_stats=stage_stats, + ) + with open(self.hparams.wer_file, "w") as w: + self.wer_metric.write_stats(w) + + def init_optimizers(self): + + self.model_optimizer = self.hparams.model_opt_class( + self.hparams.model.parameters() + ) + + if self.checkpointer is not None: + self.checkpointer.add_recoverable("modelopt", self.model_optimizer) + + def zero_grad(self, set_to_none=False): + + self.model_optimizer.zero_grad(set_to_none) + + +hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:]) + +# If distributed_launch=True then +# create ddp_group with the right communication protocol +sb.utils.distributed.ddp_init_group(run_opts) + +with open(hparams_file) as fin: + hparams = load_hyperpyyaml(fin, overrides) + +# Create experiment directory +sb.create_experiment_directory( + experiment_directory=hparams["output_folder"], + hyperparams_to_save=hparams_file, + overrides=overrides, +) +def read_labels_file(labels_file): + with open(labels_file, "r",encoding="utf-8") as lf: + lines = lf.read().splitlines() + division = "===" + numbers = {} + for line in lines : + if division in line : + break + string, number = line.split("=>") + number = int(number) + string = string[1:-2] + numbers[number] = string + return [numbers[x] for x in range(len(numbers))] +train_data, valid_data, test_datasets, label_encoder = dataio_prepare( + hparams + ) + + +labels = read_labels_file(os.path.join(hparams["save_folder"], "label_encoder.txt")) +labels = [""] + labels[1:-1] + ["1"] +if hparams["language_modelling"]: + decoder = build_ctcdecoder( + labels, + kenlm_model_path=hparams["ngram_lm_path"], # either .arpa or .bin file + alpha=0.5, # tuned on a val set + beta=1, # tuned on a val set + ) + + + + +mixer = Mixer( + modules=hparams["modules"], + hparams=hparams, + run_opts=run_opts, + checkpointer=hparams["checkpointer"], +) +mixer.tokenizer = label_encoder + + +mixer.fit( + mixer.hparams.epoch_counter, + train_data, + valid_data, + train_loader_kwargs=hparams["dataloader_options"], + valid_loader_kwargs=hparams["test_dataloader_options"], +) +print(test_datasets.keys()) +for k in test_datasets.keys(): # keys are test_clean, test_other etc + mixer.hparams.wer_file = os.path.join( + hparams["output_folder"], "wer_{}.txt".format(k) + ) + mixer.evaluate( + test_datasets[k], test_loader_kwargs=hparams["test_dataloader_options"] + ) +