Training in progress, step 200

.ipynb_checkpoints/requirements-checkpoint.txt

@@ -0,0 +1,8 @@
+unidecode
+tensorboard
+torch
+torchaudio
+jiwer~=2.3.0
+soundfile~=0.10.3
+transformers~=4.16.2
+datasets~=1.18.3

.ipynb_checkpoints/run-checkpoint.sh

@@ -1,29 +1,31 @@
-python xls-r-uzbek-cv8/run_speech_recognition_ctc.py \
+python ~/xls-r-uzbek-cv8/run_speech_recognition_ctc.py \
 	--dataset_name="mozilla-foundation/common_voice_8_0" \
 	--model_name_or_path="facebook/wav2vec2-xls-r-300m" \
 	--dataset_config_name="uz" \
-	--output_dir="./xls-r-uzbek-cv8" \
+	--output_dir="~/xls-r-uzbek-cv8" \
+	--train_split_name="train[:30%]" \
+	--eval_split_name="validation[50%:]" \
 	--overwrite_output_dir \
-	--num_train_epochs="30" \
-	--per_device_train_batch_size="16" \
+	--num_train_epochs="100" \
+	--per_device_train_batch_size="32" \
 	--per_device_eval_batch_size="8" \
 	--gradient_accumulation_steps="4" \
 	--learning_rate="3e-5" \
-	--warmup_steps="2000" \
+	--warmup_steps="500" \
 	--length_column_name="input_length" \
 	--evaluation_strategy="steps" \
 	--text_column_name="sentence" \
-	--eval_metrics="wer,cer" \
-	--save_steps="500" \
-	--eval_steps="500" \
+	--eval_metrics="cer" \
+	--save_steps="200" \
+	--eval_steps="200" \
 	--logging_steps="100" \
-	--min_duration_in_seconds="0.5" \
-	--layerdrop="0.0" \
+	--min_duration_in_seconds="0.2" \
+	--layerdrop="0.01" \
 	--activation_dropout="0.1" \
 	--save_total_limit="3" \
 	--freeze_feature_encoder \
-	--feat_proj_dropout="0.0" \
-	--mask_time_prob="0.75" \
+	--feat_proj_dropout="0.01" \
+	--mask_time_prob="0.50" \
 	--mask_time_length="10" \
 	--mask_feature_prob="0.25" \
 	--mask_feature_length="64" \

config.json

@@ -49,7 +49,7 @@
   "feat_extract_activation": "gelu",
   "feat_extract_dropout": 0.0,
   "feat_extract_norm": "layer",
-  "feat_proj_dropout": 0.0,
+  "feat_proj_dropout": 0.01,
   "feat_quantizer_dropout": 0.0,
   "final_dropout": 0.0,
   "hidden_act": "gelu",
@@ -58,13 +58,13 @@
   "initializer_range": 0.02,
   "intermediate_size": 4096,
   "layer_norm_eps": 1e-05,
-  "layerdrop": 0.0,
+  "layerdrop": 0.01,
   "mask_feature_length": 64,
   "mask_feature_min_masks": 0,
   "mask_feature_prob": 0.25,
   "mask_time_length": 10,
   "mask_time_min_masks": 2,
-  "mask_time_prob": 0.75,
+  "mask_time_prob": 0.5,
   "model_type": "wav2vec2",
   "num_adapter_layers": 3,
   "num_attention_heads": 16,
@@ -100,7 +100,7 @@
     1
   ],
   "torch_dtype": "float32",
-  "transformers_version": "4.17.0.dev0",
+  "transformers_version": "4.16.2",
   "use_weighted_layer_sum": false,
   "vocab_size": 33,
   "xvector_output_dim": 512,

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d7a4269673722dcfe01934e733ea692df4ab980becd9fbb7bf129b60ca1505af
+oid sha256:3f107f42a91fd5d39621347b43afb0c87ef3258c4bc185ec165f2870c082b35c
 size 1262058993

requirements.txt

@@ -1,3 +1,8 @@
 unidecode
 tensorboard
-
+torch
+torchaudio
+jiwer~=2.3.0
+soundfile~=0.10.3
+transformers~=4.16.2
+datasets~=1.18.3

run.sh

@@ -1,29 +1,31 @@
-python xls-r-uzbek-cv8/run_speech_recognition_ctc.py \
+python ~/xls-r-uzbek-cv8/run_speech_recognition_ctc.py \
 	--dataset_name="mozilla-foundation/common_voice_8_0" \
 	--model_name_or_path="facebook/wav2vec2-xls-r-300m" \
 	--dataset_config_name="uz" \
-	--output_dir="./xls-r-uzbek-cv8" \
+	--output_dir="~/xls-r-uzbek-cv8" \
+	--train_split_name="train[:30%]" \
+	--eval_split_name="validation[50%:]" \
 	--overwrite_output_dir \
-	--num_train_epochs="30" \
-	--per_device_train_batch_size="16" \
+	--num_train_epochs="100" \
+	--per_device_train_batch_size="32" \
 	--per_device_eval_batch_size="8" \
 	--gradient_accumulation_steps="4" \
 	--learning_rate="3e-5" \
-	--warmup_steps="2000" \
+	--warmup_steps="500" \
 	--length_column_name="input_length" \
 	--evaluation_strategy="steps" \
 	--text_column_name="sentence" \
-	--eval_metrics="wer,cer" \
-	--save_steps="500" \
-	--eval_steps="500" \
+	--eval_metrics="cer" \
+	--save_steps="200" \
+	--eval_steps="200" \
 	--logging_steps="100" \
-	--min_duration_in_seconds="0.5" \
-	--layerdrop="0.0" \
+	--min_duration_in_seconds="0.2" \
+	--layerdrop="0.01" \
 	--activation_dropout="0.1" \
 	--save_total_limit="3" \
 	--freeze_feature_encoder \
-	--feat_proj_dropout="0.0" \
-	--mask_time_prob="0.75" \
+	--feat_proj_dropout="0.01" \
+	--mask_time_prob="0.50" \
 	--mask_time_length="10" \
 	--mask_feature_prob="0.25" \
 	--mask_feature_length="64" \

runs/Feb05_08-15-45_job-0a778896-a7e2-46e9-bcf5-016f91f242cf/1644049006.003602/events.out.tfevents.1644049006.job-0a778896-a7e2-46e9-bcf5-016f91f242cf.23457.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84f165c6e40f5a1e87d21f624a9b4475d58842f270044fb50735b8f5a9d00101
+size 4826

runs/Feb05_08-15-45_job-0a778896-a7e2-46e9-bcf5-016f91f242cf/events.out.tfevents.1644049005.job-0a778896-a7e2-46e9-bcf5-016f91f242cf.23457.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:314b3baf73fa8b9becdd35cfeecc2a0a871edb0fda354e78bfa86e375dcc81fd
+size 5404

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./xls-r-uzbek-cv8", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "/workspace/xls-r-uzbek-cv8", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ebc2a615608b93ae5d470729cc3ef83ec5ad23e35190fdca59d878f4f9e7636a
+oid sha256:7a661f5e8e620b98c6c94d0bb8c2ddd7376f2412096a9ce2ff8088711cea618b
 size 3055