use tokenizer to batch_decode

run_speech_recognition_seq2seq_streaming.py

@@ -458,7 +458,7 @@ def load_maybe_streaming_dataset(
         return dataset
 
 
-def print_data_samples(dataset, processor, max_samples=5):
+def print_data_samples(dataset, tokenizer, max_samples=5):
     shown_samples = 0
     for batch in dataset:
         print("Target: ", tokenizer.batch_decode(batch["labels"]))
@@ -786,7 +786,7 @@ def main():
     # 12. Training
     if training_args.do_train:
         logger.info("*** Train ***")
-        print_data_samples(vectorized_datasets["train"], processor)
+        print_data_samples(vectorized_datasets["train"], tokenizer)
         checkpoint = None
         if training_args.resume_from_checkpoint is not None:
             checkpoint = training_args.resume_from_checkpoint
@@ -824,15 +824,15 @@ def main():
         num_beams=training_args.generation_num_beams,
     )
     logger.info("*** Test prediction done ***")
-    preds = processor.batch_decode(predictions.predictions)
-    labels = processor.batch_decode(predictions.label_ids)
+    preds = tokenizer.batch_decode(predictions.predictions)
+    labels = tokenizer.batch_decode(predictions.label_ids)
     pred_labels = [f"Prediction: {pred}\nLabel: {label}\n" for pred, label in zip(preds, labels)]
     logger.info("Before setting language and task")
     logger.info(f"{pred_labels}")
     trainer.model.config.forced_decoder_ids = \
-        processor.get_decoder_prompt_ids(language=data_args.language_eval, task=data_args.task, no_timestamps=True)
-    preds = processor.batch_decode(predictions.predictions)
-    labels = processor.batch_decode(predictions.label_ids)
+        tokenizer.get_decoder_prompt_ids(language=data_args.language_eval, task=data_args.task, no_timestamps=True)
+    preds = tokenizer.batch_decode(predictions.predictions)
+    labels = tokenizer.batch_decode(predictions.label_ids)
     pred_labels = [f"Prediction: {pred}\nLabel: {label}\n" for pred, label in zip(preds, labels)]
     logger.info("After setting language and task")
     logger.info(f"{pred_labels}")
@@ -841,7 +841,7 @@ def main():
     results = {}
     if training_args.do_eval:
         logger.info("*** Evaluate ***")
-        print_data_samples(vectorized_datasets["eval"], processor)
+        print_data_samples(vectorized_datasets["eval"], tokenizer)
         metrics = trainer.evaluate(
             metric_key_prefix="eval",
             max_length=training_args.generation_max_length,