usrbinenv python3 do not change this function computes and logs the result metrics logoutputs args logoutputs datasetid joinargs dataset split args config args split load metric wer loadmetricwer cer loadmetriccer compute metrics werresult wer computereferencesresulttarget predictionsresultprediction cerresult cer computereferencesresulttarget predictionsresultprediction print log results resultstr fwer werresultncer cerresult printresultstr with openfdatasetidevalresults txt w as f f writeresultstr log all results in text file possibly interesting for analysis if logoutputs is not none predfile flogdatasetidpredictions txt targetfile flogdatasetidtargets txt with openpredfile w as p opentargetfile w as t mapping function to write output def writetofilebatch i p writefi n p writebatchprediction n t writefi n t writebatchtarget n result mapwritetofile withindicestrue def normalizetexttext str str in addition we can normalize the target text e g removing new lines characters etc note that order is important here load dataset for testing only process the first two examples as a test dataset dataset selectrange10 load processor resample audio load eval pipeline map function to decode audio run inference on all examples compute and logresults do not change function below usr bin env python3 do not change this function computes and logs the result metrics load metric compute metrics print log results log all results in text file possibly interesting for analysis mapping function to write output do adapt for your use case this function normalizes the target text noqa w605 important this should correspond to the chars that were ignored during training in addition we can normalize the target text e g removing new lines characters etc note that order is important here load dataset for testing only process the first two examples as a test dataset dataset select range 10 load processor resample audio load eval pipeline map function to decode audio run inference on all examples compute and log_results do not change function below
import argparse
import re
from typing import Dict

import torch
from datasets import Audio, Dataset, load_dataset, load_metric

from transformers import AutoFeatureExtractor, pipeline


def log_results(result: Dataset, args: Dict[str, str]):
    

    log_outputs = args.log_outputs
    dataset_id = "_".join(args.dataset.split("/") + [args.config, args.split])

    
    wer = load_metric("wer")
    cer = load_metric("cer")

    
    wer_result = wer.compute(references=result["target"], predictions=result["prediction"])
    cer_result = cer.compute(references=result["target"], predictions=result["prediction"])

    
    result_str = f"WER: {wer_result}\nCER: {cer_result}"
    print(result_str)

    with open(f"{dataset_id}_eval_results.txt", "w") as f:
        f.write(result_str)

    
    if log_outputs is not None:
        pred_file = f"log_{dataset_id}_predictions.txt"
        target_file = f"log_{dataset_id}_targets.txt"

        with open(pred_file, "w") as p, open(target_file, "w") as t:
            
            def write_to_file(batch, i):
                p.write(f"{i}" + "\n")
                p.write(batch["prediction"] + "\n")
                t.write(f"{i}" + "\n")
                t.write(batch["target"] + "\n")

            result.map(write_to_file, with_indices=True)


def normalize_text(text: str) -> str:
    

    chars_to_ignore_regex = '[,?.!\-\;\:"“%‘”�—’…–]'  

    text = re.sub(chars_to_ignore_regex, "", text.lower())

    
    
    token_sequences_to_ignore = ["\n\n", "\n", "   ", "  "]

    for t in token_sequences_to_ignore:
        text = " ".join(text.split(t))

    return text


def main(args):
    
    dataset = load_dataset(args.dataset, args.config, split=args.split, token=True)

    
    

    
    feature_extractor = AutoFeatureExtractor.from_pretrained(args.model_id)
    sampling_rate = feature_extractor.sampling_rate

    
    dataset = dataset.cast_column("audio", Audio(sampling_rate=sampling_rate))

    
    if args.device is None:
        args.device = 0 if torch.cuda.is_available() else -1
    asr = pipeline("automatic-speech-recognition", model=args.model_id, device=args.device)

    
    def map_to_pred(batch):
        prediction = asr(
            batch["audio"]["array"], chunk_length_s=args.chunk_length_s, stride_length_s=args.stride_length_s
        )

        batch["prediction"] = prediction["text"]
        batch["target"] = normalize_text(batch["sentence"])
        return batch

    
    result = dataset.map(map_to_pred, remove_columns=dataset.column_names)

    
    
    log_results(result, args)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    parser.add_argument(
        "--model_id", type=str, required=True, help="Model identifier. Should be loadable with 🤗 Transformers"
    )
    parser.add_argument(
        "--dataset",
        type=str,
        required=True,
        help="Dataset name to evaluate the `model_id`. Should be loadable with 🤗 Datasets",
    )
    parser.add_argument(
        "--config", type=str, required=True, help="Config of the dataset. *E.g.* `'en'`  for Common Voice"
    )
    parser.add_argument("--split", type=str, required=True, help="Split of the dataset. *E.g.* `'test'`")
    parser.add_argument(
        "--chunk_length_s", type=float, default=None, help="Chunk length in seconds. Defaults to 5 seconds."
    )
    parser.add_argument(
        "--stride_length_s", type=float, default=None, help="Stride of the audio chunks. Defaults to 1 second."
    )
    parser.add_argument(
        "--log_outputs", action="store_true", help="If defined, write outputs to log file for analysis."
    )
    parser.add_argument(
        "--device",
        type=int,
        default=None,
        help="The device to run the pipeline on. -1 for CPU (default), 0 for the first GPU and so on.",
    )
    args = parser.parse_args()

    main(args)