<jupyter_start><jupyter_text>Something has gone wrong with `anb_ga_ul/text02`Audio is in `wav_trimmed/`<jupyter_code>import librosa
import numpy as np

def resample(batch):
    batch["duration"] = librosa.get_duration(y=batch["speech"], sr=batch["sampling_rate"])

    if batch["sampling_rate"] != 16_000:
        batch["speech"] = librosa.resample(np.asarray(batch["speech"]), batch["sampling_rate"], 16_000)
        batch["sampling_rate"] = 16_000

    return batch

for dir in dirl:
    sets[dir]["train"] = sets[dir]["train"].map(resample, num_proc=4)<jupyter_output>Loading cached processed dataset at /home/jim/.cache/huggingface/datasets/utterance_xml_dataset/default-187ee4831f664a6d/1.1.0/2ab4d5a39b76b2b213d176be1f94b8bb2c34198ea2fabf4b281e0784782b2d2b/cache-da006be1caebfb5b.arrow
Loading cached processed dataset at /home/jim/.cache/huggingface/datasets/utterance_xml_dataset/default-187ee4831f664a6d/1.1.0/2ab4d5a39b76b2b213d176be1f94b8bb2c34198ea2fabf4b281e0784782b2d2b/cache-c2641d558f05e8f1.arrow
Loading cached processed dataset at /home/jim/.cache/huggingface/datasets/utterance_xml_dataset/default-187ee4831f664a6d/1.1.0/2ab4d5a39b76b2b213d176be1f94b8bb2c34198ea2fabf4b281e0784782b2d2b/cache-8d515b7aebd171c2.arrow
Loading cached processed dataset at /home/jim/.cache/huggingface/datasets/utterance_xml_dataset/default-187ee4831f664a6d/1.1.0/2ab4d5a39b76b2b213d176be1f94b8bb2c34198ea2fabf4b281e0784782b2d2b/cache-7b97b92998e9a71e.arrow
Loading cached processed dataset at /home/jim/.cache/huggingface/datasets/utterance_xml_dataset/default-aa8abe91265e[...]
{'fraction_non_alphanumeric': 0.10532994923857868, 'fraction_numerical': 0.1700507614213198, 'mean_word_length': 7.343915343915344, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '417265', 'license': 'Apache-2.0', 'n_tokens_mistral': 792, 'n_tokens_neox': 663, 'n_words': 66, 'path': '_drafts/2021-09-09-w2v-phonemes.ipynb', 'repo_name': 'jimregan/notes', 'size': '1576'}