<jupyter_start><jupyter_text>Select data parts<jupyter_code>mini_librispeech = False
if mini_librispeech:
    dataset_parts = 'mini_librispeech'
    dev_set = 'dev-clean-2'
    train_set = 'train-clean-5'
else:
    dataset_parts = ('dev-clean', 'test-clean', 'train-clean-100')
    dev_set = 'dev-clean'
    train_set = 'train-clean-100'<jupyter_output><empty_output><jupyter_text>Settings for paths<jupyter_code>root_dir = Path('data')
corpus_dir = root_dir / 'LibriSpeech'
output_dir = root_dir / 'librispeech_nb'<jupyter_output><empty_output><jupyter_text>Download and untar<jupyter_code>download_librispeech(root_dir, dataset_parts)<jupyter_output><empty_output><jupyter_text>Prepare audio and supervision manifests<jupyter_code>num_jobs = os.cpu_count()
librispeech_manifests = prepare_librispeech(corpus_dir, dataset_parts, output_dir, num_jobs=num_jobs)<jupyter_output><empty_output><jupyter_text>[Optional] Data augmentation<jupyter_code>use_data_augmentation = False
augment_fn = SoxEffectTransform(effects=[
   ['reverb', 50, 50, RandomValue(0, 100)],
   ['remix', '-'],  # Merge all channels (reverb changes mono to stereo)
   ['rate', 16000],
]) if use_data_augmentation else None<jupyter_output><empty_output><jupyter_text>Extract features<jupyter_code>for partition, manifests in librispeech_manifests.items():
    manifest_path = output_dir / f'cuts_{partition}.json.gz'
    if not manifest_path.is_file():
        with ProcessPoolExecutor(num_jobs) as ex:
            cut_set = CutSet.from_manifests(
                recordings=manifests['recordings'],
                supervisions=manifests['supervisions']
            )
            if use_data_augmentation:
                cut_set = cut_set + cut_set.perturb_speed(0.9) + cut_set.perturb_speed(1.1)
            cut_set = cut_set.compute_and_store_features(
                extractor=Fbank(),
                storage_path=f'{output_dir}/feats_{partition}',
                storage_type=LilcomFilesWriter,
                augment_fn=augment_fn,
                num_jobs=num_jobs,
                executor=ex
            )
        librispeech_manifests[partition]['cuts'] = cut_set
        cut_set.to_json(manifest_path)
    librispeech_manifests[partition] = CutSet.from_json(manifest_path)<jupyter_output><empty_output><jupyter_text>Make PyTorch Dataset<jupyter_code>dataset_dev = K2SpeechRecognitionDataset(librispeech_manifests[dev_set])
dataset_train = K2SpeechRecognitionDataset(librispeech_manifests[train_set])<jupyter_output><empty_output><jupyter_text>Illustration of an example<jupyter_code>sampler = SingleCutSampler(dataset_dev.cuts, shuffle=False, max_cuts=4)
cut_ids = next(iter(sampler))
sample = dataset_dev[cut_ids]
print('Transcript:', sample['supervisions']['text'][0])
print('Supervisions start frame:', int(sample['supervisions']['start_frame'][0]))
print('Supervisions frame number:', int(sample['supervisions']['num_frames'][0]))
print('Feature matrix:')
plt.matshow(sample['inputs'][0].transpose(0, 1).flip(0));<jupyter_output>Transcript: I'M HERE BECAUSE THE MATTER IS OF UTMOST IMPORTANCE AND BRANDD IS THE ONE I MUST SEE NOW STAND ASIDE
Supervisions start frame: 0
Supervisions frame number: 774
Feature matrix:<jupyter_text>Make PyTorch Dataloader<jupyter_code>dev_dl = torch.utils.data.DataLoader(dataset_dev, batch_size=None, sampler=sampler)
for batch in dev_dl:
    for key, value in batch.items():
        print(key, '- shape or length:', end=' ')
        if isinstance(value, torch.Tensor):
            print(value.shape)
        else:
            print(len(value))
    break<jupyter_output>inputs - shape or length: torch.Size([4, 774, 40])
supervisions - shape or length: 4
{'fraction_non_alphanumeric': 0.11371055495103373, 'fraction_numerical': 0.012513601741022852, 'mean_word_length': 4.978861788617886, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '457111', 'license': 'Apache-2.0', 'n_tokens_mistral': 1330, 'n_tokens_neox': 1250, 'n_words': 202, 'path': 'examples/librispeech/librispeech.ipynb', 'repo_name': 'aarora8/lhotse', 'size': '3676'}