<jupyter_start><jupyter_text>Read the dataset frim the disk<jupyter_code>train_batch_size = 1024
dset=WCH5Dataset(path, 0.1, 0.1, reduced_dataset_size=300000)

# Initialize the dataloader
train_loader = DataLoader(dset, batch_size=train_batch_size, shuffle=False,
                          num_workers=4, sampler=SubsetRandomSampler(dset.train_indices))

# Read the tensors from the dataset and add to the list
event_charges = []
event_timings = []

labels = []
energies = []

for data in iter(train_loader):
    event_charges.append(data[0][:,:,:,:19].numpy())
    event_timings.append(data[0][:,:,:,19:].numpy())
    
    labels.append(data[1].numpy())
    energies.append(data[2].numpy())
    
print(len(event_charges), event_charges[0].shape, len(event_timings), event_timings[0].shape)
print(len(labels), labels[0].shape, len(energies), energies[0].shape)
event_charges = np.concatenate(event_charges, axis=0)
event_timings = np.concatenate(event_timings, axis=0)

labels = np.concatenate(labels, axis=0)
energies = np.concatenate(energies, axis=0)

print(event_charges.shape, event_timings.shape, labels.shape, energies.shape)<jupyter_output>(240000, 40, 40, 19) (240000, 40, 40, 19) (240000,) (240000, 1)
{'fraction_non_alphanumeric': 0.14698596201486375, 'fraction_numerical': 0.05697770437654831, 'mean_word_length': 5.9655172413793105, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '303398', 'license': 'CNRI-Python', 'n_tokens_mistral': 499, 'n_tokens_neox': 456, 'n_words': 77, 'path': 'notebooks/0811 - Normalization.ipynb', 'repo_name': 'WatChMaL/CNN', 'size': '1211'}