<jupyter_start><jupyter_text>Check the raw .h5 file using h5py<jupyter_code>path = "/fast_scratch/WatChMaL/NuPRISM_4pi_full_tank_grid/out.h5"
f = h5py.File(path, 'r')
f.keys()
raw_angles = f["angles"]
raw_energies = f["energies"]
raw_event_ids = f["event_ids"]
raw_labels = f["labels"]
raw_positions = f["positions"]

print(len(raw_angles), len(raw_energies), len(raw_event_ids), len(raw_labels), len(raw_positions))
print(len(raw_positions))<jupyter_output>9000000<jupyter_text>Use the watchmal WCH5Dataset handler to load the dataset<jupyter_code>train_batch_size = 1024
dset=WCH5Dataset("/fast_scratch/NuPRISM_4pi_full_tank_grid/out.h5", 0.1, 0.1, reduced_dataset_size=1000000)
train_loader = DataLoader(dset, batch_size=train_batch_size, shuffle=False,
                          pin_memory=True, sampler=SubsetRandomSampler(dset.train_indices))<jupyter_output><empty_output><jupyter_text>Iterate over the dataset and collect the event, label, energy and angle data per example<jupyter_code>labels = []
energies = []
angles = []
indices = []

for data in iter(train_loader):
    labels.append(data[1].numpy())
    energies.append(data[2].numpy())
    angles.append(data[3].numpy())
    indices.append(data[4].numpy())
print(len(event_charges), event_charges[0].shape, len(event_timings), event_timings[0].shape)
print(len(labels), labels[0].shape,
      len(energies), energies[0].shape,
      len(angles), angles[0].shape,
      len(indices), indices[0].shape)<jupyter_output>782 (1024,) 782 (1024, 1) 782 (1024, 2) 782 (1024,)<jupyter_text>Reshape the multi-dimensional arrays into shape = (num_examples, *)<jupyter_code>event_charges = np.concatenate(event_charges, axis=0)
event_timings = np.concatenate(event_timings, axis=0)
labels   = np.concatenate(labels, axis=0)
energies = np.concatenate(energies, axis=0)
angles   = np.concatenate(angles, axis=0)
indices  = np.concatenate(indices, axis=0)
print(event_charges.shape, event_timings.shape)
print(labels.shape, energies.shape, angles.shape, indices.shape)<jupyter_output>(800000,) (800000, 1) (800000, 2) (800000,)<jupyter_text>Plot the distrbutions of the features from the dataset  First check the distribution of the labels<jupyter_code>print(Counter(labels))<jupyter_output>Counter({1: 267189, 0: 266856, 2: 265955})<jupyter_text>Segment the event_charges, event_timings, energies, angles data using labels and plot their distributions  Plot the per label energy distribution<jupyter_code>label_dict = {0:["gamma","red"], 1:["e","blue"], 2:["mu","green"]}
energies_dict = {}
for label in label_dict.keys():
    energies_dict[label] = energies[labels == label]
    
fig, axes = plt.subplots(3, 1, figsize=(16,9), sharex=True)

for label in energies_dict.keys():
    axes[label].hist(energies_dict[label], bins=200, density=False, label=label_dict[label][0], alpha=0.8,
                     color=label_dict[label][1])
    axes[label].legend()
    axes[label].set_ylabel("Frequency")
    
plt.xlabel("Energy (MeV)")<jupyter_output><empty_output><jupyter_text>Plot the per label angle distribution<jupyter_code>angles_0_dict = {}
for label in label_dict.keys():
    angles_0_dict[label] = angles[labels == label][:,0]
    
fig, axes = plt.subplots(3, 1, figsize=(16,9), sharex=True)

for label in angles_0_dict.keys():
    axes[label].hist(angles_0_dict[label], bins=200, density=False, label=label_dict[label][0], alpha=0.8,
                     color=label_dict[label][1])
    axes[label].legend()
    axes[label].set_ylabel("Frequency")
    
plt.xlabel("Angle at index 0")
angles_1_dict = {}
for label in label_dict.keys():
    angles_1_dict[label] = angles[labels == label][:,1]
    
fig, axes = plt.subplots(3, 1, figsize=(16,9), sharex=True)

for label in angles_1_dict.keys():
    axes[label].hist(angles_1_dict[label], bins=200, density=False, label=label_dict[label][0], alpha=0.8,
                     color=label_dict[label][1])
    axes[label].legend()
    axes[label].set_ylabel("Frequency")
    
plt.xlabel("Angle at index 1")<jupyter_output><empty_output><jupyter_text>Plot the per label event charge distribution per mPMT module<jupyter_code>event_charge_dict = {}
for label in label_dict.keys():
    event_charge_dict[label] = np.mean(event_charges[labels == label], axis=3).flatten()
    
fig, axes = plt.subplots(3, 1, figsize=(32,18), sharex=True)

for label in angles_1_dict.keys():
    axes[label].hist(event_charge_dict[label], bins=200, density=False, label=label_dict[label][0], alpha=0.8,
                     color=label_dict[label][1])
    axes[label].legend(prop={"size":20})
    axes[label].set_xlabel("Average charge per mPMT", fontsize=20)
    axes[label].set_ylabel("Frequency", fontsize=20)
    axes[label].tick_params(axis="both", labelsize=20)
    axes[label].set_yscale("log")
    axes[label].grid(True, which="both", axis="both")<jupyter_output><empty_output><jupyter_text>Plot the per label event timing distribution per mPMT module<jupyter_code>print(np.mean(event_timings[labels == label], axis=3).shape)
event_timing_dict = {}
for label in label_dict.keys():
    event_timing_dict[label] = np.sum(event_timings[labels == label], axis=3).flatten()
    
fig, axes = plt.subplots(3, 1, figsize=(32,18), sharex=True)

for label in angles_1_dict.keys():
    axes[label].hist(event_timing_dict[label], bins=200, density=False, label=label_dict[label][0], alpha=0.8,
                     color=label_dict[label][1])
    axes[label].legend(prop={"size":20})
    axes[label].set_xlabel("Mean timing per mPMT (ns)", fontsize=20)
    axes[label].set_ylabel("Frequency", fontsize=20)
    axes[label].tick_params(axis="both", labelsize=20)
    axes[label].set_yscale("log")
    axes[label].grid(True, which="both", axis="both")<jupyter_output><empty_output><jupyter_text>Plot the per label per PMT charge distribution<jupyter_code>event_charge_dict = {}
for label in label_dict.keys():
    event_charge_dict[label] = event_charges[labels == label].flatten()
    
fig, axes = plt.subplots(3, 1, figsize=(32,18), sharex=True)

for label in angles_1_dict.keys():
    axes[label].hist(event_charge_dict[label], bins=200, density=False, label=label_dict[label][0], alpha=0.8,
                     color=label_dict[label][1])
    axes[label].legend(prop={"size":20})
    axes[label].set_xlabel("PMT Charge", fontsize=20)
    axes[label].set_ylabel("Frequency", fontsize=20)
    axes[label].tick_params(axis="both", labelsize=20)
    axes[label].set_yscale("log")
    axes[label].grid(True, which="both", axis="both")<jupyter_output><empty_output><jupyter_text>Plot the per label per PMT timing distribution<jupyter_code>event_timing_dict = {}
for label in label_dict.keys():
    event_timing_dict[label] = event_timings[labels == label].flatten()
    
fig, axes = plt.subplots(3, 1, figsize=(32,18), sharex=True)

for label in angles_1_dict.keys():
    axes[label].hist(event_timing_dict[label], bins=200, density=False, label=label_dict[label][0], alpha=0.8,
                     color=label_dict[label][1])
    axes[label].legend(prop={"size":20})
    axes[label].set_xlabel("PMT timing, ns", fontsize=20)
    axes[label].set_ylabel("Frequency", fontsize=20)
    axes[label].tick_params(axis="both", labelsize=20)
    axes[label].set_yscale("log")
    axes[label].grid(True, which="both", axis="both")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1539827845334062, 'fraction_numerical': 0.03634376280912693, 'mean_word_length': 5.8283582089552235, 'pattern_counts': {'":': 4, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '252960', 'license': 'CNRI-Python', 'n_tokens_mistral': 2832, 'n_tokens_neox': 2609, 'n_words': 444, 'path': 'notebooks/2510 - New data exploration.ipynb', 'repo_name': 'WatChMaL/CNN', 'size': '7319'}