<jupyter_start><jupyter_text>Plot the histogram of the two observables    1. Sum of charges in an event    2. Number of PMTs hit  1. Sum of charges in an event<jupyter_code>train_batch_size = 1024
dset=WCH5Dataset(path, 0.1, 0.1, reduced_dataset_size=300000)

train_loader = DataLoader(dset, batch_size=train_batch_size, shuffle=False,
                          num_workers=4, sampler=SubsetRandomSampler(dset.train_indices))
event_charges = []
event_timings = []

labels = []
energies = []

for data in iter(train_loader):
    event_charges.append(data[0][:,:,:,:19].numpy())
    event_timings.append(data[0][:,:,:,19:].numpy())
    
    labels.append(data[1].numpy())
    energies.append(data[2].numpy())
print(len(event_charges), event_charges[0].shape, len(event_timings), event_timings[0].shape)
print(len(labels), labels[0].shape, len(energies), energies[0].shape)
event_charges = np.concatenate(event_charges, axis=0)
event_timings = np.concatenate(event_timings, axis=0)
labels = np.concatenate(labels, axis=0)
energies = np.concatenate(energies, axis=0)
print(event_charges.shape, event_timings.shape, labels.shape, energies.shape)<jupyter_output>(240000, 40, 40, 19) (240000, 40, 40, 19) (240000,) (240000, 1)<jupyter_text>Reshape the energy and labels np arrays<jupyter_code>labels = labels.reshape(-1)
energies = energies.reshape(-1)<jupyter_output><empty_output><jupyter_text>Distrbution of the labels<jupyter_code>print(Counter(labels))<jupyter_output>Counter({0: 80930, 1: 79829, 2: 79241})<jupyter_text>Select the events based on the energy<jupyter_code>min_energy = 40
max_energy = 50
energies_mask = np.logical_and(energies >= min_energy, energies <= max_energy)
energies_mask = energies_mask.reshape(energies_mask.shape[0])
print(energies_mask.shape)
local_event_charges = event_charges[energies_mask]
local_event_timings = event_timings[energies_mask]
local_labels        = labels[energies_mask].reshape(-1)
local_energies      = energies[energies_mask]
print(Counter(local_labels))
sum_event_charges = np.sum(local_event_charges.reshape(local_event_charges.shape[0], -1), axis=1)
sum_event_timings = np.sum(local_event_timings.reshape(local_event_timings.shape[0], -1), axis=1)
local_event_charges.reshape(local_event_charges.shape[0], -1).shape
print(sum_event_charges.shape, sum_event_timings.shape, local_labels.shape)<jupyter_output><empty_output><jupyter_text>Plot the per event sum charge distribution<jupyter_code>label_dict = {0:["gamma", "red"], 1:["e", "blue"], 2:["mu", "green"]}
event_charge_dict = {}
for label in label_dict.keys():
    event_charge_dict[label] = sum_event_charges[local_labels == label]
    
fig, axes = plt.subplots(3, 1, figsize=(32,18), sharex=True)

for label in label_dict.keys():
    axes[label].hist(event_charge_dict[label], bins=200, density=False, label=label_dict[label][0], alpha=0.8,
                     color=label_dict[label][1])
    axes[label].legend(prop={"size":30})
    axes[label].set_xlabel("Total event charge", fontsize=30)
    axes[label].set_ylabel("Frequency", fontsize=30)
    axes[label].tick_params(axis="both", labelsize=30)
    axes[label].grid(True, which="both", axis="both")<jupyter_output><empty_output><jupyter_text>Place a cut on the total observed event charge and plot the energy effciency<jupyter_code>print(event_charges.shape, event_timings.shape, labels.shape, energies.shape)
sum_global_event_charges = np.sum(event_charges.reshape(event_charges.shape[0], -1), axis=1)
sum_global_event_timings = np.sum(event_timings.reshape(event_timings.shape[0], -1), axis=1)
print(sum_global_event_charges.shape, sum_global_event_timings.shape)<jupyter_output><empty_output><jupyter_text>Placing a cut at 500 p.e.<jupyter_code>min_charge = 500

charges_mask = sum_global_event_charges >= min_charge
charges_mask = charges_mask.reshape(charges_mask.shape[0])
select_labels   = labels[charges_mask]
select_energies = energies[charges_mask]

print(select_labels.shape, select_energies.shape)<jupyter_output><empty_output><jupyter_text>Plot the original energy distribution<jupyter_code>energy_dict = {}
for label in label_dict.keys():
    energy_dict[label] = energies[labels == label]
    
fig, axes = plt.subplots(3, 1, figsize=(32,18), sharex=True)

for label in label_dict.keys():
    axes[label].hist(energy_dict[label], bins=200, density=False, label=label_dict[label][0], alpha=0.8,
                     color=label_dict[label][1])
    axes[label].legend(prop={"size":30})
    axes[label].set_xlabel(r"Total true energy, $MeV$", fontsize=30)
    axes[label].set_ylabel("Frequency", fontsize=30)
    axes[label].tick_params(axis="both", labelsize=30)
    axes[label].grid(True, which="both", axis="both")
    axes[label].set_ylim(0, 500)<jupyter_output><empty_output><jupyter_text>Plot the resulting energy distribution<jupyter_code>energy_dict = {}
for label in label_dict.keys():
    energy_dict[label] = select_energies[select_labels == label]
    
fig, axes = plt.subplots(3, 1, figsize=(32,18), sharex=True)

for label in label_dict.keys():
    axes[label].hist(energy_dict[label], bins=200, density=False, label=label_dict[label][0], alpha=0.8,
                     color=label_dict[label][1])
    axes[label].legend(prop={"size":30})
    axes[label].set_xlabel(r"Total true energy, $MeV$", fontsize=30)
    axes[label].set_ylabel("Frequency", fontsize=30)
    axes[label].tick_params(axis="both", labelsize=30)
    axes[label].grid(True, which="both", axis="both")
    axes[label].set_ylim(0, 500)<jupyter_output><empty_output><jupyter_text>Plot the efficiency at different energy intervals<jupyter_code>energy_dict = {}
select_energy_dict = {}
for label in label_dict.keys():
    energy_dict[label] = energies[labels == label]
    select_energy_dict[label] = select_energies[select_labels == label]

num_bins = 801
min_energy = min(np.amin(energies), np.amin(select_energies))
max_energy = max(np.amax(energies), np.amax(select_energies))

bins = np.arange(min_energy, max_energy, (max_energy-min_energy)/num_bins)
fig, axes = plt.subplots(3, 1, figsize=(32,18), sharex=True)

for label in label_dict.keys():
    values, _       = np.histogram(energy_dict[label], bins=bins)
    select_values, _ = np.histogram(select_energy_dict[label], bins=bins)
    
    #print(values, select_values)
    
    ratio = []
    for value, select_value in zip(values, select_values):
        curr_ratio = select_value/value if value != 0 else 0
        ratio.append(curr_ratio)
        
    #print(ratio)
        
    axes[label].plot(bins[:-1], ratio, label=label_dict[label][0], alpha=1.0, color=label_dict[label][1],
                     fillstyle="full", linewidth=5.0)
    axes[label].legend(prop={"size":30}, loc={0.85, 0.5})
    axes[label].set_xlabel(r"Total true energy, $MeV$", fontsize=30)
    axes[label].set_ylabel("Efficiency", fontsize=30)
    axes[label].tick_params(axis="both", labelsize=30)
    axes[label].grid(True, which="both", axis="both")
    axes[label].set_ylim(0, 1)
    axes[label].set_xlim(0, 200)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14425810119386015, 'fraction_numerical': 0.03268902785673678, 'mean_word_length': 6.108080808080808, 'pattern_counts': {'":': 4, '<': 40, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '88102', 'license': 'CNRI-Python', 'n_tokens_mistral': 2727, 'n_tokens_neox': 2535, 'n_words': 403, 'path': 'notebooks/0511 - Full tank dataset - Cut metrics 4.ipynb', 'repo_name': 'WatChMaL/CNN', 'size': '7036'}