<jupyter_start><jupyter_text>Read h5 files<jupyter_code>### This function reads the h5 files and saves the jets in numpy arrays
def h5_to_npy(filename,Njets,Nstart=None):
#     file = pandas.HDFStore(filename)
    file = pandas.read_hdf(filename)
    jets=np.array(file.select("table",start=Nstart,stop=Njets))
    
    
    jets2=jets[:,0:800].reshape((len(jets),200,4)) #This way I'm getting the 1st 199 constituents. jets[:,800:804] is the constituent 200. jets[:,804] has a label=0 for train, 1 for test, 2 for val. jets[:,805] has the label sg/bg
#     print('jets2=',jets2[0])
    labels=jets[:,805:806]
#     print('labels=',labels)
    npy_jets=[]
    for i in range(len(jets2)):
#         print('~np.all(jets2[i] == 0, axis=1)=',~np.all(jets2[i] == 0, axis=1))
        # Get the index of non-zero entries
        nonzero_entries=jets2[i][~np.all(jets2[i] == 0, axis=1)]
        npy_jets.append([nonzero_entries,0 if labels[i] == 0 else 1])
        
    file.close()
    return npy_jets
file = pandas.HDFStore('in_data/test.h5')
file
pd.DataFrame(file)
file.info()
file.keys()<jupyter_output><empty_output><jupyter_text>Check True labels<jupyter_code>def h5_to_npy_test(filename,Njets):
    file = pandas.HDFStore(filename)
    jets=np.array(file.select("table",start=None,stop=Njets))
    jets2=jets[:,0:800].reshape((len(jets),200,4))
#     print('jets2=',jets2[0])
    labels=jets[:,804::]
    print('labels=',labels)

# val=h5_to_npy('in_data/val.h5',None)

# np.shape(val)
# train=h5_to_npy('in_data/train.h5',None)
# np.shape(train)
# test_data=h5_to_npy('in_data/test.h5',None)
# np.shape(test_data)
# labels=np.asarray(test_data)[:,-1]
# np.shape(labels)
# with open('best_models/labelsTest.pkl', "wb") as f: pickle.dump(labels, f)<jupyter_output><empty_output><jupyter_text>Load .h5 and save np arrays<jupyter_code># train_data=h5_to_npy('in_data/train.h5',None)
# with open('out_data/train_jets.pkl', "wb") as f: pickle.dump(train_data, f, protocol=2)
# val_data=h5_to_npy('in_data/val.h5',None)
# with open('out_data/val_jets.pkl', "wb") as f: pickle.dump(val_data, f, protocol=2)
Nstart=0
test_data=h5_to_npy('in_data/test.h5',None,Nstart=Nstart)
print('test data shape=',np.shape(test_data))
# with open('out_data/test_jets.pkl', "wb") as f: pickle.dump(test_data, f, protocol=2)
# LOAD FULL DATASEST
# test_data_full=h5_to_npy('in_data/test.h5',None,Nstart=0)
# with open('out_data/test_jets_last.pkl', "wb") as f: pickle.dump(test_data_last, f, protocol=2)
# labels_full=np.asarray(test_data_full)[:,-1]
# with open('comparison_all_groups/labels.pkl', "wb") as f: pickle.dump(labels_full, f, protocol=2)
# len(labels_full)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17682242990654207, 'fraction_numerical': 0.03663551401869159, 'mean_word_length': 5.774683544303797, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '520023', 'license': 'MIT', 'n_tokens_mistral': 1178, 'n_tokens_neox': 1103, 'n_words': 174, 'path': 'code/top_reference_dataset/.ipynb_checkpoints/Read_data-checkpoint.ipynb', 'repo_name': 'SebastianMacaluso/RecNN_PyTorch', 'size': '2675'}