<jupyter_start><jupyter_text>0 | Import packages<jupyter_code>import os
import h5py
import numpy as np
import pandas as pd
import tkinter
from tkinter.filedialog import askopenfilename, askopenfilenames
from collections import defaultdict
from nptdms import TdmsFile
print("done!")<jupyter_output>done!<jupyter_text>1 | Open an .hdf5 file and extract channels<jupyter_code># This was Yu Lin's function:
def openHDF5file(in_path, channel_list = ['Channel A', 'Channel B', 'Output A', 'Output B'], curated_channel = None):
    hdf5_file = h5py.File(in_path, 'r')
     
    dataDict = defaultdict(list)
     
    for channel in hdf5_file.keys():
        
        #to fix hdf5 indexing
        if 'Channel' in channel or 'Output' in channel:
            trialKeysInHDF5 = list(hdf5_file[channel].keys()) # which has been sorted alphabatically as in strings. can be simulated by sorted([str(x) for x in range(numTrials)]) 
            sortI, numericalOrder = zip(*sorted(enumerate([int(x) for x in trialKeysInHDF5]),key=lambda i:i[1])) # resort numerically, and get the sorting indexes 
            
            if curated_channel is not None: 
                sortI, numericalOrder = zip(*sorted(enumerate([int(x) for x in list(hdf5_file[curated_channel].keys())]),
                                                    key=lambda i:i[1]))
                if 'Channel' in channel: trialKeysInHDF5 = list(hdf5_file[curated_channel].keys())
                if 'Output' in channel: sortI = [int(x)-1 for x in numericalOrder] #note, may not work as expected if very first trial is deleted ? but doesnt matter as output is usually the same across trials in a given file
#            print(channel, trialKeysInHDF5, sortI)
        else:
            trialKeysInHDF5 = list(hdf5_file[channel].keys())
            sortI = range(len(trialKeysInHDF5))
#            print('not sorted')
            
        for i in range(len(sortI)):
            correctedTrialKey = trialKeysInHDF5[sortI[i]] 
            dataDict[channel].append(np.array(hdf5_file[channel][correctedTrialKey]))
   
    extractedChannels = []
    
    for channel in channel_list:
        extractedChannels.append(dataDict[channel])

    
    if len(dataDict['Time']) >0:
        time = dataDict['Time'][0]
        dt = np.mean(np.diff(time))
    else:
        dt = 0.04
        time = np.linspace(0, len(dataDict['Channel A'][0])*dt, len(['Channel A'][0]))
    
    return extractedChannels, time, dt<jupyter_output><empty_output><jupyter_text>1.1 | Load an .hdf5 file<jupyter_code># Choose a .hdf5 file to test the code
root = tkinter.Tk()
root.attributes('-topmost', True) # Make window appear on top
in_path = askopenfilename() # Open dialogue to select file
print(in_path)
root.destroy() # Close the root window

# Split the pathname path into a pair, (head, tail) where tail is the last pathname component and head is everything leading up to that.
folder_name = os.path.split(in_path)[0] # Get path until folder
file_name = os.path.split(in_path)[1] # Get filename
print(folder_name)
print(file_name)<jupyter_output>D:/Dropbox (UCL)/Project_paginhibition/analysis/loose_seal/analysis_sample_cells/dlpag_vgat_171114_c4_LIAM_OP_clear_VC_2.hdf5
D:/Dropbox (UCL)/Project_paginhibition/analysis/loose_seal/analysis_sample_cells
dlpag_vgat_171114_c4_LIAM_OP_clear_VC_2.hdf5<jupyter_text>1.2 | Check how zip() and sorted() work<jupyter_code>#print(in_path)
test_file = h5py.File(in_path, 'r')
print(list(test_file.keys()))
test_file['Channel B'].keys()

trialKeysInHDF5 = list(test_file['Channel B'].keys())
print(trialKeysInHDF5)

# This is how Yu Lin was doing the sorting:
dataDict = defaultdict(list)
sortI, numericalOrder = zip(*sorted(enumerate([int(x) for x in trialKeysInHDF5]), 
                                    key = lambda i:i[1]))

print(sortI)
print(numericalOrder)
# How zip() and sorted() work:
a = ("John", "Charles", "Mike")
b = (2, 1, 3)
x = sorted(zip(b, a))
list(x)<jupyter_output><empty_output><jupyter_text>2 | Test how the core of the function will work with one channel<jupyter_code>test_file = h5py.File(in_path, 'r')

# Get trial keys from one of the channels:
a = list(test_file['Channel B'].keys())
print(a)

# Transform to integer:
b = [int(x) for x in a]
print(b)

# If trial keys don't start at 1, force them to (useful for "Output" channels):
c = [(x-(min(b))+1) for x in b]
print(c)

# Sort the integers:
d = sorted(c)
print(d)

# Use numpy argsort() to get the indices that would sort the array:
e = list(np.argsort(c, axis=None))
print(e)

# Turn integers into indices (i.e. starting from 0)
f = [(x-b[0]) for x in b]
print(f)

# Get a list of sorted trial keys
# Note this is not useful as you need the indices to access the keys in a sorted way so you can extract the data from the right sweep.
z = [x for _, x in sorted(zip(c, a))]
print(z)

corrected_test_file = defaultdict(list)
for i in range(len(e)):
    correctedTrialKey = a[e[i]]
    corrected_test_file['Channel B'].append(np.array(test_file['Channel B'][correctedTrialKey]))
print(corrected_test_file['Channel B'][0:10])

uncorrected_test_file = defaultdict(list)
for i in range(len(f)):
    uncorrectedTrialKey = a[f[i]]
    uncorrected_test_file['Channel B'].append(np.array(test_file['Channel B'][uncorrectedTrialKey]))
print(uncorrected_test_file['Channel B'][0:10])
#print(in_path)
test_file = h5py.File(in_path, 'r')
print(list(test_file.keys())) # Check channel names in file

trialKeysInHDF5 = list(test_file['Output B'].keys())
print(trialKeysInHDF5) # Check trial names in a channel

trialKeysInHDF5_int = [int(x) for x in trialKeysInHDF5]
print(trialKeysInHDF5_int)

trialKeysInHDF5_int.sort()
print(trialKeysInHDF5_int)
print(range(len(list(test_file['Time'].keys())))) # check items in time
print(test_file['Output B'].keys()) # check trial names
test_file['Output B']['11795'] # check data in a sweep
test_file = h5py.File(in_path, 'r')
a = list(test_file['Channel B'].keys())
a_curated = list(test_file['Sweeps_Analysis'].keys())
print(a)
print(a_curated)

b = [int(x) for x in a]
b_curated = [int(x) for x in a_curated]
print(b)
print(b_curated)

c = [(x-(min(b))+1) for x in b]
c_curated = [(x-(min(b_curated))+1) for x in b_curated]
print(c)
print(c_curated)

e = list(np.argsort(c))
e_curated = list(np.argsort(c_curated))
print(e)
print(e_curated)

f = list(np.argsort(b))
f_curated = list(np.argsort(b_curated))
print(f)
print(f_curated)


# Test data extraction using the corrected indices
dataDict = defaultdict(list)
extractedChannels = []
for i in range(len(f_curated)):
    corrected_a_curated = a_curated[f_curated[i]]
    dataDict['Sweeps_Analysis'].append(np.array(test_file['Sweeps_Analysis'][corrected_a_curated]))
print(list(dataDict['Sweeps_Analysis']))
extractedChannels.append(dataDict['Sweeps_Analysis'])
print(extractedChannels)
test_file = h5py.File(in_path, 'r')

# Get keys from hdf5
trialKeysInHDF5 = list(test_file['Output B'].keys())
trialKeysInHDF5_curated = list(test_file['Sweeps_Analysis'].keys())
print(trialKeysInHDF5)
print(trialKeysInHDF5_curated)

# Convert to integers so you can sort numerically
trialKeysInHDF5_int = [int(x) for x in trialKeysInHDF5]
trialKeysInHDF5_curated_int = [int(x) for x in trialKeysInHDF5_curated]
print(trialKeysInHDF5_int)
print(trialKeysInHDF5_curated_int)

# Fix those cases when trial names start at a random number (e.g. sometimes "Output" start at "14197", whereas "Channel" always start at "1")
trialKeysInHDF5_int_from1 = [(x-min(trialKeysInHDF5_int)+1) for x in trialKeysInHDF5_int] # aka numerical order
print(trialKeysInHDF5_int_from1)

trialKeysInHDF5_sorting_indices = list(np.argsort(trialKeysInHDF5_int_from1)) # aka sortIndices
print(trialKeysInHDF5_sorting_indices)

trialKeysInHDF5_sorting_indices_2 = list(np.argsort(trialKeysInHDF5_int)) # aka sortIndices
trialKeysInHDF5_curated_sorting_indices_2 = list(np.argsort(trialKeysInHDF5_curated_int)) # aka sortIndices
print(trialKeysInHDF5_sorting_indices_2)
print(trialKeysInHDF5_curated_sorting_indices_2)

trialKeysInHDF5_sorting_indices == trialKeysInHDF5_sorting_indices_2


# corrected_test_file = defaultdict(list)
# for i in range(len(trialKeysInHDF5_sorting_indices)):
#      correctedTrialKey = trialKeysInHDF5[trialKeysInHDF5_sorting_indices[i]]
#      corrected_test_file['Channel B'].append(np.array(test_file['Channel B'][correctedTrialKey]))
# For the curated version of an Output:
test_file = h5py.File(in_path, 'r')

# Get keys from hdf5
trialKeysInHDF5 = list(test_file['Output B'].keys())
trialKeysInHDF5_curated = list(test_file['Sweeps_Analysis'].keys())
print(trialKeysInHDF5)
print(trialKeysInHDF5_curated)

# Convert to integers so you can sort numerically
trialKeysInHDF5_int = [int(x) for x in trialKeysInHDF5]
trialKeysInHDF5_curated_int = [int(x) for x in trialKeysInHDF5_curated]
print(trialKeysInHDF5_int)
print(trialKeysInHDF5_curated_int)

# Fix those cases when trial names start at a random number (e.g. sometimes "Output" start at "14197", whereas "Channel" always start at "1")
trialKeysInHDF5_int_from1 = [(x-min(trialKeysInHDF5_int)+1) for x in trialKeysInHDF5_int] # aka numerical order
print(trialKeysInHDF5_int_from1)

# trialKeysInHDF5_sorting_indices = list(np.argsort(trialKeysInHDF5_int_from1)) # aka sortIndices
# print(trialKeysInHDF5_sorting_indices)

trialKeysInHDF5_sorting_indices_2 = list(np.argsort(trialKeysInHDF5_int)) # aka sortIndices
trialKeysInHDF5_curated_sorting_indices_2 = list(np.argsort(trialKeysInHDF5_curated_int)) # aka sortIndices
print(trialKeysInHDF5_sorting_indices_2)
print(trialKeysInHDF5_curated_sorting_indices_2)

a = sorted(trialKeysInHDF5_curated_int)
b = trialKeysInHDF5_int_from1
print(a)
print(b)
print(b.index(a[0]))
c = [b.index(a[i]) for i in range(len(a))]
print(c)
test_file['Output B'][trialKeysInHDF5[c[0]]]
# Check the output channel loop
trialKeysInHDF5 = list(test_file['Output B'].keys())
print(trialKeysInHDF5)
trialKeysInHDF5_int = [int(x) for x in trialKeysInHDF5]
print(trialKeysInHDF5_int)
trialKeysInHDF5_sorting_indices = list(np.argsort(trialKeysInHDF5_int))
print(trialKeysInHDF5_sorting_indices)
# Transform them so they start from 1 and can be compared to the curated_channel keys:
trialKeysInHDF5_int_from1 = [(x-min(trialKeysInHDF5_int)+1) for x in trialKeysInHDF5_int]
print(trialKeysInHDF5_int_from1)

trialKeysInHDF5_curated = list(test_file['Sweeps_Analysis'].keys())
print(trialKeysInHDF5_curated)
trialKeysInHDF5_curated_int = [int(x) for x in trialKeysInHDF5_curated]
print(trialKeysInHDF5_curated_int)
trialKeysInHDF5_curated_sorting_indices = list(np.argsort(trialKeysInHDF5_curated_int))
print(trialKeysInHDF5_curated_sorting_indices)

# Sort the curated integer keys so you can use them in the list.index() step
trialKeysInHDF5_curated_int_sorted = sorted(trialKeysInHDF5_curated_int)
print(trialKeysInHDF5_curated_int_sorted)

# For the sorted curated trial keys, find the index of the value matching each curated_channel trial.
# Use this as the sorting indices
trialKeysInHDF5_sorting_indices = [trialKeysInHDF5_int_from1.index(trialKeysInHDF5_curated_int_sorted[i]) for i in range(len(trialKeysInHDF5_curated_int_sorted))]
print(trialKeysInHDF5_sorting_indices)<jupyter_output>['11795', '11796', '11797', '11798', '11799', '11800', '11801', '11802', '11803', '11804', '11805', '11806', '11807', '11808', '11809', '11810', '11811', '11812', '11813', '11814', '11815', '11816', '11817', '11818', '11819', '11820', '11821', '11822', '11823', '11824', '11825', '11826', '11827', '11828', '11829', '11830', '11831', '11832', '11833', '11834', '11835', '11836', '11837', '11838', '11839', '11840', '11841', '11842', '11843', '11844', '11845', '11846', '11847', '11848', '11849', '11850', '11851', '11852', '11853', '11854', '11855', '11856', '11857', '11858', '11859', '11860', '11861', '11862', '11863', '11864', '11865', '11866', '11867', '11868', '11869', '11870', '11871', '11872', '11873', '11874', '11875', '11876', '11877', '11878', '11879', '11880', '11881', '11882', '11883', '11884', '11885', '11886', '11887', '11888', '11889', '11890', '11891', '11892', '11893', '11894']
[11795, 11796, 11797, 11798, 11799, 11800, 11801, 11802, 11803, 11804, 11805, 11806, 11807, 11808, [...]<jupyter_text>2.1 | Test data extraction<jupyter_code>dataDict['Sweeps_Analysis']
# Define empty dictionary to populate with correctly sorted data:
data_dict = defaultdict(list)
# Define empty dictionary to populate with corrected trialKeys:
key_dict = defaultdict(list)
print(len(trialKeysInHDF5_sorting_indices))
for i in range(len(trialKeysInHDF5_sorting_indices)):
    correctedTrialKey = trialKeysInHDF5[trialKeysInHDF5_sorting_indices[i]] 
    data_dict['Output B'].append(np.array(test_file['Output B'][correctedTrialKey]))
    key_dict['Output B'].append(correctedTrialKey)
    print(data_dict['Output B'])
    print(key_dict['Output B'])

print(len(trialKeysInHDF5_curated))
print(len(key_dict['Output B']))
extracted_channels = []
corrected_trial_keys = []

# Keep only the useful channels and their trial keys:
extracted_channels.append(data_dict['Output B'])
print(len(extracted_channels[0]))
corrected_trial_keys.append(key_dict['Output B'])
print(len(corrected_trial_keys[0]))
    
# Create data frame of data:
channel_names = ['Output B']
channels_data_frame = pd.DataFrame(extracted_channels, index = channel_names, columns = corrected_trial_keys[0])
channels_data_frame<jupyter_output>36
[array([0., 0., 0., ..., 0., 0., 0.])]
['11807']
[array([0., 0., 0., ..., 0., 0., 0.]), array([0., 0., 0., ..., 0., 0., 0.])]
['11807', '11808']
[array([0., 0., 0., ..., 0., 0., 0.]), array([0., 0., 0., ..., 0., 0., 0.]), array([0., 0., 0., ..., 0., 0., 0.])]
['11807', '11808', '11809']
[array([0., 0., 0., ..., 0., 0., 0.]), array([0., 0., 0., ..., 0., 0., 0.]), array([0., 0., 0., ..., 0., 0., 0.]), array([0., 0., 0., ..., 0., 0., 0.])]
['11807', '11808', '11809', '11810']
[array([0., 0., 0., ..., 0., 0., 0.]), array([0., 0., 0., ..., 0., 0., 0.]), array([0., 0., 0., ..., 0., 0., 0.]), array([0., 0., 0., ..., 0., 0., 0.]), array([0., 0., 0., ..., 0., 0., 0.])]
['11807', '11808', '11809', '11810', '11811']
[array([0., 0., 0., ..., 0., 0., 0.]), array([0., 0., 0., ..., 0., 0., 0.]), array([0., 0., 0., ..., 0., 0., 0.]), array([0., 0., 0., ..., 0., 0., 0.]), array([0., 0., 0., ..., 0., 0., 0.]), array([0., 0., 0., ..., 0., 0., 0.])]
['11807', '11808', '11809', '11810', '11811', '11812'[...]<jupyter_text>2.2 | Make a pandas data frame<jupyter_code># np.array(test_file['Sweeps_Analysis']['13'])
# channel_list = ['Channel A', 'Channel B']
# data = [np.arange(0, 18), np.arange(100, 118)]
# print(data)
# print(extractedChannels)
indices = ['Sweeps_Analysis']
df = pd.DataFrame(extractedChannels, index = indices, columns = a_curated)
df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14938640132669984, 'fraction_numerical': 0.06918739635157546, 'mean_word_length': 4.77624521072797, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '452493', 'license': 'MIT', 'n_tokens_mistral': 6361, 'n_tokens_neox': 5626, 'n_words': 1286, 'path': 'loose_seal/develop/loose_seal_0_openHDF5_develop.ipynb', 'repo_name': 'opavon/PAG_ephys_analysis', 'size': '15075'}