<jupyter_start><jupyter_text>get files under all MWT folders in dir_driveexample path: /Volumes/COBOLT/MWT/20190418X_XX_100s30x10s10s_slo1/VG903_400mM/20190418_141335/VG903_OH_15x3_t96h20C_100s30x10s10s_A_0418_jv410014.pngparse the file names into:* extension* filename prefix* filename suffix (e.g. shanespark)* mwt name* group name* expname    * exp date    * tracker    * experimenter    * exp condition        * pre-plate        * taps        * ISI        * post-tap    * exp name tag* MWT DB source (e.g. MWT, MWT bad)<jupyter_code># get all unique suffix from all file paths
# load all file paths
filename = os.path.join(dir_save,'allfilepaths.pickle')
with open(filename,'rb') as f:
    filepaths = pickle.load(f)
# make it into dataframe
df_filepaths = pd.DataFrame(filepaths,columns=['filepath'])
# transform files into path obj
file_path_obj = list(map(PurePath,df_filepaths['filepath']))
# get MWT path
df_filepaths['mwtpath'] = np.array(list(map(lambda x: os.path.dirname(x), df_filepaths['filepath'])))
# get suffix
df_filepaths['suffix'] = np.array(list(map(lambda x: x.suffix, file_path_obj)))

df_filenames  = pd.DataFrame(list(map(lambda x: x.name, file_path_obj)),
                             columns=['filenames'])
# get .dat
a = df_filenames['filenames'].str.extract(r'(?<=[.])([A-z]{1,}\d{0,}[.]dat)')
ind_datext = ~a.iloc[:,0].isna()
df_filepaths.loc[ind_datext, 'suffix'] = a.loc[ind_datext,0]
# get xx.000.dat
a = df_filenames['filenames'].str.extract(r'(?<=[.])([A-z]{1,}[.]\d{1,}[.]dat)')
b = a.iloc[:,0].str.replace(r'(?<=[.])(\d{3,})(?=[.]dat)','id')
ind_datext = ~b.isna()
df_filepaths.loc[ind_datext, 'suffix'] = b[ind_datext]
# count suffix by plate
df = df_filepaths.groupby(['mwtpath', 'suffix']).count()
df_suffix_count = df.unstack(level=-1)
df_suffix_count
# save file dataframe
path_save = os.path.join(dir_save, 'file_summary_mwt.pickle')
with open(path_save,'wb') as f:
    pickle.dump(df_suffix_count,f)
print('complete')<jupyter_output>complete
{'fraction_non_alphanumeric': 0.1691842900302115, 'fraction_numerical': 0.039778449144008056, 'mean_word_length': 5.190031152647975, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '299813', 'license': 'MIT', 'n_tokens_mistral': 834, 'n_tokens_neox': 777, 'n_words': 142, 'path': '0_developed_draft/file_summary_mwt_chor_outputs.ipynb', 'repo_name': 'conny-lin/brainstation_capstone', 'size': '1986'}