<jupyter_start><jupyter_text>* ETL_trinity_predictetoh_before_tap : get worm behavior data from trinity_clean.csv    * only from wildtype    * only from before first tap    * only from 400mM and 0mM groups<jupyter_code>import pandas as pd
import numpy as np
import os
import sys
import re
# import functions
dir_function_collection = [
'/Users/connylin/Dropbox/Code/language/python_lib/localpackage/mwt',
'/Users/connylin/Dropbox/CA/ED _20200119 Brain Station Data Science Diploma/Capstone/brainstation_capstone/0_lib'
]
for p in dir_function_collection:
    sys.path.insert(1, p)
import BrainStationLib as bs
# define local function 
def extract_MWT_feature_preplate(pmwt):
    p_parsed = pmwt.split('/')
    if os.path.isfile(pmwt):
        expname = p_parsed[-4]
    elif os.path.isfile(ptrinity):
        expname = p_parsed[-3]
    a = re.search(r'(?<=_)\d{1,}', expname)
    preplate_time = int(a.group(0))
    return preplate_time
# define local variable
pMWTDB = '/Users/connylin/Dropbox/MWT/db/MWTDB.csv'
pCapstone = '/Users/connylin/Dropbox/CA/ED _20200119 Brain Station Data Science Diploma/Capstone/data'
# load MWTDB from dropbox
MWTDB = pd.read_csv(pMWTDB)
# get only N2 and N2_400mM plates
i = MWTDB['groupname'].isin(['N2','N2_400mM'])
print(f'{sum(i)} plates found')
# keep only those plates
MWTDB = MWTDB.loc[i,:].copy()
MWTDB.shape
# get trinity paths for all and check how many exist
ptrinity_list = list(map(lambda x: os.path.join(x,'trinity_cleaned.csv'), 
                         MWTDB['mwtpath']))
ptrinity_exist = list(map(lambda x: os.path.isfile(x), ptrinity_list))
print(f'{sum(ptrinity_exist)} trinity_cleaned.csv exist')
ptrinity_list = np.array(ptrinity_list)
ptrinity_list = ptrinity_list[ptrinity_exist].copy()
nfiles = ptrinity_list.shape[0]
print(nfiles)
# get only data before preplate (first tap)
df = pd.read_csv(ptrinity_list[0])
# get preplate time
preplate_time = extract_MWT_feature_preplate(pmwt)
# reduce data
df = df.loc[df['time'] < preplate_time,:].copy()
# get sizse
bytesize = sys.getsizeof(df)
# estimate total size and see if can concat
print(f'estimated total {bytesize/(1000**2)*nfiles:.1f} MB')
# look into each trinity_clean.csv
df_collection = []
for i, ptrinity in enumerate(ptrinity_list):
    print(f'processing {i} file', end='\r')
    # get only data before preplate (first tap)
    df = pd.read_csv(ptrinity)
    # get preplate time
    preplate_time = extract_MWT_feature_preplate(pmwt)
    # get only sub set
    df = df.loc[df['time'] < preplate_time,:].copy()
    # append
    df_collection.append(df)
df_collection = pd.concat(df_collection, ignore_index=True)
df_collection.shape
# get sizse
bytesize = sys.getsizeof(df_collection)
# estimate total size and see if can concat
print(f'estimated total {bytesize/(1000**3):.1f} GB')
# save
df_collection.to_csv(os.path.join(pCapstone, 'trinity_cleaned_N2_etoh_b4tap.csv'))
df_collection<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11975435005117707, 'fraction_numerical': 0.017741385192766974, 'mean_word_length': 4.9714867617107945, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '302270', 'license': 'MIT', 'n_tokens_mistral': 1108, 'n_tokens_neox': 1052, 'n_words': 241, 'path': '9_not_in_final/ETL_trinity_predictetoh_behav_beforetap.ipynb', 'repo_name': 'conny-lin/brainstation_capstone', 'size': '2931'}