<jupyter_start><jupyter_text>Code development<jupyter_code>df = api.database.excitatory_voltage_clamp_recordings.copy()
api.database.excitatory_voltage_clamp_recordings.columns
import pandas as pd
include = {'cell_type': 'GABA'}
exclude = {'session_cell_id': 'C3'}
def apply_inclusion_criteria(df: DataFrame, inclusion_criteria: Dict) -> DataFrame:
    chunks_of_df_to_include = []
    for key, value in inclusion_criteria.items():
        if key not in df.columns:
            raise ValueError(f'{key} has to be in columns of the DataFrame. Please find a list of all possible columns: {list(df.columns)}')
        else:
            if type(value) == str:
                chunks_of_df_to_include.append(df.loc[df[key] == value].copy())
            else:
                raise ValueError(f'{value} is not a valid input! Only strings (e.g. "vlPAG") are possible, simply add a column multiple times with different values if needed.')
    df_after_inclusions = pd.concat(chunks_of_df_to_include, ignore_index = True)
    df_after_inclusions = df_after_inclusions.drop_duplicates(ignore_index = True)
    return df_after_inclusions

def apply_exclusion_criteria(df: DataFrame, exclusion_criteria: Dict) -> DataFrame:
    df_after_exclusions = df.copy()
    for key, value in exclusion_criteria.items():
        if key not in df.columns:
            raise ValueError(f'{key} has to be in columns of the DataFrame. Please find a list of all possible columns: {list(df.columns)}')
        else:
            if type(value) == str:
                df_after_exclusions = df_after_exclusions.loc[df_after_exclusions[key] != value]
            else:
                raise ValueError(f'{value} is not a valid input! Only strings (e.g. "vlPAG") are possible, simply add a column multiple times with different values if needed.')
    return df_after_exclusions

4 % 4
df_after_inclusions
df['cell_type'].unique()
import seaborn as sns

columns = list(df.columns)
relevant_columns = columns[columns.index('pharmacology') + 1 : columns.index('filepath_main_excel_sheet')]

for column in relevant_columns:
    sns.stripplot(data=df.loc[df['cell_type'] == 'GABA'], y = column, x = 'stimulation_string')
def run_analysis(self, group_column: str, group_id: str, show: bool, save: bool):
    self.group_column = group_column
    self.group_id = group_id
    
    
    
    filepaths = self.get_all_recording_filepaths()
    for recording_filepath in filepaths:
        self.get_data_for_cumulative_distributions(filepath = recording_filepath)
    df_all_events = pd.DataFrame(data=self.events_all_stim_paradigms)
    self.plot_cdfs(data = df_all_events,
                   columns_for_cdfs = ['amplitude', 'inter_event_interval'], 
                   show = show,
                   save = save)
name = '.2021_12_09_C2'
if name.startswith('.') == False:
    print('no_hidden_file')
[3] * 5 + [7]
relevant_columns1 = ['recording_type','pharmacology', 'filepath_main_excel_sheet', 'filepath_detected_events']
relevant_columns2 = ['stimulation_string', 'stimulation_frequency-Hz', 'stimulation_duration-ms']

relevant_columns1 + relevant_columns2
import pandas as pd
import numpy as np
from pathlib import Path
import datetime
path = Path('/mnt/c/Users/dsege/TEMP/Alexia/NET-cre_vGat-flp/2021_12_09_C2/')
test = pd.read_excel('/mnt/c/Users/dsege/TEMP/Alexia/NET-cre_vGat-flp/2021_12_09_C2/2021_12_09_C2.xlsx', sheet_name = None)
input_df = test['Recordings and cell properties']
'stimulation_string', 'stimulation_frequency-Hz', 'stimulation_duration-ms', 'filepath_detected_events'
input_df.columns = input_df.iloc[1]
input_df = input_df.iloc[2:, 1:]
input_df = input_df.loc[input_df['Type of protocol'].notnull()]

columns = list(input_df.columns)
relevant_data_columns = columns[columns.index('Pharmacology') +1 :]

all_final_single_stimulation_dfs = []
for row_index in range(input_df.shape[0]):
    single_stimulation_df = input_df.iloc[[row_index]].copy()
    if type(single_stimulation_df['Time since cutting'].iloc[0]) == datetime.time:
        time_since_cutting = single_stimulation_df['Time since cutting'].iloc[0].hour * 60 + single_stimulation_df['Time since cutting'].iloc[0].minute
    else:
        time_since_cutting = 'not_available'

    if single_stimulation_df['Vh (mV)'].iloc[0] == -70:
        postsynaptic_current_type = 'excitatory'
    elif single_stimulation_df['Vh (mV)'].iloc[0] == 0:
        postsynaptic_current_type = 'inhibitory'
    else:
        raise ValueError(f'{single_stimulation_df["Vh (mV)"].iloc[0]} is not a valid entry for holding potential!')

    if pd.isna(single_stimulation_df['Pharmacology'].iloc[0]):
        pharmacology = 'none'
    else:
        pharmacology = single_stimulation_df['Pharmacology'].iloc[0]

    if single_stimulation_df['Type of protocol'].iloc[0] == 'Bsl':
        stimulation_frequency = np.NaN
        stimulation_duration = np.NaN
        stimulation_string = 'baseline'
        # filepath?
    elif single_stimulation_df['Type of protocol'].iloc[0] == 'Bsl2':
        stimulation_frequency = np.NaN
        stimulation_duration = np.NaN
        stimulation_string = 'control_baseline'
        # filepath?
    elif 'Hz' in single_stimulation_df['Type of protocol'].iloc[0]:
        stimulation_frequency = single_stimulation_df['Type of protocol'].iloc[0].replace(' Hz', '')
        stimulation_duration = int(single_stimulation_df['Timing of stim (s)'].iloc[0] * 1000)
        stimulation_string = f'{stimulation_frequency}-Hz_for_{stimulation_duration}-ms'
        # filepath


    sorted_data = {'postsynaptic_current_type': [postsynaptic_current_type],
                   'stimulation_string': [stimulation_string],
                   'stimulation_frequency-Hz': [stimulation_frequency],
                   'stimulation_duration-ms': [stimulation_duration],
                   'pharmacology': [pharmacology],
                   'time_since_cutting-M': [time_since_cutting]}

    sorted_single_stimulation_df = pd.DataFrame(data = sorted_data)
    remaining_data_df = single_stimulation_df[relevant_data_columns].copy()
    remaining_data_df = remaining_data_df.reset_index(drop=True)
    all_final_single_stimulation_dfs.append(pd.concat([sorted_single_stimulation_df, remaining_data_df], axis=1))
recording_specific_dataframe = pd.concat(all_final_single_stimulation_dfs)
final_df = pd.concat([general_metadata_df, recording_specific_dataframe], axis=1)
final_df.insert(0, 'global_cell_id', [global_cell_id]*final_df.shape[0])
metadata_df = test['General information']
general_metadata = {'date': path.name[:10],
                    'session_cell_id': path.name[path.name.rfind('_') + 1:],
                    'mouse_line': metadata_df['Animal line'][0],
                    'brain_region': metadata_df['Region'][0],
                    'cell_type': metadata_df['Type'][0],
                    'sex': metadata_df['Sex'][0],
                    'age': metadata_df['Age (days)'][0],
                    'stimulation_in_percent': metadata_df['Stimulation (%)'][0],
                    'internal_patch_solution': metadata_df['Internal used'][0]}
general_metadata_df = pd.DataFrame(general_metadata, index=[0])
general_metadata_df = pd.DataFrame([general_metadata]*3)
general_metadata_df
list(general_metadata_df['date'].values)
global_cell_id = '0000'
final_df = pd.concat([general_metadata_df, recording_specific_dataframe], axis=1)
final_df.insert(0, 'global_cell_id', [global_cell_id]*final_df.shape[0])
a = {'hund': 'dog', 
     'katze': 'cat'}

for key, value in a.items():
    print(key, value)
pd.concat([final_df, final_df])
time_since_cutting = single_stimulation_df['Time since cutting'].iloc[0].hour * 60 + single_stimulation_df['Time since cutting'].iloc[0].minute

remaining_data_df
single_stimulation_df[data_columns]
import datetime
isinstance(time_since_cutting, datetime.time)
tmp_str = tmp_str.replace(' Hz', '')
tmp_str
'stimulation_string', 'stimulation_frequency-Hz', 'stimulation_duration-ms', 'filepath_detected_events'
database = api.database
database.__dict__.keys()
database.subdirectories
df = api.database.cell_recordings_metadata
df['sex'].unique().shape[0]
list(df.loc[df['sex'] == 'Female', 'global_cell_id'].unique())
import os
os.path.isdir('/mnt/c/Users/dsege/TEMP/Alexia/analysis_with_dcl_patch/')
p = Path('/mnt/c/Users/dsege/TEMP/Alexia/analysis_with_dcl_patch/')
p.joinpath('results/')
import os
from pathlib import Path
from typing import Dict

import pandas as pd
import numpy as np
def get_general_metadata_df(path_to_recordings_dir: Path) -> pd.DataFrame:
    metadata_df = pd.read_excel(path_to_recordings_dir.joinpath(f'{path_to_recordings_dir.name}.xlsx'),
                                sheet_name = 'General information')
    general_metadata = {'date': path_to_recordings_dir.name[:10],
                        'session_cell_id': path_to_recordings_dir.name[path_to_recordings_dir.name.rfind('_') + 1:],
                        'mouse_line': metadata_df['Animal line'][0],
                        'brain_region': metadata_df['Region'][0],
                        'cell_type': metadata_df['Type'][0],
                        'sex': metadata_df['Sex'][0]}
    return pd.DataFrame(general_metadata, index=[0])
def get_stimulation_paradigms_df(path_to_recordings_dir: Path) -> pd.DataFrame:
    filepaths_stimulation_recordings = list()
    for elem in path_to_recordings_dir.iterdir():
        if 'datapoints' not in elem.name:
            if elem.name.endswith('.csv'):
                filepaths_stimulation_recordings.append(elem)
    stimulation_paradigms = {'stimulation_string': list(),
                             'stimulation_frequency-Hz': list(),
                             'stimulation_duration-ms': list(), 
                             'filepath_detected_events': list()}
    for filepath in filepaths_stimulation_recordings:
        filename = filepath.name
        filename = filename.replace('.csv', '')
        yyyy_mm_dd_ = filename[:11]
        filename = filename.replace(yyyy_mm_dd_, '')
        cell_id = filename[:filename.index('_')]
        filename = filename.replace(cell_id + '_', '')
        if 'Hz' in filename:
            stimulation_frequency = int(filename[:filename.index('Hz')])
            stimulation_duration = filename[filename.index('Hz') + 2 :]
            if 'ms' not in stimulation_duration:
                stimulation_duration = int(stimulation_duration[:stimulation_duration.find('s')]) * 1000
            else:
                stimulation_duration = int(stimulation_duration[:stimulation_duration.find('ms')])
            stimulation_paradigm = f'{stimulation_frequency}-Hz_for_{stimulation_duration}-ms'
        elif 'Bsl' in filename:
            stimulation_frequency = np.NaN
            stimulation_duration = np.NaN
            stimulation_paradigm = 'baseline'
        else:
            print(f'Warning: stimulation paradigm could not be identified for: {filepath.name}')
            stimulation_frequency = np.NaN
            stimulation_duration = np.NaN
            stimulation_paradigm = 'unknown'
        stimulation_paradigms['stimulation_string'].append(stimulation_paradigm)
        stimulation_paradigms['stimulation_frequency-Hz'].append(stimulation_frequency)
        stimulation_paradigms['stimulation_duration-ms'].append(stimulation_duration)
        stimulation_paradigms['filepath_detected_events'].append(filepath)
    return pd.DataFrame(stimulation_paradigms)
global_cell_id = 1
str(global_cell_id).zfill(4)
def get
    general_metadata = get_general_metadata_df(cell_recording_dir)
    stimulation_paradigms = get_stimulation_paradigms_df(cell_recording_dir)
    stimulations_adjusted_general_metadata = pd.concat([general_metadata]*stimulation_paradigms.shape[0], ignore_index=True)
    recordings_overview = pd.concat([stimulations_adjusted_general_metadata, stimulation_paradigms], axis=1)
    recordings_overview['global_cell_id'] = str(global_cell_id).zfill(4)

columns = ['global_cell_id', 'date', 'session_cell_id', 'mouse_line', 'sex', 'brain_region', 'cell_type', 
           'stimulation_string', 'stimulation_frequency-Hz', 'stimulation_duration-ms', 'filepath_detected_events']
recordings_overview[columns]
recordings_overview
stimulation_paradigms
df_cell = pd.DataFrame(recording_info)
df_cell
cell_recording_dir = Path('/mnt/c/Users/dsege/TEMP/Alexia/NET-cre_vGat-flp/2021_12_09_C2/')



metadata_df = pd.read_excel(cell_recording_dir.joinpath(f'{cell_recording_dir.name}.xlsx'),
                            sheet_name = 'General information')

mix_and_matchable_categories = dict()

general_metadata = {'mouse_line': metadata_df['Animal line'][0],
                    'brain_region': metadata_df['Region'][0],
                    'cell_type': metadata_df['Type'][0],
                    'sex': metadata_df['Sex'][0]}

filepaths_stimulation_recordings = list()
for elem in cell_recording_dir.iterdir():
    if 'datapoints' not in elem.name:
        if elem.name.endswith('.csv'):
            filepaths_stimulation_recordings.append(elem)


stimulation_paradigms = dict()
for filepath in filepaths_stimulation_recordings:
    filename = filepath.name
    filename = filename.replace('.csv', '')
    yyyy_mm_dd_ = filename[:11]
    filename = filename.replace(yyyy_mm_dd_, '')
    cell_id = filename[:filename.index('_')]
    filename = filename.replace(cell_id + '_', '')
    if 'Hz' in filename:
        stimulation_frequency = filename[:filename.index('Hz') + 2]
        stimulation_duration = filename[filename.index('Hz') + 2 :]
        stimulation_paradigm = f'{stimulation_frequency}_for_{stimulation_duration}'
    elif 'Bsl' in filename:
        stimulation_paradigm = 'baseline'
    else:
        print(f'Warning: stimulation paradigm could not be identified for: {filepath.name}')
        stimulation_paradigm = 'unknown'
    stimulation_paradigms[stimulation_paradigm] = filepath
filepaths_stimulation_recordings

    
    
stimulation_paradigms
?filename.replace
metadata_df['Animal line'][0]
metadata_df
df = pd.read_excel(cell_recording_dir.as_posix() + '/' + cell_recording_dir.name + '.xlsx', sheet_name = 'General information')
df
df = pd.read_excel(cell_recording_dir.as_posix() + '/' + cell_recording_dir.name + '.xlsx', sheet_name = 'Recordings and cell properties', skiprows=2)
df.drop(['Unnamed: 0'], axis=1, inplace=True)
df.columns
dfs['General information']
dfs['Recordings and cell properties']
for elem in cell_recording_dir.iterdir():
    print(elem)
pd.read_csv(filepath)
path = Path('/mnt/c/Users/dsege/TEMP/Alexia/')
for elem in path.iterdir():
    print(elem)
for elem in path.glob(pattern='*'):
    print(elem)
path.parts
database = Database(path)
import os
from pathlib import Path
from abc import ABC, abstractmethod

class Feature(ABC):
    
    @abstractmethod
    def add_to_database(self):
        pass
    

class CellRecording(Feature):
    
    def add_to_database(self):
        pass
    
    
class Subdirectories:
    
    def __init__(self, root_dir: Path):
        self.root_dir = root_dir
        self.create_missing_subdirectories()
        self.assign_subdirectories_as_attributes()
        
    def create_missing_subdirectories(self):
        # check for each element in a list of subdirs, whether they exist --> create if not
        pass
    
    def assign_subdirectories_as_attributes(self):
        # use list of subdirs and set the path to each as attribute
        pass   


class Database:
    
    """
    The database is supposed to hold all information about all recorded cells that were added to it.
    These information exclude the raw data (only contain filepaths to raw data),
    but include general metadata like celltype, stimulus type, brain region, pharamacological treatment.
    Potentially, some intermediate data could be added (like how the cell reacted upon stimulation, see dashboard).
    Overall, the database should allow the selection of cells based on mix-and-match criteria 
    for further (statistical) analyses and plotting.
    """
    
    def __init__(self, root_dir: Path):
        self.root_dir = root_dir
        self.subdirectories = Subdirectories(root_dir = root_dir)

    def load_database_from_disk(self):
        # Option to load previously created and saved database
        pass
    
    def save_database_to_disk(self):
        # Save all information to disk
        pass
    
    def add_new_feature(self, new_feature: Feature):
        # By providing the Path to a new directory, the user has the option to add new recordings to the database
        # should trigger the database to update itself (e.g. categories dictionary)
        pass<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11001080302484696, 'fraction_numerical': 0.008642419877565718, 'mean_word_length': 3.7992511520737327, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '592055', 'license': 'Apache-2.0', 'n_tokens_mistral': 5413, 'n_tokens_neox': 5163, 'n_words': 1028, 'path': 'patch_clamp_analysis/api_playground.ipynb', 'repo_name': 'DSegebarth/DCLmeetsRETUNE', 'size': '16662'}