<jupyter_start><jupyter_text>Compare the last sessions of 70645, 70644 and 71343 with the rest of the deep layers / layer II (arch) recordings<jupyter_code># ts fresh stuff<jupyter_output><empty_output><jupyter_text>Load dataset for layer2 (superficial inhibited Dave's arch mice) from hard disk.<jupyter_code># the following two lines indicate that external functions are auto-reloaded as soon as they change. 
%load_ext autoreload
%autoreload 2
# Print statements 
from __future__ import print_function # Python 2.x
# General stuff:
import sys
import argparse
import os
import json
import numpy as np
import math
import psycopg2
import cPickle
import numpy as np
import pandas as pd
from datetime import date
from tqdm import tqdm_notebook

# Plotting:
from matplotlib import pyplot as plt
import seaborn as sns
import matplotlib.cm as cm
import matplotlib as mpl
%matplotlib inline

# External functions from subfolder /database_helpers. 
# as soon as you change something in there and press save, it will auto reload on next execution.
from database_helpers.psql_start import *
from database_helpers.create_tables import *
from database_helpers.write2tables import *
from postgres_analysis_helpers.general import *
from postgres_analysis_helpers.get_data import *
from tsfresh import extract_features

from tsfresh import select_features
from tsfresh.utilities.dataframe_functions import impute


# register pickle type to retrieve binary data from database
psycopg2.extensions.register_type(psycopg2.extensions.new_type(psycopg2.BINARY.values, 'BINARY-PICKLE', cast_pickle))
db_status = test_connect()
if db_status == False:
    print('Grrr... no database connection could be established.')
else:
    print('Yippiyeah! Database connection is established!')<jupyter_output>Connecting to the PostgreSQL database...
Yippiyeah! Database connection is established!<jupyter_text>Make an empty dataframe and call the retrieval function<jupyter_code># if you want to save
export_path_pickle  = r"C:\work\python\klusta_analysis\postgres_notebooks\export_dataframes"
export_path_pickle = "/".join(export_path_pickle.split("\\"))
#base_dataframe.to_pickle(export_path_pickle + "/70645_70644_71343.pkl")
deep_layer_confirmed = pd.read_pickle(export_path_pickle + "/70645_70644_71343.pkl")
deep_layer_confirmed.head(2)
deep_layer_confirmed.columns<jupyter_output><empty_output><jupyter_text>"Deep layer confirmed" contains dataset comprising 70645,70644,71343<jupyter_code>deep_layer_confirmed.tail(2)
deep_layer_confirmed.dropna(subset=['tetrode_no','cluster_no'],inplace=True)
deep_layer_confirmed.drop_duplicates(subset=['session_ts','animal_id','tetrode_no','cluster_no'],inplace=True)
deep_layer_confirmed.sort_values('session_ts',ascending=True, inplace=True)
deep_layer_confirmed.reset_index(drop=True,inplace=True)<jupyter_output><empty_output><jupyter_text>ts fresh<jupyter_code># load superficial layer dataset
lfp_cells_sup = pd.read_pickle(export_path_pickle + "/lfp_cells_superficial.pkl")
lfp_cells_sup.animal_id.unique()<jupyter_output><empty_output><jupyter_text>take only the last 5 sessions for 70645, 70644 and 71343<jupyter_code>_70645_last = deep_layer_confirmed[deep_layer_confirmed.animal_id == '70645']
_70645_last.drop_duplicates(subset=['session_ts'],inplace=True) 
_70645_last.reset_index(drop=True,inplace=True)
_70645_last = _70645_last[[x in _70645_last.session_ts.unique()[-5:] for x in _70645_last.session_ts]]
    
_70644_last = deep_layer_confirmed[deep_layer_confirmed.animal_id == '70644']
_70644_last.drop_duplicates(subset=['session_ts'],inplace=True) 
_70644_last.reset_index(drop=True,inplace=True)
_70644_last = _70644_last[[x in _70644_last.session_ts.unique()[-5:] for x in _70644_last.session_ts]]

_71343_last = deep_layer_confirmed[deep_layer_confirmed.animal_id == '71343']
_71343_last.drop_duplicates(subset=['session_ts'],inplace=True) 
_71343_last.reset_index(drop=True,inplace=True)
_71343_last = _71343_last[[x in _71343_last.session_ts.unique()[-5:] for x in _71343_last.session_ts]]
lfp_cells_sup.animal_id.unique()<jupyter_output><empty_output><jupyter_text>edit dataframe structure ...<jupyter_code>mouse_labels = dict()
mouse_labels['70645'] = 0
mouse_labels['70644'] = 1
mouse_labels['71343'] = 2

mouse_labels['70114'] = 3
mouse_labels['T4743'] = 4
mouse_labels['70110'] = 5
mouse_labels['T4935'] = 6
mouse_labels['70113'] = 7
mouse_labels
max_snippet = 5000 # this corresponds to 4 seconds sampled at 250 Hz
time_axis = np.arange(0,max_snippet)/250.

#labels = []

counter = 0 
for no,df in tqdm_notebook(enumerate([_70645_last,_70644_last,_71343_last,lfp_cells_sup])):
    for i in tqdm_notebook(xrange(len(df))):
        
        #if df.iloc[i].animal_id in ['70645','70644','71343']:
        #    label_col = np.zeros(max_snippet) # '0' for deep layers 
        #else:
        #    label_col = np.zeros(max_snippet) + 1 # '1' for sup layers

        #labels.append(label_col)

        #id_column = np.array(((df.iloc[i].animal_id + " ")* (max_snippet-1)).split(" "))
        #print(df.iloc[i].animal_id)
        id_column = np.array(np.zeros(max_snippet) + mouse_labels[df.iloc[i].animal_id],dtype=int)
        
        
        moving = df.lfp_session.iloc[i].eeg_mean.values[df.lfp_session.iloc[i].speed > 5][:max_snippet]
        still = df.lfp_session.iloc[i].eeg_mean.values[df.lfp_session.iloc[i].speed < 5][:max_snippet]

        # pad values if there are not enough:
        moving = np.lib.pad(moving, (0, max_snippet - len(moving)), 'wrap')
        still = np.lib.pad(still, (0, max_snippet - len(still)), 'wrap')

        
        #lfp_temp = pd.DataFrame({'time': time_axis, 'moving': moving,'still':still,'id':id_column})
        lfp_temp = pd.DataFrame({'time': time_axis, 'moving': moving,'id':id_column})

        if counter == 0:
            lfp_for_tsfresh = pd.DataFrame(lfp_temp)      
        else:
            lfp_for_tsfresh = pd.concat([lfp_for_tsfresh,lfp_temp])

        counter += 1
#labels =[item for sublist in labels for item in sublist]
lfp_for_tsfresh.id.unique()
lfp_for_tsfresh.dropna(inplace=True,how='any')
lfp_for_tsfresh = lfp_for_tsfresh[pd.isnull(lfp_for_tsfresh.id) == False]
lfp_for_tsfresh = lfp_for_tsfresh[np.isfinite(lfp_for_tsfresh.moving) == True]
lfp_for_tsfresh.reset_index(drop=True,inplace=True)
len(lfp_for_tsfresh)
#filter out bad labels:
#labels = np.array(labels,dtype=np.int64)[[len(x) > 0 for x in lfp_for_tsfresh.id.values]]
#lfp_for_tsfresh = lfp_for_tsfresh[[len(x) > 0 for x in lfp_for_tsfresh.id.values]]
len(lfp_for_tsfresh),len(labels)
lfp_for_tsfresh.id.unique()
lfp_for_tsfresh.head(4)<jupyter_output><empty_output><jupyter_text>start by extracting features<jupyter_code>lfp_for_tsfresh.id.unique()
labels = [0,0,0,1,1,1,1,1]
len(labels)
labels = pd.Series(labels)
labels
print('starting ts fresh....')
extracted_features = extract_features(lfp_for_tsfresh, column_id="id", column_sort="time")
impute(extracted_features)
labels
features_filtered = select_features(extracted_features, labels)

export_path_pickle  = r"C:\work\python\klusta_analysis\postgres_notebooks\export_dataframes"
export_path_pickle = "/".join(export_path_pickle.split("\\"))
features_filtered.to_pickle(export_path_pickle + "/features_filtered.pkl")
print('finished!')
features_filtered<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11407468198604842, 'fraction_numerical': 0.043906442347148134, 'mean_word_length': 6.31931931931932, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '22595', 'license': 'MIT', 'n_tokens_mistral': 2888, 'n_tokens_neox': 2607, 'n_words': 486, 'path': 'Notebook archive/70645_70644_71343_last_sessions-LFP_tsfresh.ipynb', 'repo_name': 'davidcrowland/layer_vb_tagging', 'size': '7311'}