<jupyter_start><jupyter_text>Test a class for the easy retrieval of postgres data via Jupyter widgets  Playground<jupyter_code># the following two lines indicate that external functions are auto-reloaded as soon as they change. 
%load_ext autoreload
%autoreload 2
# Print statements 
from __future__ import print_function # Python 2.x
# General stuff:
import sys
import argparse
import os
import json
import numpy as np
import math
import psycopg2
import numpy as np
import pandas as pd
from datetime import date
from tqdm import tqdm_notebook

# Plotting:
from matplotlib import pyplot as plt
import seaborn as sns
import matplotlib.cm as cm
import matplotlib as mpl
%matplotlib inline
%config InlineBackend.figure_format = 'retina'

# External functions from subfolder /database_helpers. 
# as soon as you change something in there and press save, it will auto reload on next execution.
from database_helpers.psql_start import *
from database_helpers.create_tables import *
from database_helpers.write2tables import *
from postgres_analysis_helpers.general import *
from postgres_analysis_helpers.get_data import *

# register pickle type to retrieve binary data from database
psycopg2.extensions.register_type(psycopg2.extensions.new_type(psycopg2.BINARY.values, 'BINARY-PICKLE', cast_pickle))
db_status = test_connect()
if db_status == False:
    print('Grrr... no database connection could be established.')
else:
    print('Yippiyeah! Database connection is established!')<jupyter_output>Connecting to the PostgreSQL database...
Yippiyeah! Database connection is established!<jupyter_text>Make an empty dataframe and call the retrieval function<jupyter_code>base_dataframe = pd.DataFrame() # create empty dataframe or feed in a base dataframe from before! 
psql = psql_neuroballs(base_dataframe)
psql.retrieve_data()
base_dataframe = psql.data()
base_dataframe.animal_id.unique()
base_dataframe
inhib_p_thresh = 0.001
latency_thresh = 3
original_len = len(base_dataframe)
base_dataframe = base_dataframe.dropna(subset=['tetrode_no','cluster_no'])
base_dataframe.sort_values('inhib_lowest_p',ascending=True, inplace=True)
base_dataframe.drop_duplicates(subset=['session_ts','animal_id','tetrode_no','cluster_no'],inplace=True)
base_dataframe.reset_index(drop=True,inplace=True)
base_dataframe['inhibited'] = (base_dataframe.inhib_lowest_p < inhib_p_thresh) & (base_dataframe.inhibited == True)
print('There are now {} cells in the dataframe (original: {}).'.format(len(base_dataframe),original_len))<jupyter_output>There are now 197 cells in the dataframe (original: 207).<jupyter_text>Inhibited cells only<jupyter_code>base_dataframe = base_dataframe[base_dataframe.inhibited == True]
print('There are {} inhibited cells in this dataframe.'.format(len(base_dataframe)))<jupyter_output>There are 83 inhibited cells in this dataframe.<jupyter_text>Now drop "session_name" and retrieve the base sessions...<jupyter_code>base_dataframe.drop('session_name',1,inplace=True)
# Draw quick latency histogram:
plt.figure(figsize=(7,3))
plt.hist(base_dataframe.change_point_ms,200)
plt.xlim(-2,6)
plt.title('Latency histogram')
plt.xlabel('Latency [ms]')
psql = psql_neuroballs(base_dataframe)
psql.retrieve_data()
base_dataframe = psql.data()
#base_dataframe_inhib.dropna(subset=['theta_strength','theta_mean_phase','calbindin','gridscore'],inplace=True)
base_dataframe.dropna(subset=['theta_mean_phase_bnt','theta_strength_bnt','grid_score_bnt'],inplace=True)

calbindin = base_dataframe[base_dataframe.calbindin_bnt == 1]
stellate = base_dataframe[base_dataframe.calbindin_bnt == 0]
rest = base_dataframe[pd.isnull(base_dataframe.calbindin_bnt)]
print('Classifier results: {} Calbindin, {} Stellate, {} Unidentified'.format(len(calbindin),len(stellate),len(rest)))
#Save the 4 datasets: base_dataframe_inhib, calbindin,stellate,rest

#export_path_pickle  = r"C:\work\python\klusta_analysis\postgres_notebooks\export_dataframes\dave_arch"
#export_path_pickle = "/".join(export_path_pickle.split("\\"))
#base_dataframe_inhib.to_pickle(export_path_pickle + "/all_inhib_arch.pkl")
#calbindin.to_pickle(export_path_pickle + "/calbindin_inhib.pkl")
#stellate.to_pickle(export_path_pickle + "/stellate_inhib.pkl")
#rest.to_pickle(export_path_pickle + "/rest_inhib.pkl")
sns.set(font_scale=1.3)
figure = plt.figure(figsize=(15,7))
ax = figure.add_subplot(1,2,1,projection='polar')
ax.scatter([np.mean(x) for x in calbindin.theta_mean_phase_bnt.values],[np.mean(x) for x in calbindin.theta_strength_bnt.values],c='r',s=(calbindin.grid_score_bnt*500).astype(int),alpha=.7,label='Calbindin')
ax.scatter([np.mean(x) for x in stellate.theta_mean_phase_bnt.values],[np.mean(x) for x in stellate.theta_strength_bnt.values],c='r',s=(stellate.grid_score_bnt*500).astype(int),alpha=.7,label='Stellate')
ax.scatter([np.mean(x) for x in rest.theta_mean_phase_bnt.values],[np.mean(x) for x in rest.theta_strength_bnt.values],c='k',s=(rest.grid_score_bnt*500).astype(int),alpha=.7,label='Unclassified')
ax.set_xlabel('KLUSTA')<jupyter_output>C:\Users\horsto\AppData\Local\Continuum\Miniconda2\envs\analysis\lib\site-packages\matplotlib\collections.py:877: RuntimeWarning: invalid value encountered in sqrt
  scale = np.sqrt(self._sizes) * dpi / 72.0 * self._factor<jupyter_text>If you want a "layer2" dataset, drop the cluster_no column and proceed<jupyter_code>base_dataframe.drop('cluster_no',1,inplace=True)
#base_dataframe.drop(['theta_strength','theta_mean_phase','phase_stats_mean','phase_stats_mvl'],1,inplace=True)
psql = psql_neuroballs(base_dataframe)
psql.retrieve_data()
base_dataframe = psql.data()
base_dataframe.dropna(subset=['phase_stats_mvl','phase_stats_mean'],inplace=True)
calbindin = base_dataframe[base_dataframe.calbindin == 1]
stellate = base_dataframe[base_dataframe.calbindin == 0]
rest = base_dataframe[pd.isnull(base_dataframe.calbindin)]
# scatter 
sns.set(font_scale=1.3)
figure = plt.figure(figsize=(15,7))
ax = figure.add_subplot(1,2,1,projection='polar')
ax.scatter(calbindin.phase_stats_mean.values.astype(float),calbindin.phase_stats_mvl.values.astype(float),c='r',s=26,alpha=.7,label='Calbindin')
ax.scatter(stellate.phase_stats_mean.values.astype(float),stellate.phase_stats_mvl.values.astype(float),c='b',s=26,alpha=.7,label='Stellate')
ax.scatter(rest.phase_stats_mean.values.astype(float),rest.phase_stats_mvl.values.astype(float),c='k',s=26,alpha=.7,label='Unclassified')
ax.set_xlabel('KLUSTA')
#ax.set_ylim(0,0.85)

ax = figure.add_subplot(1,2,2,projection='polar')
ax.scatter(calbindin.theta_mean_phase.values.astype(float),calbindin.theta_strength.values.astype(float),c='r',s=26,alpha=.7,label='Calbindin')
ax.scatter(stellate.theta_mean_phase.values.astype(float),stellate.theta_strength.values.astype(float),c='b',s=26,alpha=.7,label='Stellate')
ax.scatter(rest.theta_mean_phase.values.astype(float),rest.theta_strength.values.astype(float),c='k',s=26,alpha=.7,label='Unclassified')
ax.set_xlabel('BNT')

ax.legend(loc=1)
#plt.axis('square')
# scatter 
sns.set(font_scale=1.3)
figure = plt.figure(figsize=(15,7))
ax = figure.add_subplot(1,2,1)
ax.scatter(base_dataframe.phase_stats_mvl,base_dataframe.theta_strength,c='k',s=26,lw=0,alpha=.77)
ax.set_xlabel('KLUSTA')
ax.set_ylabel('BNT')
ax.set_title('Theta tuning strength')
ax = figure.add_subplot(1,2,2)
ax.scatter(base_dataframe.phase_stats_mean,base_dataframe.theta_mean_phase,c='k',s=26,lw=0,alpha=.77)
ax.set_title('Theta tuning angle')
ax.set_xlabel('KLUSTA')
ax.set_ylabel('BNT')
plt.axis('square')
base_dataframe.head()
draw_ratemaps(base_dataframe,'masked_ratemap_bnt',25,0)
draw_ratemaps(base_dataframe,'autocorr_bnt',25,0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13298429319371727, 'fraction_numerical': 0.016230366492146597, 'mean_word_length': 9.174434087882823, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '611343', 'license': 'MIT', 'n_tokens_mistral': 2905, 'n_tokens_neox': 2786, 'n_words': 425, 'path': 'Notebook archive/psql_neuroballs_tests2.ipynb', 'repo_name': 'davidcrowland/layer_vb_tagging', 'size': '7640'}