<jupyter_start><jupyter_text>Comparisons BNT _ KLUSTA<jupyter_code># the following two lines indicate that external functions are auto-reloaded as soon as they change. 
%load_ext autoreload
%autoreload 2
# Print statements 
from __future__ import print_function # Python 2.x
# General stuff:
import sys
import argparse
import os
import json
import numpy as np
import math
import psycopg2
import cPickle
import numpy as np
import pandas as pd
from datetime import date
from tqdm import tqdm_notebook

# Plotting:
from matplotlib import pyplot as plt
import seaborn as sns
import matplotlib.cm as cm
import matplotlib as mpl
%matplotlib inline

# External functions from subfolder /database_helpers. 
# as soon as you change something in there and press save, it will auto reload on next execution.
from database_helpers.psql_start import *
from database_helpers.create_tables import *
from database_helpers.write2tables import *
from postgres_analysis_helpers.general import *
from postgres_analysis_helpers.get_data import *

# register pickle type to retrieve binary data from database
psycopg2.extensions.register_type(psycopg2.extensions.new_type(psycopg2.BINARY.values, 'BINARY-PICKLE', cast_pickle))
db_status = test_connect()
if db_status == False:
    print('Grrr... no database connection could be established.')
else:
    print('Yippiyeah! Database connection is established!')<jupyter_output>Connecting to the PostgreSQL database...
Yippiyeah! Database connection is established!<jupyter_text>Make an empty dataframe and call the retrieval function<jupyter_code>base_dataframe = pd.DataFrame() # create empty dataframe or feed in a base dataframe from before! 
psql = psql_neuroballs(base_dataframe)
psql.retrieve_data()
data = psql.data()
data.iloc[9]
np.array(data.iloc[9].spikes)[:,0]
data.iloc[9].spiket_tracking_session.time
len(a),len(b)
from itertools import islice

def findOverlap(a, b, rtol = 1e-05, atol = 1e-08):
    ovr_a = []
    ovr_b = []
    start_b = 0
    for i, ai in enumerate(a):
        for j, bj in islice(enumerate(b), start_b, None):
            if np.isclose(ai, bj, rtol=rtol, atol=atol, equal_nan=False):
                ovr_a.append(i)
                ovr_b.append(j)
            elif bj > ai: # (more than tolerance)
                break # all the rest will be farther away
                #print('fail!')
            else: # bj < ai (more than tolerance)
                start_b += 1 # ignore further tests of this item
                #print('fail (2)!')
    return (ovr_a, ovr_b)
ovr_a,ovr_b= findOverlap(a, b, rtol = 1e-05, atol = 1e-08)
diffs = []
for no,i in enumerate(xrange(len(data))):
    try:
        a = np.array(data.iloc[no].spikes)[:,0]
        b = data.iloc[no].spiket_tracking_session.time.values
        ovr_a,ovr_b= findOverlap(a, b, rtol = 1e-6, atol = 1e-10)
        diffs.append(len(ovr_a)-len(a))
        print('{}:   {},{},{}'.format(no,len(ovr_a),len(a),len(b)))
    except AttributeError as err: 
        continue
    except IndexError as err: 
        continue
diffs_old  = diffs
diffs_old<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09585492227979274, 'fraction_numerical': 0.010362694300518135, 'mean_word_length': 3.7669753086419755, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '326928', 'license': 'MIT', 'n_tokens_mistral': 1067, 'n_tokens_neox': 1042, 'n_words': 302, 'path': 'Notebook archive/Compare_spiketimes_BNT_KLUSTA.ipynb', 'repo_name': 'davidcrowland/layer_vb_tagging', 'size': '3088'}