<jupyter_start><jupyter_text>Table of Contents<jupyter_code>import pandas as pd
import numpy as np
import scipy as scipy
import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns
import numba
import maxentropy
from matplotlib import rc

# plotting settings
rc('text', usetex=True)
rc('text.latex', preamble=r'\usepackage{cmbright}')
rc('font', **{'family': 'sans-serif', 'sans-serif': ['Helvetica']})

%matplotlib inline

# This enables SVG graphics inline. 
%config InlineBackend.figure_formats = {'png', 'retina'}

# JB's favorite Seaborn settings for notebooks
rc = {'lines.linewidth': 2, 
      'axes.labelsize': 18, 
      'axes.titlesize': 18, 
      'axes.facecolor': 'DFDFE5'}
sns.set_context('notebook', rc=rc)
sns.set_style("dark")

# more parameters
mpl.rcParams['xtick.labelsize'] = 16 
mpl.rcParams['ytick.labelsize'] = 16 
mpl.rcParams['legend.fontsize'] = 14


from scipy.signal import savgol_filter
from sklearn.decomposition import PCA
import os

import scipy.io as sio
# load kato data
fps = 30.3
frames = []
no_animals = 0
for root, dirs, files in os.walk('../input/kato_data/stim', topdown=False):
    for i, file in enumerate(files):
        mat = sio.loadmat('../input/kato_data/stim/' + file)

        npmat = np.array(mat['neuronTraces'].tolist())
        smoothnpmat = savgol_filter(npmat, int(np.floor(fps) + 1), 2)

        columns = [a[0] for a in mat['neuronNames'][0]]
        columns2 = [a[0] + '_smooth' for a in mat['neuronNames'][0]]

        temp = pd.DataFrame(npmat)
        temp2 = pd.DataFrame(smoothnpmat)
        temp.columns = columns
        temp2.columns = columns2
        temp['frame'] = temp.index
        temp = pd.concat([temp, temp2], axis=1)
        temp = temp.melt(id_vars='frame', var_name='Neuron', value_name='Fluorescence')
        temp['animal'] = i
        frames += [temp]
        no_animals += 1
df = pd.concat(frames)
# figure out subset of neurons with replicates:
neurons_with_replicates = []
for neuron, group in df.groupby('Neuron'):
    animals = group.animal.unique()
    if len(animals) == no_animals:
        neurons_with_replicates += [neuron]
print(neurons_with_replicates)
def get_data_from_worm(i, normed=True):

    a = (df.Neuron.isin(neurons_with_replicates))
    tmp = df[a].copy()
    b = (tmp.Neuron.str.contains('_smooth'))
    
    single_animal = tmp[(tmp.animal == i) & b][['frame', 'Neuron', 'Fluorescence']]
    single_animal.sort_values(['Neuron', 'frame'], inplace=True)
    
    print(single_animal.Neuron.unique())
    single_animal = single_animal.pivot(index='frame', columns='Neuron')
    
    data = single_animal.as_matrix()
    columns = single_animal.columns.droplevel().values.T
    columns = np.array([c.replace('_smooth','') for c in columns]).T
    if normed:
        data = (data - data.mean(axis=0))/data.std(axis=0)
    return data, columns
data0, columns = get_data_from_worm(0)
sns.heatmap(data0.T, cmap='PiYG')
plt.yticks(rotation=0)
covar = data0.T.dot(data0)
covar = pd.DataFrame(covar, columns = columns, index=columns)
cg = sns.clustermap(covar.as_matrix(), cmap="PiYG")
data1, columns1 = get_data_from_worm(1)
data2, columns2 = get_data_from_worm(2)
subset = data0
pca = PCA(3)
pca.fit(subset)
print(pca.explained_variance_ratio_)
W = pca.components_
tD0 = W.dot(data0.T)
tD1 = W.dot(data1.T)
tD2 = W.dot(data2.T)
subsample = 10
color = np.linspace(0, 1, len(tD0[0, ::subsample]))
color1 = np.linspace(0, 1, len(tD1[0, ::subsample]))
color2 = np.linspace(0, 1, len(tD2[0, ::subsample]))

plt.scatter(tD0[0, ::subsample], tD0[1, ::subsample], s=20, alpha=0.3, c=color)
plt.scatter(tD1[0, ::subsample], tD1[1, ::subsample], s=20, alpha=0.3, c=color1)
plt.scatter(tD2[0, ::subsample], tD2[1, ::subsample], s=20, alpha=0.3, c=color2)
plt.plot(tD0[0, :], alpha=0.3)
plt.plot(tD0[1, :], alpha=0.3)
plt.plot(tD0[2, :], alpha=0.3)
plt.plot(tD0[0, :], alpha=0.3)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16292134831460675, 'fraction_numerical': 0.027834525025536263, 'mean_word_length': 4.329251700680272, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '624369', 'license': 'MIT', 'n_tokens_mistral': 1607, 'n_tokens_neox': 1505, 'n_words': 308, 'path': 'src/Untitled1.ipynb', 'repo_name': 'dangeles/WormsDream', 'size': '3916'}