<jupyter_start><jupyter_text>DIO<jupyter_code>from loren_frank_data_processing.DIO import get_DIO

epoch_key = ('dav', 2, 2)
dio = get_DIO(epoch_key, ANIMALS)
import numpy as np
import pandas as pd

from loren_frank_data_processing.core import get_data_structure
from loren_frank_data_processing.tetrodes import get_trial_time

_PULSE_TIMES_NAMES = ['times', 'pulsetimes']
_PULSE_IND_NAMES = ['values', 'pulse_ind']


def get_DIO(epoch_key, animals):
    ''''''
    animal, day, epoch = epoch_key
    pins = get_data_structure(
        animals[animal], day, 'DIO', 'DIO')[epoch - 1].squeeze()
    pins_df = []

    for pin_ind, pin in enumerate(pins):
        if pin.dtype.names is not None:
            pulse_times_ind = np.nonzero(
                [name in _PULSE_TIMES_NAMES
                 for name in pin.dtype.names])[0].item()
            pulse_times_field_name = pin.dtype.names[pulse_times_ind]
            try:
                times = pin[pulse_times_field_name][0, 0].squeeze()
                time = pd.to_timedelta(times.flat, unit='us')
            except ValueError:
                times = pin[pulse_times_field_name][0, 0].item()
                time = pd.to_timedelta(times, unit='us')

            try:
                values = pin['values'][0, 0].squeeze()
            except ValueError:
                values = np.zeros_like(times)
                values[:, 1] = 1
                values = values.flat
            try:
                pin_id = pin['original_id'][0, 0].item()
            except ValueError:
                pin_id = pin_ind

            try:
                series = pd.Series(values, index=time, name=pin_id)
            except TypeError:
                series = pd.Series(name=pin_id)

            pins_df.append(series.loc[series.index.drop_duplicates(keep=False)])
    return pd.concat(pins_df, axis=1).fillna(0).sort_index()
epoch_key = ('dav', 2, 2)
dio = get_DIO(epoch_key, ANIMALS)
def get_DIO_indicator(epoch_key, animals, time_function=get_trial_time):
    time = time_function(epoch_key, animals)
    dio = get_DIO(epoch_key, animals)
    time_index = np.digitize(dio.index.total_seconds(),
                             time.total_seconds())
    time_index[time_index == time.size] = time.size - 1
    return (dio.groupby(time[time_index]).sum()
            .reindex(index=time, fill_value=0))
from loren_frank_data_processing import get_position_dataframe

get_DIO_indicator(epoch_key, ANIMALS)<jupyter_output><empty_output><jupyter_text>Spike Indicators<jupyter_code>from loren_frank_data_processing import get_all_spike_indicators, make_neuron_dataframe

epoch_key = ('dud', 5, 2)

neuron_info = make_neuron_dataframe(ANIMALS).xs(epoch_key, drop_level=False)
brain_areas = ['CA1', 'CA3']

neuron_info = neuron_info.loc[
    (neuron_info.numspikes > 0) &
    neuron_info.area.isin(brain_areas)]
spikes = get_all_spike_indicators(neuron_info.index, ANIMALS)

from loren_frank_data_processing.neurons import get_spikes_dataframe, get_trial_time

def get_all_spike_indicators(neuron_keys, animals,
                             time_function=get_trial_time):
    time = time_function(neuron_keys[0][:3], animals)
    spikes_dfs = [get_spikes_dataframe(neuron_key, animals)
                  for neuron_key in neuron_keys]
    n_time = time.size
    for ind, spikes_df in enumerate(spikes_dfs):
        spike_time_ind = np.digitize(
            spikes_df.index.total_seconds(), time.total_seconds())
        spike_time_ind = spike_time_ind[spike_time_ind < n_time]
        spikes_dfs[ind] = (spikes_df
                           .iloc[spike_time_ind < n_time]
                           .groupby(time[spike_time_ind]).sum()
                           .reindex(index=time, fill_value=0))

    return pd.concat(spikes_dfs, axis=1)
spikes = get_all_spike_indicators(neuron_info.index, ANIMALS)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10418821096173733, 'fraction_numerical': 0.00749741468459152, 'mean_word_length': 2.8923541247484907, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '551541', 'license': 'MIT', 'n_tokens_mistral': 1379, 'n_tokens_neox': 1341, 'n_words': 221, 'path': 'notebooks/2018_10_24_Debugging_Errors.ipynb', 'repo_name': 'edeno/Replay_ID_2018', 'size': '3868'}