<jupyter_start><jupyter_text>Data Quality Checks<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline

import importlib
# import OpenEphys tools for Python3
oep = importlib.import_module("analysis-tools.Python3.OpenEphys")<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>datadir = 'data/'
signal_chans = [1, 2, 3, 4, 12, 13, 14, 15]
REF_CHAN = 16
channel = signal_chans[0]
fname = datadir + '100_CH{}.continuous'.format(channel)
reffname = datadir + '100_CH{}.continuous'.format(REF_CHAN)
dat_dict = oep.load(fname)
dat_dict
dat = dat_dict['data']
dat.shape, dat
offset = 2048
length = 512
plt.figure(figsize=(16, 4))
plt.plot(dat[offset:(offset+length)])
reffname = datadir + '100_CH{}.continuous'.format(REF_CHAN)
ref_dict = oep.load(reffname)
ref_dat = ref_dict['data']
offset = 2048
length = 512
plt.figure(figsize=(16, 4))
plt.plot(ref_dat[offset:(offset+length)])
offset = 2048
length = 512
plt.figure(figsize=(16, 4))
plt.plot((dat - ref_dat)[offset:(offset+length)])<jupyter_output><empty_output><jupyter_text>Reference signal by subtracting reference wire input<jupyter_code>dat = dat - ref_dat
import scipy.signal as ssig
fs = int(dat_dict['header']['sampleRate'])
def bandlimit(df, band=(0.01, 600)):
    """
    Computes bandpass-filtered version of time series in df.
    Band is either a two-element indexed sequence or a conventionally
    defined electrophysiological frequency band.
    """
    dt = 1/fs

    b, a = ssig.iirfilter(2, [2 * dt * f for f in band], rp=0.1, rs=40,
        ftype='ellip')
    return ssig.filtfilt(b, a, df)
def decimate(x, decfrac, axis=-1):
    """
    just like in the scipy source code, except I use filtfilt
    and opt for hardcode defaults
    q is the fraction to decimate by; length of returned data is len(x)/q
    """
    if decfrac > 15:
        wrnstr = """You are attempting to decimate by a factor > 15. You are risking numerical instability. Consider performing multiple successive decimations instead."""
        warnings.warn(wrnstr)

    n = 8
    b, a = ssig.filter_design.cheby1(n, 0.05, 0.8 / decfrac)

    y = ssig.filtfilt(b, a, x, axis=axis)

    sl = [slice(None)] * y.ndim
    sl[axis] = slice(None, None, decfrac)
    return y[sl]
dat = bandlimit(dat)
dat = decimate(decimate(dat, 10), 5)[1:]
dat -= np.mean(dat)
Ns = len(dat)
fs /= 50 # because we downsampled by 50
dt = 1/fs
T = Ns * dt
fmin = 1/(2 * T)
fmax = fs/2
print(fmin)
T
offset = 0
length = 512
plt.figure(figsize=(16, 4))
plt.plot(dat[offset:(offset+length)])
psd = ssig.welch(dat, fs=fs, nperseg=256)<jupyter_output><empty_output><jupyter_text>Plot power spectral density<jupyter_code>plt.figure(figsize=(16, 4))
plt.plot(psd[0], psd[1])<jupyter_output><empty_output><jupyter_text>Plot PSD in dB:<jupyter_code>plt.figure(figsize=(16, 4))
plt.plot(psd[0], 10*np.log10(psd[1]))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13956686146442077, 'fraction_numerical': 0.04022000687521485, 'mean_word_length': 4.808383233532934, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '38687', 'license': 'MIT', 'n_tokens_mistral': 1209, 'n_tokens_neox': 1091, 'n_words': 273, 'path': 'data_quality_check.ipynb', 'repo_name': 'pearsonlab/ephys_data_validation', 'size': '2909'}