<commit_before>import sys
from collections import namedtuple

import numpy as np
import pandas as pd
from scipy import signal, interpolate


Filter = namedtuple('Filter', ['name', 'parameters'])
Parameter = namedtuple('Parameter', ['description', 'request'])

class TF(object):
    def __init__(self, num, den, name=''):
        self.num  = num
        self.den  = den
        self.name = name

    def discretize(self, samplerate):
        sys = (self.num, self.den)
        (dnum, dden, dt) = signal.cont2discrete(sys, 1 / samplerate)
        return (np.squeeze(dnum), np.squeeze(dden))

pulseheartnum = [0.693489245308734, 132.978069767093, 87009.5691967337, 10914873.0713084, 218273825.541909, 6489400920.14402]
pulseheartden = [1, 180.289425270434, 174563.510125383, 17057258.6774222, 555352944.277185, 6493213494.43661]
tfpulseheart = TF(pulseheartnum, pulseheartden, name='HeartPulse')

TFs = {tfpulseheart.name : tfpulseheart}
interpkind = ['linear', 'nearest', 'zero', 'slinear', 'quadratic', 'cubic']

Filters = {'Lowpass filter' : Filter(name='lowpass', parameters=[Parameter('Cutoff frequency (Hz)', int), Parameter('Filter order', int)]),
           'Savitzky-Golay' : Filter(name='savgol', parameters=[Parameter('Window size (s)', float), Parameter('Polynomial order', int)]),
           'Interpolate' : Filter(name='interp', parameters=[Parameter('New sampling rate (Hz)', int), Parameter('Interpolation type', interpkind)]),
           'Doppler cut' : Filter(name='dopplercut', parameters=[Parameter('Minimum value', int)]),
           'Fill NaNs' : Filter(name='fillnan', parameters=[]),
           'Pressure scale' : Filter(name='pscale', parameters=[Parameter('Systole', int), Parameter('Diastole', int), Parameter('Mean', int)]),
           'Affine scale' : Filter(name='affine', parameters=[Parameter('Scale factor', float), Parameter('Translation factor', float)])}

def updateTFs():
    tflist = list(TFs.keys())
    Filters['Transfer function'] = Filter(name='tf', parameters=[Parameter('Transfer function', tflist)])
updateTFs()

def _savgol(series, samplerate, parameters):
    window, order = parameters
    window = np.floor(window * samplerate)
    if not window % 2:
        # window is even, we need odd
        window += 1
    filtered = signal.savgol_filter(series, int(window), order)
    newname = "{}-{}".format(series.name, 'filtered')
    newseries = pd.Series(filtered, index=series.index, name=newname)
    return (newseries, samplerate)

def _affine(series, samplerate, parameters):
    a, b = parameters
    filtered = a * series + b
    newname = "{}-{}-{}-{}".format(series.name, 'affine', a, b)
    newseries = pd.Series(filtered, index=series.index, name=newname)
    return (newseries, samplerate)

def _tf(series, samplerate, parameters):
    filtname, = parameters
    tf = TFs[filtname]
    b, a = tf.discretize(samplerate)
    filtered = signal.lfilter(b, a, series)
    newname = "{}-{}".format(series.name, tf.name)
    newseries = pd.Series(filtered, index=series.index, name=newname)
    return (newseries, samplerate)

def _lowpass(series, samplerate, parameters):
    Fc, order = parameters
    Wn = Fc * 2 / samplerate
    b, a = signal.butter(order, Wn)
    filtered = signal.lfilter(b, a, series)
    newname = "{}-lp{}".format(series.name, Fc)
    newseries = pd.Series(filtered, index=series.index, name=newname)
    return (newseries, samplerate)

def _interp(series, samplerate, parameters):
    newsamplerate, method = parameters
    npoints = len(series) * newsamplerate / samplerate
    oldidx = series.index
    f = interpolate.interp1d(oldidx, series.values, kind=method)
    newidx = np.linspace(oldidx[1], oldidx[-1], num=npoints)
    resampled = f(newidx)
    newname = "{}-{}Hz".format(series.name, newsamplerate)
    newseries = pd.Series(resampled, index=newidx, name=newname)
    return (newseries, newsamplerate)

def _dopplercut(series, samplerate, parameters):
    minvel, = parameters
    notlow, = (series < minvel).nonzero()
    newname = "{}-{}+".format(series.name, minvel)
    newseries = series.rename(newname)
    newseries.iloc[notlow] = 0
    return (newseries, samplerate)

def _fillnan(series, samplerate, parameters):
    newseries = series.interpolate(method='index', limit_direction='both')
    newname = "{}-nona".format(series.name)
    return (newseries.rename(newname), samplerate)

def _pscale(series, samplerate, parameters):
    sys, dia, mean = parameters
    detrend = series - series.mean()
    scaled = detrend * (sys - dia) / (series.max() - series.min())
    newseries = scaled + mean
    newname = "{}-pscale".format(series.name)
    return (newseries.rename(newname), samplerate)<commit_msg>Fix a stupid indexing error. Found out the hard way.<commit_after>import sys
from collections import namedtuple

import numpy as np
import pandas as pd
from scipy import signal, interpolate


Filter = namedtuple('Filter', ['name', 'parameters'])
Parameter = namedtuple('Parameter', ['description', 'request'])

class TF(object):
    def __init__(self, num, den, name=''):
        self.num  = num
        self.den  = den
        self.name = name

    def discretize(self, samplerate):
        sys = (self.num, self.den)
        (dnum, dden, dt) = signal.cont2discrete(sys, 1 / samplerate)
        return (np.squeeze(dnum), np.squeeze(dden))

pulseheartnum = [0.693489245308734, 132.978069767093, 87009.5691967337, 10914873.0713084, 218273825.541909, 6489400920.14402]
pulseheartden = [1, 180.289425270434, 174563.510125383, 17057258.6774222, 555352944.277185, 6493213494.43661]
tfpulseheart = TF(pulseheartnum, pulseheartden, name='HeartPulse')

TFs = {tfpulseheart.name : tfpulseheart}
interpkind = ['linear', 'nearest', 'zero', 'slinear', 'quadratic', 'cubic']

Filters = {'Lowpass filter' : Filter(name='lowpass', parameters=[Parameter('Cutoff frequency (Hz)', int), Parameter('Filter order', int)]),
           'Savitzky-Golay' : Filter(name='savgol', parameters=[Parameter('Window size (s)', float), Parameter('Polynomial order', int)]),
           'Interpolate' : Filter(name='interp', parameters=[Parameter('New sampling rate (Hz)', int), Parameter('Interpolation type', interpkind)]),
           'Doppler cut' : Filter(name='dopplercut', parameters=[Parameter('Minimum value', int)]),
           'Fill NaNs' : Filter(name='fillnan', parameters=[]),
           'Pressure scale' : Filter(name='pscale', parameters=[Parameter('Systole', int), Parameter('Diastole', int), Parameter('Mean', int)]),
           'Affine scale' : Filter(name='affine', parameters=[Parameter('Scale factor', float), Parameter('Translation factor', float)])}

def updateTFs():
    tflist = list(TFs.keys())
    Filters['Transfer function'] = Filter(name='tf', parameters=[Parameter('Transfer function', tflist)])
updateTFs()

def _savgol(series, samplerate, parameters):
    window, order = parameters
    window = np.floor(window * samplerate)
    if not window % 2:
        # window is even, we need odd
        window += 1
    filtered = signal.savgol_filter(series, int(window), order)
    newname = "{}-{}".format(series.name, 'filtered')
    newseries = pd.Series(filtered, index=series.index, name=newname)
    return (newseries, samplerate)

def _affine(series, samplerate, parameters):
    a, b = parameters
    filtered = a * series + b
    newname = "{}-{}-{}-{}".format(series.name, 'affine', a, b)
    newseries = pd.Series(filtered, index=series.index, name=newname)
    return (newseries, samplerate)

def _tf(series, samplerate, parameters):
    filtname, = parameters
    tf = TFs[filtname]
    b, a = tf.discretize(samplerate)
    filtered = signal.lfilter(b, a, series)
    newname = "{}-{}".format(series.name, tf.name)
    newseries = pd.Series(filtered, index=series.index, name=newname)
    return (newseries, samplerate)

def _lowpass(series, samplerate, parameters):
    Fc, order = parameters
    Wn = Fc * 2 / samplerate
    b, a = signal.butter(order, Wn)
    filtered = signal.lfilter(b, a, series)
    newname = "{}-lp{}".format(series.name, Fc)
    newseries = pd.Series(filtered, index=series.index, name=newname)
    return (newseries, samplerate)

def _interp(series, samplerate, parameters):
    newsamplerate, method = parameters
    npoints = len(series) * newsamplerate / samplerate
    oldidx = series.index
    f = interpolate.interp1d(oldidx, series.values, kind=method)
    newidx = np.linspace(oldidx[0], oldidx[-1], num=npoints)
    resampled = f(newidx)
    newname = "{}-{}Hz".format(series.name, newsamplerate)
    newseries = pd.Series(resampled, index=newidx, name=newname)
    return (newseries, newsamplerate)

def _dopplercut(series, samplerate, parameters):
    minvel, = parameters
    notlow, = (series < minvel).nonzero()
    newname = "{}-{}+".format(series.name, minvel)
    newseries = series.rename(newname)
    newseries.iloc[notlow] = 0
    return (newseries, samplerate)

def _fillnan(series, samplerate, parameters):
    newseries = series.interpolate(method='index', limit_direction='both')
    newname = "{}-nona".format(series.name)
    return (newseries.rename(newname), samplerate)

def _pscale(series, samplerate, parameters):
    sys, dia, mean = parameters
    detrend = series - series.mean()
    scaled = detrend * (sys - dia) / (series.max() - series.min())
    newseries = scaled + mean
    newname = "{}-pscale".format(series.name)
    return (newseries.rename(newname), samplerate)
{'fraction_non_alphanumeric': 0.14938023095666914, 'fraction_numerical': 0.037292086026062085, 'mean_word_length': 4.297418630751964, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4703426', 'n_tokens_mistral': 3362, 'n_tokens_neox': 3089, 'n_words': 716}