<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Filtering, integrate and equation equivalence1.0.0.1&nbsp;&nbsp;Create the signal1.0.0.2&nbsp;&nbsp;Filter the signal with a differential equation1.0.0.3&nbsp;&nbsp;Build the filter1.0.0.4&nbsp;&nbsp;Plot the filter for different modes1.1&nbsp;&nbsp;Functions1.2&nbsp;&nbsp;Profiling1.3&nbsp;&nbsp;Systematic comparisons of outcomes1.3.0.1&nbsp;&nbsp;Compare Tp, and tau_z1.3.0.2&nbsp;&nbsp;Compare T1 (start time)1.4&nbsp;&nbsp;Compare norms instead of mean distance1.4.0.1&nbsp;&nbsp;Tp and tau_z1.4.0.2&nbsp;&nbsp;T1 (start time)  Filtering, integrate and equation equivalenceThis is to show that the on-line training and off-line learning are the same<jupyter_code>import  pprint
import  subprocess 
import sys 
sys.path.append('../')

import numpy as np
import scipy as sp
import matplotlib.pyplot as plt
import matplotlib
import matplotlib.gridspec as gridspec
from matplotlib.colors import Normalize
from mpl_toolkits.axes_grid1 import make_axes_locatable
import seaborn as sns

%matplotlib inline
plt.rcParams['figure.figsize'] = (12.9, 12)

np.set_printoptions(suppress=True, precision=5)

sns.set(font_scale=3.5)
class MidpointNormalize(matplotlib.colors.Normalize):
    def __init__(self, vmin=None, vmax=None, midpoint=None, clip=False):
        self.midpoint = midpoint
        matplotlib.colors.Normalize.__init__(self, vmin, vmax, clip)

    def __call__(self, value, clip=None):
        # I'm ignoring masked values and all kinds of edge cases to make a
        # simple example...
        x, y = [self.vmin, self.midpoint, self.vmax], [0, 0.5, 1]
        return np.ma.masked_array(np.interp(value, x, y))<jupyter_output><empty_output><jupyter_text>Create the signal<jupyter_code>T = 1.0
dt = 0.001
Tp = 0.100
T1 = 0.400

nt = int(T / dt)
ns = int(T1 / dt)
n_dur = int(Tp / dt)
ne = ns + n_dur

time = np.linspace(0.0, T, num=nt)
s = np.zeros(nt)
s[ns:ne] = 1.0

fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(time, s, label='signa')

ax.axhline(0, ls='--', color='gray')
ax.axvline(0, ls='--', color='gray')

ax.legend();<jupyter_output><empty_output><jupyter_text>Filter the signal with a differential equation<jupyter_code>tau_z = 0.050
z = np.zeros(nt)
z[0] = 0

for index in range(nt - 1):
    z[index + 1] = z[index] + (dt / tau_z) * (s[index] - z[index])
    
fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(time, s, label='signal')
ax.plot(time, z, label='filter')


ax.axhline(0, ls='--', color='gray')
ax.axvline(0, ls='--', color='gray')

ax.legend();<jupyter_output><empty_output><jupyter_text>Build the filter<jupyter_code>exp_filter = np.exp(-time / tau_z)

fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(time, exp_filter, label='exp filter')


ax.axhline(0, ls='--', color='gray')
ax.axvline(0, ls='--', color='gray')

ax.legend();<jupyter_output><empty_output><jupyter_text>Plot the filter for different modes<jupyter_code>normalizing = dt / tau_z
z1 = np.convolve(s, exp_filter, mode='same') * normalizing
z2 = np.convolve(s, exp_filter, mode='full') * normalizing

fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(s, label='signal')
ax.plot(z1, label='same')
ax.plot(z2, label='full')


ax.axhline(0, ls='--', color='gray')
ax.axvline(0, ls='--', color='gray')

ax.legend();
z3 = np.convolve(s, exp_filter, mode='valid') * normalizing
print('same shape', z1.shape)
print('full shape', z2.shape)
print('valid shape', z3.shape)
print('signal shape', s.shape)<jupyter_output>same shape (1000,)
full shape (1999,)
valid shape (1,)
signal shape (1000,)<jupyter_text>* Full: it seems that full takes care of the proper alignment, but produces a tail that we have to get rid off. * Same: Same on the other hand gets the number of points right but is misaligned.* Valid is just one point, because just for one point the signals overlap fully. Note that we used a normalizing factor (dt / tau_z)<jupyter_code>z_filtered = z2[:nt]

fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(time, s, label='Signal')
ax.plot(time, z, label='Differential Equation')
ax.plot(time, z_filtered, label='Convolution')


ax.axhline(0, ls='--', color='gray')
ax.axvline(0, ls='--', color='gray')

ax.legend();<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>def ODE_filter(signal, dt, tau_z):
    nt = signal.shape[0]
    z = np.zeros(nt)
    z[0] = 0

    for index in range(nt - 1):
        z[index + 1] = z[index] + (dt / tau_z) * (signal[index] - z[index])
        
    return z

def Integral_filter(signal, time, dt, tau_z):
    nt = signal.shape[0]
    exp_filter = np.exp(-time / tau_z)
    normalizing = dt / tau_z
    z = np.convolve(s, exp_filter, mode='full') * normalizing
    
    return z[:nt]

def create_signal(T, dt, Tp, T1):
    nt = int(T / dt)
    ns = int(T1 / dt)
    n_dur = int(Tp / dt)
    ne = ns + n_dur

    time = np.linspace(0.0, T, num=nt)
    signal = np.zeros(nt)
    signal[ns:ne] = 1.0
    
    return signal, time
T = 3.0
dt = 0.0001
Tp = 0.500
T1 = 0.400
tau_z = 0.100

s, time = create_signal(T, dt, Tp, T1)
z_ode = ODE_filter(s, dt, tau_z)
z_int = Integral_filter(s, time, dt, tau_z)

fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(time, s, label='Signal')
ax.plot(time, z_ode, label='Differential Equation')
ax.plot(time, z_int, label='Convolution')


ax.axhline(0, ls='--', color='gray')
ax.axvline(0, ls='--', color='gray')

ax.legend();<jupyter_output><empty_output><jupyter_text>Profiling<jupyter_code>def test_ode(tau_z_collection, T=1.0):
    T = T
    dt = 0.0001
    Tp = 0.100
    T1 = 0.400
    s, time = create_signal(T, dt, Tp, T1)
    for tau_z in tau_z_collection:
        z_ode = ODE_filter(s, dt, tau_z)

def test_integral(tau_z_collection, T=1.0):
    T = T
    dt = 0.0001
    Tp = 0.100
    T1 = 0.400
    s, time = create_signal(T, dt, Tp, T1)
    for tau_z in tau_z_collection:
        z_int = Integral_filter(s, time, dt, tau_z)
tau_z_set = np.random.uniform(low=0.005, high=0.150, size=1000)
%time test_ode(tau_z_collection=tau_z_set)
%time test_integral(tau_z_collection=tau_z_set)<jupyter_output>CPU times: user 2min 19s, sys: 64 ms, total: 2min 19s
Wall time: 1min 9s<jupyter_text>The integral is way way slower, incredible so, maybe when the time series is longer the performance of the filter by integral is better.<jupyter_code>T = 10.0
tau_z_set = np.random.uniform(low=0.005, high=0.150, size=100)
%time test_ode(tau_z_collection=tau_z_set)
%time test_integral(tau_z_collection=tau_z_set)<jupyter_output>CPU times: user 15.1 s, sys: 4 ms, total: 15.1 s
Wall time: 7.63 s<jupyter_text>Even longer<jupyter_code>T = 100.0
tau_z_set = np.random.uniform(low=0.005, high=0.150, size=100)
%time test_ode(tau_z_collection=tau_z_set)
%time test_integral(tau_z_collection=tau_z_set)<jupyter_output>CPU times: user 13.4 s, sys: 12 ms, total: 13.4 s
Wall time: 6.73 s<jupyter_text>Systematic comparisons of outcomesWe have to test both T_p, T_total, and tau_z which I supsect are the most important. Maybe dt?<jupyter_code>%%time

num1 = 50
num2 = 50
times_collection = np.linspace(1.0, 10.0, num=num1)
tau_z_collection = np.linspace(0.005, 0.150, num=num2)


dt = 0.001
Tp = 0.500
T1 = 0.400
tau_z = 0.100

distances = np.zeros((num1, num2))

for index1, T in enumerate(times_collection):
    for index2 , tau_z in enumerate(tau_z_collection):
        
        s, time = create_signal(T, dt, Tp, T1)
        z_ode = ODE_filter(s, dt, tau_z)
        z_int = Integral_filter(s, time, dt, tau_z)
        
        distances[index1, index2] = np.mean(z_ode - z_int)
fig = plt.figure()
ax = fig.add_subplot(111)

cmap = matplotlib.cm.RdBu_r
norm = MidpointNormalize(midpoint=0)
extent = [tau_z_collection[0], tau_z_collection[-1], times_collection[0], times_collection[-1]]
im = ax.imshow(distances, norm=norm, origin='lower', cmap=cmap, extent=extent, aspect='auto')
divider = make_axes_locatable(ax)
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im, cax=cax, orientation='vertical')
ax.set_xlabel(r'$\tau_z$')
ax.set_ylabel('Time')
ax.grid();<jupyter_output><empty_output><jupyter_text>Compare Tp, and tau_z<jupyter_code>%%time

num1 = 50
num2 = 50
times_collection = np.linspace(0.100, 1.0, num=num1)
tau_z_collection = np.linspace(0.005, 0.150, num=num2)


T = 3.0
dt = 0.001
Tp = 0.500
T1 = 0.400
tau_z = 0.100

distances = np.zeros((num1, num2))

for index1, Tp in enumerate(times_collection):
    for index2 , tau_z in enumerate(tau_z_collection):
        
        s, time = create_signal(T, dt, Tp, T1)
        z_ode = ODE_filter(s, dt, tau_z)
        z_int = Integral_filter(s, time, dt, tau_z)
        
        distances[index1, index2] = np.mean(z_ode - z_int)
fig = plt.figure()
ax = fig.add_subplot(111)

cmap = matplotlib.cm.RdBu_r
norm = MidpointNormalize(midpoint=0)
extent = [tau_z_collection[0], tau_z_collection[-1], times_collection[0], times_collection[-1]]
im = ax.imshow(distances, norm=norm, origin='lower', cmap=cmap, extent=extent, aspect='auto')
divider = make_axes_locatable(ax)
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im, cax=cax, orientation='vertical')
ax.set_xlabel(r'$\tau_z$')
ax.set_ylabel('Tp')
ax.grid();<jupyter_output><empty_output><jupyter_text>Compare T1 (start time)<jupyter_code>%%time

num1 = 50
num2 = 50
times_collection = np.linspace(0.100, 1.900, num=num1)
tau_z_collection = np.linspace(0.005, 0.150, num=num2)


T = 2.0
dt = 0.001
Tp = 0.100
T1 = 0.400
tau_z = 0.100

distances = np.zeros((num1, num2))

for index1, T1 in enumerate(times_collection):
    for index2 , tau_z in enumerate(tau_z_collection):
        
        s, time = create_signal(T, dt, Tp, T1)
        z_ode = ODE_filter(s, dt, tau_z)
        z_int = Integral_filter(s, time, dt, tau_z)
        
        distances[index1, index2] = np.mean(z_ode - z_int)
fig = plt.figure()
ax = fig.add_subplot(111)

cmap = matplotlib.cm.RdBu_r
norm = MidpointNormalize(midpoint=0)
extent = [tau_z_collection[0], tau_z_collection[-1], times_collection[0], times_collection[-1]]
im = ax.imshow(distances, norm=norm, origin='lower', cmap=cmap, extent=extent, aspect='auto')
divider = make_axes_locatable(ax)
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im, cax=cax, orientation='vertical')
ax.set_xlabel(r'$\tau_z$')
ax.set_ylabel('T1')
ax.grid();<jupyter_output><empty_output><jupyter_text>Compare norms instead of mean distance<jupyter_code>%%time

num1 = 50
num2 = 50
times_collection = np.linspace(1.0, 10.0, num=num1)
tau_z_collection = np.linspace(0.005, 0.150, num=num2)


dt = 0.001
Tp = 0.500
T1 = 0.400
tau_z = 0.100

distances = np.zeros((num1, num2))

for index1, T in enumerate(times_collection):
    for index2 , tau_z in enumerate(tau_z_collection):
        
        s, time = create_signal(T, dt, Tp, T1)
        z_ode = ODE_filter(s, dt, tau_z)
        z_int = Integral_filter(s, time, dt, tau_z)
        
        distances[index1, index2] = np.linalg.norm(z_ode - z_int)
fig = plt.figure()
ax = fig.add_subplot(111)

cmap = matplotlib.cm.RdBu_r
norm = MidpointNormalize(midpoint=0)
extent = [tau_z_collection[0], tau_z_collection[-1], times_collection[0], times_collection[-1]]
im = ax.imshow(distances, norm=norm, origin='lower', cmap=cmap, extent=extent, aspect='auto')
divider = make_axes_locatable(ax)
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im, cax=cax, orientation='vertical')
ax.set_xlabel(r'$\tau_z$')
ax.set_ylabel('Time')
ax.grid();<jupyter_output><empty_output><jupyter_text>Tp and tau_z<jupyter_code>%%time

num1 = 50
num2 = 50
times_collection = np.linspace(0.100, 1.0, num=num1)
tau_z_collection = np.linspace(0.005, 0.150, num=num2)


T = 3.0
dt = 0.001
Tp = 0.500
T1 = 0.400
tau_z = 0.100

distances = np.zeros((num1, num2))

for index1, Tp in enumerate(times_collection):
    for index2 , tau_z in enumerate(tau_z_collection):
        
        s, time = create_signal(T, dt, Tp, T1)
        z_ode = ODE_filter(s, dt, tau_z)
        z_int = Integral_filter(s, time, dt, tau_z)
        
        distances[index1, index2] = np.linalg.norm(z_ode - z_int)
fig = plt.figure()
ax = fig.add_subplot(111)

cmap = matplotlib.cm.RdBu_r
norm = MidpointNormalize(midpoint=0)
extent = [tau_z_collection[0], tau_z_collection[-1], times_collection[0], times_collection[-1]]
im = ax.imshow(distances, norm=norm, origin='lower', cmap=cmap, extent=extent, aspect='auto')
divider = make_axes_locatable(ax)
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im, cax=cax, orientation='vertical')
ax.set_xlabel(r'$\tau_z$')
ax.set_ylabel('Tp')
ax.grid();<jupyter_output><empty_output><jupyter_text>T1 (start time)<jupyter_code>%%time

num1 = 50
num2 = 50
times_collection = np.linspace(0.100, 1.900, num=num1)
tau_z_collection = np.linspace(0.005, 0.150, num=num2)


T = 2.0
dt = 0.001
Tp = 0.100
T1 = 0.400
tau_z = 0.100

distances = np.zeros((num1, num2))

for index1, T1 in enumerate(times_collection):
    for index2 , tau_z in enumerate(tau_z_collection):
        
        s, time = create_signal(T, dt, Tp, T1)
        z_ode = ODE_filter(s, dt, tau_z)
        z_int = Integral_filter(s, time, dt, tau_z)
        
        distances[index1, index2] = np.linalg.norm(z_ode - z_int)
fig = plt.figure()
ax = fig.add_subplot(111)

cmap = matplotlib.cm.RdBu_r
norm = MidpointNormalize(midpoint=0)
extent = [tau_z_collection[0], tau_z_collection[-1], times_collection[0], times_collection[-1]]
im = ax.imshow(distances, norm=norm, origin='lower', cmap=cmap, extent=extent, aspect='auto')
divider = make_axes_locatable(ax)
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im, cax=cax, orientation='vertical')
ax.set_xlabel(r'$\tau_z$')
ax.set_ylabel('T1')
ax.grid();<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15136711629602626, 'fraction_numerical': 0.04659132336857456, 'mean_word_length': 4.577877185847906, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '132562', 'license': 'MIT', 'n_tokens_mistral': 5969, 'n_tokens_neox': 5541, 'n_words': 1199, 'path': 'jupyter/2018-07-19(Filtering, integrate and equation equivalence).ipynb', 'repo_name': 'h-mayorquin/sequence_learning_paper', 'size': '13715'}