<commit_before>import unittest, csv, os
import numpy as N
from scipy.sandbox.models.utils import recipr0
from scipy.sandbox.models import contrast

from neuroimaging.modalities.fmri import hrf, protocol

# this is not a test until it is fixed (find HRF)
class ProtocolTest(unittest.TestCase):

    def setUp(self):
        """
        Setup an iterator corresponding to the following .csv file:

        hot,9.0,18.0
        warm,27.0,36.0
        hot,45.0,54.0
        warm,63.0,72.0
        hot,81.0,90.0
        warm,99.0,108.0
        hot,117.0,126.0
        warm,135.0,144.0
        hot,153.0,162.0
        warm,171.0,180.0
        hot,189.0,198.0
        warm,207.0,216.0
        hot,225.0,234.0
        warm,243.0,252.0
        hot,261.0,270.0
        warm,279.0,288.0
        hot,297.0,306.0
        warm,315.0,324.0
        hot,333.0,342.0
        warm,351.0,360.0

        """

        on = [9.0, 27.0, 45.0, 63.0, 81.0, 99.0, 117.0,
              135.0, 153.0, 171.0, 189.0, 207.0, 225.0,
              243.0, 261.0, 279.0, 297.0, 315.0, 333.0, 351.0]
        off = [18.0, 36.0, 54.0, 72.0, 90.0, 108.0, 126.0,
               144.0, 162.0, 180.0, 198.0, 216.0, 234.0,
               252.0, 270.0, 288.0, 306.0, 324.0, 342.0, 360.0]
        p = ['hot', 'warm'] * 10
        all = []
        for i in range(10):
            all.append([p[i], on[i], off[i]])
        self.all = all

    def setup_terms(self):
        self.p = protocol.ExperimentalFactor('pain', self.all, delta=False)
        self.p.convolved = False

        self.IRF1 = hrf.glover
        self.IRF2 = hrf.glover_deriv

        self.t = N.arange(0,300,1)

    def testFromFile(self):
        out = file('tmp.csv', 'w')
        writer = csv.writer(out)
        for row in self.all:
            writer.writerow(row)
        out.close()
        
        p = protocol.ExperimentalFactor('pain', file('tmp.csv'), delta=False)
        os.remove('tmp.csv')

    def testFromFileName(self):
        out = file('tmp.csv', 'w')
        writer = csv.writer(out)
        for row in self.all:
            writer.writerow(row)
        out.close()
        
        p = protocol.ExperimentalFactor('pain', 'tmp.csv', delta=False)
        os.remove('tmp.csv')

    def testShape(self):
        self.setup_terms()
        
        for df in range(4, 10):
            drift_fn = protocol.SplineConfound(window=[0,300], df=df)
            drift = protocol.ExperimentalQuantitative('drift', drift_fn)

            formula = self.p + drift
            y = formula(self.t)
            self.assertEquals(y.shape, (2 + df, self.t.shape[0]))

            self.p.convolve(self.IRF1)
            formula = self.p + drift
            y = formula(self.t)
            self.assertEquals(y.shape, (2 + df, self.t.shape[0]))

            self.p.convolve(self.IRF2)
            formula = self.p + drift
            y = formula(self.t)
            self.assertEquals(y.shape, (4 + df, self.t.shape[0]))

            self.p.convolve(self.IRF1)
            formula = self.p + drift * self.p
            y = formula(self.t)
            self.assertEquals(y.shape, (2 + df * 2, self.t.shape[0]))

    def testContrast1(self):
        self.setup_terms()
        drift_fn = protocol.SplineConfound(window=[0,300], df=4)
        drift = protocol.ExperimentalQuantitative('drift', drift_fn)
        formula = self.p + drift
        c = contrast.Contrast(self.p, formula)
        c.getmatrix(time=N.arange(0,300,3.))
        N.testing.assert_almost_equal(c.matrix,
                                          [[0.,0.,0.,0.,1.,0.],
                                           [0.,0.,0.,0.,0.,1.]])

    def testContrast2(self):
        self.setup_terms()
        drift_fn = protocol.SplineConfound(window=[0,300], df=4)
        drift = protocol.ExperimentalQuantitative('drift', drift_fn)
        formula = self.p + drift
        c = contrast.Contrast(self.p.main_effect(), formula)
        c.getmatrix(time=self.t)
        N.testing.assert_almost_equal(c.matrix,
                                          [0.,0.,0.,0.,-1.,1.])

    def testDesign1(self):
        self.setup_terms()
        
        for df in range(4, 10):
            drift_fn = protocol.SplineConfound(window=[0,300], df=df)
            drift = protocol.ExperimentalQuantitative('drift', drift_fn)

            formula = self.p + drift
            y = formula.design(time=self.t)
            self.assertEquals(y.shape[::-1], (2 + df, self.t.shape[0]))

            self.p.convolve(self.IRF1)
            self.p.convolved = True
            formula = self.p + drift
            y = formula.design(time=self.t)
            self.assertEquals(y.shape[::-1], (2 + df, self.t.shape[0]))

            self.p.convolve(self.IRF2)
            self.p.convolved = True
            formula = self.p + drift
            y = formula.design(time=self.t)
            self.assertEquals(y.shape[::-1], (4 + df, self.t.shape[0]))

            self.p.convolve(self.IRF1)
            formula = self.p + drift * self.p
            y = formula.design(time=self.t)
            self.assertEquals(y.shape[::-1], (2 + df * 2, self.t.shape[0]))

    def testDesign2(self):
        self.setup_terms()
        
        for df in range(4, 10):
            drift_fn = protocol.SplineConfound(window=[0,300], df=df)
            drift = protocol.ExperimentalQuantitative('drift', drift_fn)

            formula = self.p + drift
            y = formula.names()
            self.assertEquals(len(y), 2 + df)

            formula = self.p + drift
            y = formula.names()
            self.assertEquals(len(y), 2 + df)

            self.p.convolve(self.IRF2)
            self.convolved = True
            formula = self.p + drift
            y = formula.names()
            self.assertEquals(len(y), 4 + df)

            self.p.convolve(self.IRF1)
            formula = self.p + drift * self.p
            y = formula.names()
            self.assertEquals(len(y), 2 + df * 2)

    def testTimeFn1(self):
        self.setup_terms()
        
        drift_fn = protocol.SplineConfound(window=[0,300], df=7)
        drift = protocol.ExperimentalQuantitative('drift', drift_fn)
        d = drift.astimefn()
        t = N.arange(0,30,1.)

        Z = float(N.random.standard_normal(()))

        D = d * Z
        N.testing.assert_almost_equal(D(t), N.array(drift_fn(t)) * Z)

        D = d / Z
        N.testing.assert_almost_equal(D(t), N.array(drift_fn(t)) / Z)

        D = d - Z
        N.testing.assert_almost_equal(D(t), N.array(drift_fn(t)) - Z)

        D = d + Z
        N.testing.assert_almost_equal(D(t), N.array(drift_fn(t)) + Z)


    def testTimeFn2(self):
        self.setup_terms()
        
        drift_fn = protocol.SplineConfound(window=[0,300], df=7)
        drift = protocol.ExperimentalQuantitative('drift', drift_fn)
        d = drift.astimefn()
        t = N.arange(0,30,1.)

        D = d * d
        N.testing.assert_almost_equal(D(t), N.array(drift_fn(t))**2)

        D = d / d
        N.testing.assert_almost_equal(D(t), N.array(drift_fn(t)) * recipr0(drift_fn(t)))

        D = d - d
        N.testing.assert_almost_equal(D(t), N.zeros(D(t).shape, N.float64))

        d = d + d
        N.testing.assert_almost_equal(D(t), 2 * N.array(drift_fn(t)))

    def testTimeFn3(self):
        self.setup_terms()
        
        drift_fn = protocol.SplineConfound(window=[0,300], df=7)
        drift = protocol.ExperimentalQuantitative('drift', drift_fn)
        d = drift.astimefn()
        t = N.arange(0,30,1.)
<commit_msg>fix broken protocol test<commit_after>import unittest, csv, os
import numpy as N
from scipy.sandbox.models.utils import recipr0
from scipy.sandbox.models import contrast

from neuroimaging.modalities.fmri import hrf, protocol

# this is not a test until it is fixed (find HRF)
class ProtocolTest(unittest.TestCase):

    def setUp(self):
        """
        Setup an iterator corresponding to the following .csv file:

        hot,9.0,18.0
        warm,27.0,36.0
        hot,45.0,54.0
        warm,63.0,72.0
        hot,81.0,90.0
        warm,99.0,108.0
        hot,117.0,126.0
        warm,135.0,144.0
        hot,153.0,162.0
        warm,171.0,180.0
        hot,189.0,198.0
        warm,207.0,216.0
        hot,225.0,234.0
        warm,243.0,252.0
        hot,261.0,270.0
        warm,279.0,288.0
        hot,297.0,306.0
        warm,315.0,324.0
        hot,333.0,342.0
        warm,351.0,360.0

        """

        on = [9.0, 27.0, 45.0, 63.0, 81.0, 99.0, 117.0,
              135.0, 153.0, 171.0, 189.0, 207.0, 225.0,
              243.0, 261.0, 279.0, 297.0, 315.0, 333.0, 351.0]
        off = [18.0, 36.0, 54.0, 72.0, 90.0, 108.0, 126.0,
               144.0, 162.0, 180.0, 198.0, 216.0, 234.0,
               252.0, 270.0, 288.0, 306.0, 324.0, 342.0, 360.0]
        p = ['hot', 'warm'] * 10
        all = []
        for i in range(10):
            all.append([p[i], on[i], off[i]])
        self.all = all

    def setup_terms(self):
        self.p = protocol.ExperimentalFactor('pain', self.all, delta=False)
        self.p.convolved = False

        self.IRF1 = hrf.glover
        self.IRF2 = hrf.glover_deriv

        self.t = N.arange(0,300,1)

    def testFromFile(self):
        out = file('tmp.csv', 'w')
        writer = csv.writer(out)
        for row in self.all:
            writer.writerow(row)
        out.close()
        
        p = protocol.ExperimentalFactor('pain', file('tmp.csv'), delta=False)
        os.remove('tmp.csv')

    def testFromFileName(self):
        out = file('tmp.csv', 'w')
        writer = csv.writer(out)
        for row in self.all:
            writer.writerow(row)
        out.close()
        
        p = protocol.ExperimentalFactor('pain', 'tmp.csv', delta=False)
        os.remove('tmp.csv')

    def testShape(self):
        self.setup_terms()
        
        for df in range(4, 10):
            drift_fn = protocol.SplineConfound(window=[0,300], df=df)
            drift = protocol.ExperimentalQuantitative('drift', drift_fn)

            formula = self.p + drift
            y = formula(self.t)
            self.assertEquals(y.shape, (2 + df, self.t.shape[0]))

            self.p.convolve(self.IRF1)
            formula = self.p + drift
            y = formula(self.t)
            self.assertEquals(y.shape, (2 + df, self.t.shape[0]))

            self.p.convolve(self.IRF2)
            formula = self.p + drift
            y = formula(self.t)
            self.assertEquals(y.shape, (4 + df, self.t.shape[0]))

            self.p.convolve(self.IRF1)
            formula = self.p + drift * self.p
            y = formula(self.t)
            self.assertEquals(y.shape, (2 + df * 2, self.t.shape[0]))

    def testContrast1(self):
        self.setup_terms()
        drift_fn = protocol.SplineConfound(window=[0,300], df=4)
        drift = protocol.ExperimentalQuantitative('drift', drift_fn)
        formula = self.p + drift
        c = contrast.Contrast(self.p, formula)
        c.getmatrix(time=N.arange(0,300,3.))
        N.testing.assert_almost_equal(c.matrix,
                                          [[0.,0.,0.,0.,1.,0.],
                                           [0.,0.,0.,0.,0.,1.]])

    def testContrast2(self):
        self.setup_terms()
        drift_fn = protocol.SplineConfound(window=[0,300], df=4)
        drift = protocol.ExperimentalQuantitative('drift', drift_fn)
        formula = self.p + drift
        c = contrast.Contrast(self.p.main_effect(), formula)
        c.getmatrix(time=self.t)
        N.testing.assert_almost_equal(c.matrix,
                                          [0.,0.,0.,0.,-1.,1.])

    def testDesign1(self):
        self.setup_terms()
        
        for df in range(4, 10):
            drift_fn = protocol.SplineConfound(window=[0,300], df=df)
            drift = protocol.ExperimentalQuantitative('drift', drift_fn)

            formula = self.p + drift
            y = formula.design(time=self.t)
            self.assertEquals(y.shape[::-1], (2 + df, self.t.shape[0]))

            self.p.convolve(self.IRF1)
            self.p.convolved = True
            formula = self.p + drift
            y = formula.design(time=self.t)
            self.assertEquals(y.shape[::-1], (2 + df, self.t.shape[0]))

            self.p.convolve(self.IRF2)
            self.p.convolved = True
            formula = self.p + drift
            y = formula.design(time=self.t)
            self.assertEquals(y.shape[::-1], (4 + df, self.t.shape[0]))

            self.p.convolve(self.IRF1)
            formula = self.p + drift * self.p
            y = formula.design(time=self.t)
            self.assertEquals(y.shape[::-1], (2 + df * 2, self.t.shape[0]))

    def testDesign2(self):
        self.setup_terms()
        
        for df in range(4, 10):
            drift_fn = protocol.SplineConfound(window=[0,300], df=df)
            drift = protocol.ExperimentalQuantitative('drift', drift_fn)

            formula = self.p + drift
            y = formula.names()
            self.assertEquals(len(y), 2 + df)

            formula = self.p + drift
            y = formula.names()
            self.assertEquals(len(y), 2 + df)

            self.p.convolve(self.IRF2)
            self.convolved = True
            formula = self.p + drift
            y = formula.names()
            self.assertEquals(len(y), 4 + df)

            self.p.convolve(self.IRF1)
            formula = self.p + drift * self.p
            y = formula.names()
            self.assertEquals(len(y), 2 + df * 2)

    def testTimeFn1(self):
        self.setup_terms()
        
        drift_fn = protocol.SplineConfound(window=[0,300], df=7)
        drift = protocol.ExperimentalQuantitative('drift', drift_fn)
        d = drift.astimefn()
        t = N.arange(0,30,1.)

        Z = float(N.random.standard_normal(()))

        D = d * Z
        N.testing.assert_almost_equal(D(t), N.array(drift_fn(t)) * Z)

        D = d / Z
        N.testing.assert_almost_equal(D(t), N.array(drift_fn(t)) / Z)

        D = d - Z
        N.testing.assert_almost_equal(D(t), N.array(drift_fn(t)) - Z)

        D = d + Z
        N.testing.assert_almost_equal(D(t), N.array(drift_fn(t)) + Z)


    def testTimeFn2(self):
        self.setup_terms()
        
        drift_fn = protocol.SplineConfound(window=[0,300], df=7)
        drift = protocol.ExperimentalQuantitative('drift', drift_fn)
        d = drift.astimefn()
        t = N.arange(0,30,1.)

        D = d * d
        N.testing.assert_almost_equal(D(t), N.array(drift_fn(t))**2)

        D = d / d
        N.testing.assert_almost_equal(D(t), N.array(drift_fn(t)) * recipr0(drift_fn(t)))

        D = d - d
        N.testing.assert_almost_equal(D(t), N.zeros(D(t).shape, N.float64))

        D = d + d
        N.testing.assert_almost_equal(D(t), 2 * N.array(drift_fn(t)))

    def testTimeFn3(self):
        self.setup_terms()
        
        drift_fn = protocol.SplineConfound(window=[0,300], df=7)
        drift = protocol.ExperimentalQuantitative('drift', drift_fn)
        d = drift.astimefn()
        t = N.arange(0,30,1.)

{'fraction_non_alphanumeric': 0.1461369534837293, 'fraction_numerical': 0.058028881346908895, 'mean_word_length': 1.7635159985288709, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2977333', 'n_tokens_mistral': 5748, 'n_tokens_neox': 5141, 'n_words': 927}