<commit_before>"""
Test the signals module
"""
# Author: <NAME>, <NAME>
# License: simplified BSD

import numpy as np
from nose.tools import assert_true, assert_false, assert_raises

# Use nisignal here to avoid name collisions (using nisl.signal is
# not possible)
from .. import signal as nisignal
from ..signal import clean
import scipy.signal


def generate_signals(feature_number=17, n_confounds=5, length=41,
                     same_variance=True, order="C"):
    """Generate test signals.

    Returned signals have no trends at all (to machine precision).
    """
    randgen = np.random.RandomState(0)

    # Generate random confounds
    confounds_shape = (length, n_confounds)
    confounds = np.ndarray(confounds_shape, order=order)
    confounds[...] = randgen.randn(*confounds_shape)
    confounds[...] = scipy.signal.detrend(confounds, axis=0)

    # Compute noise based on confounds, with random factors
    factors = randgen.randn(n_confounds, feature_number)
    noises_shape = (length, feature_number)
    noises = np.ndarray(noises_shape, order=order)
    noises[...] = np.dot(confounds, factors)
    noises[...] = scipy.signal.detrend(noises, axis=0)

    # Generate random signals with random amplitudes
    signals_shape = noises_shape
    signals = np.ndarray(signals_shape, order=order)
    if same_variance:
        signals[...] = randgen.randn(*signals_shape)
    else:
        signals[...] = (4. * abs(randgen.randn(signals_shape[1])) + 0.5
                        ) * randgen.randn(*signals_shape)

    signals[...] = scipy.signal.detrend(signals, axis=0)
    return signals, noises, confounds


def generate_trends(feature_number=17, length=41):
    """Generate linearly-varying signals, with zero mean.

    Returns
    =======
    trends (numpy.ndarray)
        shape: (length, feature_number)
    """
    randgen = np.random.RandomState(0)
    trends = scipy.signal.detrend(np.linspace(0, 1.0, length), type="constant")
    trends = np.repeat(np.atleast_2d(trends).T, feature_number, axis=1)
    factors = randgen.randn(feature_number)
    return trends * factors


def test_butterworth():
    randgen = np.random.RandomState(0)
    n_features = 20000
    n_samples = 100

    sampling = 100
    low_pass = 30
    high_pass = 10

    # Compare output for different options.
    # single timeseries
    data = randgen.randn(n_samples)
    data_original = data.copy()

    out_single = nisignal.butterworth(data, sampling,
                                       low_pass=low_pass, high_pass=high_pass,
                                       copy=True)
    np.testing.assert_almost_equal(data, data_original)
    nisignal.butterworth(data, sampling,
                          low_pass=low_pass, high_pass=high_pass,
                          copy=False, save_memory=True)
    np.testing.assert_almost_equal(out_single, data)

    # multiple timeseries
    data = randgen.randn(n_samples, n_features)
    data[:, 0] = data_original  # set first timeseries to previous data
    data_original = data.copy()

    out1 = nisignal.butterworth(data, sampling,
                                 low_pass=low_pass, high_pass=high_pass,
                                 copy=True)
    np.testing.assert_almost_equal(data, data_original)
    # check that multiple- and single-timeseries filtering do the same thing.
    np.testing.assert_almost_equal(out1[:, 0], out_single)
    nisignal.butterworth(data, sampling,
                          low_pass=low_pass, high_pass=high_pass,
                          copy=False)
    np.testing.assert_almost_equal(out1, data)


def test_standardize():
    randgen = np.random.RandomState(0)
    n_features = 10
    n_samples = 17

    # Create random signals with offsets
    a = randgen.random_sample((n_samples, n_features))
    a += np.linspace(0, 2., n_features)

    # transpose array to fit _standardize input.
    # Without trend removal
    b = nisignal._standardize(a, normalize=True)
    energies = (b ** 2).sum(axis=0)
    np.testing.assert_almost_equal(energies, np.ones(n_features))
    np.testing.assert_almost_equal(b.sum(axis=0), np.zeros(n_features))

    # With trend removal
    a = np.atleast_2d(np.linspace(0, 2., n_features)).T
    b = nisignal._standardize(a, detrend=True, normalize=False)
    np.testing.assert_almost_equal(b, np.zeros(b.shape))<commit_msg>Renamed every occurence of "randgen" "to rand_gen"<commit_after>"""
Test the signals module
"""
# Author: <NAME>, <NAME>
# License: simplified BSD

import numpy as np
from nose.tools import assert_true, assert_false, assert_raises

# Use nisignal here to avoid name collisions (using nisl.signal is
# not possible)
from .. import signal as nisignal
from ..signal import clean
import scipy.signal


def generate_signals(feature_number=17, n_confounds=5, length=41,
                     same_variance=True, order="C"):
    """Generate test signals.

    Returned signals have no trends at all (to machine precision).
    """
    rand_gen = np.random.RandomState(0)

    # Generate random confounds
    confounds_shape = (length, n_confounds)
    confounds = np.ndarray(confounds_shape, order=order)
    confounds[...] = rand_gen.randn(*confounds_shape)
    confounds[...] = scipy.signal.detrend(confounds, axis=0)

    # Compute noise based on confounds, with random factors
    factors = rand_gen.randn(n_confounds, feature_number)
    noises_shape = (length, feature_number)
    noises = np.ndarray(noises_shape, order=order)
    noises[...] = np.dot(confounds, factors)
    noises[...] = scipy.signal.detrend(noises, axis=0)

    # Generate random signals with random amplitudes
    signals_shape = noises_shape
    signals = np.ndarray(signals_shape, order=order)
    if same_variance:
        signals[...] = rand_gen.randn(*signals_shape)
    else:
        signals[...] = (4. * abs(rand_gen.randn(signals_shape[1])) + 0.5
                        ) * rand_gen.randn(*signals_shape)

    signals[...] = scipy.signal.detrend(signals, axis=0)
    return signals, noises, confounds


def generate_trends(feature_number=17, length=41):
    """Generate linearly-varying signals, with zero mean.

    Returns
    =======
    trends (numpy.ndarray)
        shape: (length, feature_number)
    """
    rand_gen = np.random.RandomState(0)
    trends = scipy.signal.detrend(np.linspace(0, 1.0, length), type="constant")
    trends = np.repeat(np.atleast_2d(trends).T, feature_number, axis=1)
    factors = rand_gen.randn(feature_number)
    return trends * factors


def test_butterworth():
    rand_gen = np.random.RandomState(0)
    n_features = 20000
    n_samples = 100

    sampling = 100
    low_pass = 30
    high_pass = 10

    # Compare output for different options.
    # single timeseries
    data = rand_gen.randn(n_samples)
    data_original = data.copy()

    out_single = nisignal.butterworth(data, sampling,
                                       low_pass=low_pass, high_pass=high_pass,
                                       copy=True)
    np.testing.assert_almost_equal(data, data_original)
    nisignal.butterworth(data, sampling,
                          low_pass=low_pass, high_pass=high_pass,
                          copy=False, save_memory=True)
    np.testing.assert_almost_equal(out_single, data)

    # multiple timeseries
    data = rand_gen.randn(n_samples, n_features)
    data[:, 0] = data_original  # set first timeseries to previous data
    data_original = data.copy()

    out1 = nisignal.butterworth(data, sampling,
                                 low_pass=low_pass, high_pass=high_pass,
                                 copy=True)
    np.testing.assert_almost_equal(data, data_original)
    # check that multiple- and single-timeseries filtering do the same thing.
    np.testing.assert_almost_equal(out1[:, 0], out_single)
    nisignal.butterworth(data, sampling,
                          low_pass=low_pass, high_pass=high_pass,
                          copy=False)
    np.testing.assert_almost_equal(out1, data)


def test_standardize():
    rand_gen = np.random.RandomState(0)
    n_features = 10
    n_samples = 17

    # Create random signals with offsets
    a = rand_gen.random_sample((n_samples, n_features))
    a += np.linspace(0, 2., n_features)

    # transpose array to fit _standardize input.
    # Without trend removal
    b = nisignal._standardize(a, normalize=True)
    energies = (b ** 2).sum(axis=0)
    np.testing.assert_almost_equal(energies, np.ones(n_features))
    np.testing.assert_almost_equal(b.sum(axis=0), np.zeros(n_features))

    # With trend removal
    a = np.atleast_2d(np.linspace(0, 2., n_features)).T
    b = nisignal._standardize(a, detrend=True, normalize=False)
    np.testing.assert_almost_equal(b, np.zeros(b.shape))
{'fraction_non_alphanumeric': 0.10691391941391941, 'fraction_numerical': 0.01304945054945055, 'mean_word_length': 2.940911141181777, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5868435', 'n_tokens_mistral': 2923, 'n_tokens_neox': 2779, 'n_words': 601}