<commit_before>'''Utilities.'''
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import itertools
import operator
import os

import numpy as np
import scipy.io.wavfile as wavfile
import scipy.signal as signal

from chainer.dataset.dataset_mixin import DatasetMixin


def binarize(images, xp=np):
    """
    Stochastically binarize values in [0, 1] by treating them as p-values of
    a Bernoulli distribution.
    """
    return (xp.random.uniform(size=images.shape) < images).astype('i')


def quantisize(images, levels):
    return (np.digitize(images, np.arange(levels) / levels) - 1).astype('i')


def convert_to_rgb(images, xp=np):
    return xp.tile(images, [1, 3, 1, 1])


def sample_from(distribution):
    batch_size, bins = distribution.shape
    return np.array([np.random.choice(bins, p=distr) for distr in distribution])


def extract_labels(data):
    return np.fromiter(map(operator.itemgetter(1), data), dtype='i')


def extract_images(data):
    return np.array(list(map(operator.itemgetter(0), data))).astype('f')


def mulaw(audio, mu=255):
    return np.sign(audio) * np.log1p(mu * audio) / np.log1p(mu)


def wav_to_float(audio, bits=16):
    '''Squash -2 ** 15; 2 ** 15 into [-1, 1] range'''
    return audio / 2 ** (bits-1)


def wav_files_in(dir):
    for path, _, files in os.walk(dir):
        names = [name for name in files if '.wav' in name]
        for name in names:
            yield os.path.join(path, name)


def _preprocess(ifilename, rate, chunk_length):
    baserate, data = wavfile.read(ifilename)
    audio = signal.resample_poly(data, rate, baserate)
    audio = mulaw(wav_to_float(audio))
    while len(audio) >= chunk_length:
        yield audio[:chunk_length]<commit_msg>Fix signal preprocessing<commit_after>'''Utilities.'''
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import itertools
import operator
import os

import numpy as np
import scipy.io.wavfile as wavfile
import scipy.signal as signal

from chainer.dataset.dataset_mixin import DatasetMixin


def binarize(images, xp=np):
    """
    Stochastically binarize values in [0, 1] by treating them as p-values of
    a Bernoulli distribution.
    """
    return (xp.random.uniform(size=images.shape) < images).astype('i')


def quantisize(images, levels):
    return (np.digitize(images, np.arange(levels) / levels) - 1).astype('i')


def convert_to_rgb(images, xp=np):
    return xp.tile(images, [1, 3, 1, 1])


def sample_from(distribution):
    batch_size, bins = distribution.shape
    return np.array([np.random.choice(bins, p=distr) for distr in distribution])


def extract_labels(data):
    return np.fromiter(map(operator.itemgetter(1), data), dtype='i')


def extract_images(data):
    return np.array(list(map(operator.itemgetter(0), data))).astype('f')


def mulaw(audio, mu=255):
    return np.sign(audio) * np.log1p(mu * np.abs(audio)) / np.log1p(mu)


def wav_to_float(audio, bits=16):
    '''Squash -2 ** 15; 2 ** 15 into [-1, 1] range'''
    return audio / 2 ** (bits-1)


def wav_files_in(dir):
    for path, _, files in os.walk(dir):
        names = [name for name in files if '.wav' in name]
        for name in names:
            yield os.path.join(path, name)


def _preprocess(ifilename, rate, chunk_length):
    baserate, data = wavfile.read(ifilename)
    audio = signal.resample_poly(data, rate, baserate)
    audio = mulaw(wav_to_float(audio))
    while len(audio) >= chunk_length:
        yield audio[:chunk_length]
{'fraction_non_alphanumeric': 0.12965555866704004, 'fraction_numerical': 0.014561747409689163, 'mean_word_length': 3.7374005305039786, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '62452', 'n_tokens_mistral': 1279, 'n_tokens_neox': 1212, 'n_words': 315}