<commit_before>__author__ = 'noe'

import psutil
import numpy as np

from pyemma.coordinates.clustering.interface import AbstractClustering
from pyemma.coordinates.transform.transformer import Transformer
from pyemma.coordinates.io.reader import ChunkedReader
from pyemma.coordinates.io.feature_reader import FeatureReader
from pyemma.coordinates.util.chaining import build_chain

from pyemma.util.log import getLogger


logger = getLogger('Discretizer')
__all__ = ['Discretizer']


class Discretizer(object):

    """
    A Discretizer gets a FeatureReader, which defines the features (distances,
    angles etc.) of given trajectory data and passes this data in a memory
    efficient way through the given pipeline of a Transformer and a clustering.
    The clustering object is responsible for assigning the data to discrete
    states.

    Currently the constructor will calculate everything instantly.


    Parameters
    ----------
    reader : a FeatureReader object
        reads trajectory data and selects features.
    transform : a Transformer object (optional)
        the Transformer will be used to e.g reduce dimensionality of inputs.
    cluster : a clustering object
        used to assign input data to discrete states/ discrete trajectories.
    """

    def __init__(self, reader, transform=None, cluster=None, chunksize=None):
        # check input
        assert isinstance(reader, ChunkedReader), \
            'reader is not of the correct type'
        if (transform is not None):<commit_msg>[discretizer] set default chunksize to 100<commit_after>__author__ = 'noe'

import psutil
import numpy as np

from pyemma.coordinates.clustering.interface import AbstractClustering
from pyemma.coordinates.transform.transformer import Transformer
from pyemma.coordinates.io.reader import ChunkedReader
from pyemma.coordinates.io.feature_reader import FeatureReader
from pyemma.coordinates.util.chaining import build_chain

from pyemma.util.log import getLogger


logger = getLogger('Discretizer')
__all__ = ['Discretizer']


class Discretizer(object):

    """
    A Discretizer gets a FeatureReader, which defines the features (distances,
    angles etc.) of given trajectory data and passes this data in a memory
    efficient way through the given pipeline of a Transformer and a clustering.
    The clustering object is responsible for assigning the data to discrete
    states.

    Currently the constructor will calculate everything instantly.


    Parameters
    ----------
    reader : a FeatureReader object
        reads trajectory data and selects features.
    transform : a Transformer object (optional)
        the Transformer will be used to e.g reduce dimensionality of inputs.
    cluster : a clustering object
        used to assign input data to discrete states/ discrete trajectories.
    chunksize : int 
        how many frames shall be processed at once.
    """

    def __init__(self, reader, transform=None, cluster=None, chunksize=100):
        # check input
        assert isinstance(reader, ChunkedReader), \
            'reader is not of the correct type'
        if (transform is not None):
{'fraction_non_alphanumeric': 0.05917912822144448, 'fraction_numerical': 0.0019090041361756285, 'mean_word_length': 3.536796536796537, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5088065', 'n_tokens_mistral': 818, 'n_tokens_neox': 771, 'n_words': 326}