<commit_before># Author: <NAME> <<EMAIL>>
# Contributors:
# Copyright (c) 2016, Stanford University
# All rights reserved.

from __future__ import absolute_import, division
from multiprocessing import Pool, cpu_count
from msmbuilder.utils import list_of_1d
from sklearn.utils import resample
from ..core import _MappingTransformMixin
from ..msm import MarkovStateModel
import numpy as np

class BootStrapMarkovStateModel(_MappingTransformMixin):
    """Bootstrap MarkovState Model.

    This model fits a series of first-order Markov models
    to bootstrap samples obtained from a dataset of
    integer-valued timeseries.The sequence of transition
    matrices are obtained using random sampling with
    replacement over the set of input trajectories.
    The model also fits the mle over the original set.

    Parameters
    ----------
    n_samples : int
        Number of bootstrap models to construct
    n_procs : int
        Number of processors to use.
        Defaults to int(cpu_count/2)
    msm_args: dict
        Dictionary containing arguments to pass unto
        the MSM models.

    Attributes
    ----------
    mle_ : Markov State Model
        MSM model fit unto original dataset. The state mapping
        inferred here is used through the rest of the models.
    all_populations_ : list of lists
        Array of all populations obtained from each model
    resample_ind_ : list
        lis of resample indices used to fit each bootstrap mdl.
        This can be used to regenerate any mdl without having
        to store it inside this object.
    mapped_populations_: array (n_samples, mle.n_states_)
        Array containing population estimates from all the
        models for the states retained in the mle model.
    mapped_populations_mean_: array shape = (mle.n_states_)
        Mean population across the set of models for the states
        contained in the mle model.
    mapped_populations_std_: array shape = (mle.n_states_)
        Population std across the set of models for the states
        contained in the mle model.
    mapped_populations_sem_: array shape = (mle.n_states_)
         Population sem across the set of models for the states
         contained in the mle model.(std/self._succesfully_fit)

    Notes
    -----
    The correct number of bootstrap sample is subject to
    debate with several hundred samples(n_samples)
    being the recommended starting figure.

    The fit function for this model optionally takes in a pool
    of workers making it capable of parallelizing across
    compute nodes via mpi or ipyparallel. This can lead to
    a significant speed up for larger number of samples.
    """
    def __init__(self, n_samples=10,  n_procs=None, msm_args={}):
        self.n_samples = n_samples
        self.n_procs = n_procs
        self.msm_args = msm_args
        self.mle_ = MarkovStateModel(**self.msm_args)

        self._succesfully_fit = 0
        self.all_populations_ = None
        self.mapped_populations_ = None
        self.resample_ind_ = None

    def fit(self, sequences, y=None, pool=None):
        sequences = list_of_1d(sequences)
        self.mle_.fit(sequences, y=y)
        self._parallel_fit(sequences, pool)


    def _fit_one(self, sequences):
        #not sure if i need to actually clone the original mdl but it seems
        #like a safe bet
        mdl = MarkovStateModel(**self.msm_args)
        #there is no guarantee that the mdl fits this sequence set so
        #we return None in that instance.
        try:
            mdl.fit(sequences)
            # solve the eigensystem
            mdl.eigenvalues_[0]
        except:
            mdl = None
        return mdl


    def _parallel_fit(self, sequences, pool=None):
        """
        :param sequences:
        :return:
        """
        if self.n_procs is None:
            self.n_procs = int(cpu_count()/2)
        if pool is None:
            pool = Pool(self.n_procs)


        self.all_populations_ = []
        self.mapped_populations_ = np.zeros((self.n_samples, self.mle_.n_states_))


        #we cache the sequencs of re sampling indices so that any mdl can be
        #regenerated later on
        self.resample_ind_ = [resample(range(len(sequences)))
                                 for _ in range(self.n_samples)]


        jbs =[[sequences[trj_ind] for trj_ind in sample_ind]
              for sample_ind in self.resample_ind_]

        all_mdls = pool.map(self._fit_one, jbs)<commit_msg>whoops forgot to add file<commit_after># Author: <NAME> <<EMAIL>>
# Contributors:
# Copyright (c) 2016, Stanford University
# All rights reserved.

from __future__ import absolute_import, division
from multiprocessing import Pool, cpu_count
from msmbuilder.utils import list_of_1d
from sklearn.utils import resample
from ..core import _MappingTransformMixin
from ..msm import MarkovStateModel
import numpy as np

class BootStrapMarkovStateModel(_MappingTransformMixin):
    """Bootstrap MarkovState Model.

    This model fits a series of first-order Markov models
    to bootstrap samples obtained from a dataset of
    integer-valued timeseries.The sequence of transition
    matrices are obtained using random sampling with
    replacement over the set of input trajectories.
    The model also fits the mle over the original set.

    Parameters
    ----------
    n_samples : int
        Number of bootstrap models to construct
    n_procs : int
        Number of processors to use.
        Defaults to int(cpu_count/2)
    msm_args: dict
        Dictionary containing arguments to pass unto
        the MSM models.

    Attributes
    ----------
    mle_ : Markov State Model
        MSM model fit unto original dataset. The state mapping
        inferred here is used through the rest of the models.
    all_populations_ : list of lists
        Array of all populations obtained from each model
    resample_ind_ : list
        lis of resample indices used to fit each bootstrap mdl.
        This can be used to regenerate any mdl without having
        to store it inside this object.
    mapped_populations_: array (n_samples, mle.n_states_)
        Array containing population estimates from all the
        models for the states retained in the mle model.
    mapped_populations_mean_: array shape = (mle.n_states_)
        Mean population across the set of models for the states
        contained in the mle model.
    mapped_populations_std_: array shape = (mle.n_states_)
        Population std across the set of models for the states
        contained in the mle model.
    mapped_populations_sem_: array shape = (mle.n_states_)
         Population sem across the set of models for the states
         contained in the mle model.(std/self._succesfully_fit)

    Notes
    -----
    The correct number of bootstrap sample is subject to
    debate with several hundred samples(n_samples)
    being the recommended starting figure.

    The fit function for this model optionally takes in a pool
    of workers making it capable of parallelizing across
    compute nodes via mpi or ipyparallel. This can lead to
    a significant speed up for larger number of samples.
    """
    def __init__(self, n_samples=10,  n_procs=None, msm_args={}):
        self.n_samples = n_samples
        self.n_procs = n_procs
        self.msm_args = msm_args
        self.mle_ = MarkovStateModel(**self.msm_args)

        self._succesfully_fit = 0
        self.all_populations_ = None
        self.mapped_populations_ = None
        self.resample_ind_ = None

    def fit(self, sequences, y=None, pool=None):
        sequences = list_of_1d(sequences)
        self.mle_.fit(sequences, y=y)
        self._parallel_fit(sequences, pool)


    def _fit_one(self, sequences):
        mdl = MarkovStateModel(**self.msm_args)
        #there is no guarantee that the mdl fits this sequence set so
        #we return None in that instance.
        try:
            mdl.fit(sequences)
            # solve the eigensystem
            mdl.eigenvalues_[0]
        except:
            mdl = None
        return mdl


    def _parallel_fit(self, sequences, pool=None):

        if self.n_procs is None:
            self.n_procs = int(cpu_count()/2)
        if pool is None:
            pool = Pool(self.n_procs)


        self.all_populations_ = []
        self.mapped_populations_ = np.zeros((self.n_samples, self.mle_.n_states_))


        #we cache the sequencs of re sampling indices so that any mdl can be
        #regenerated later on
        self.resample_ind_ = [resample(range(len(sequences)))
                                 for _ in range(self.n_samples)]


        jbs =[[sequences[trj_ind] for trj_ind in sample_ind]
              for sample_ind in self.resample_ind_]

        all_mdls = pool.map(self._fit_one, jbs)
{'fraction_non_alphanumeric': 0.059487179487179485, 'fraction_numerical': 0.002735042735042735, 'mean_word_length': 2.517434869739479, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1356465', 'n_tokens_mistral': 2493, 'n_tokens_neox': 2378, 'n_words': 912}