<commit_before># -*- coding: utf-8 -*-

#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import sys

import numpy as np
from scipy.misc import logsumexp

from gpmcc.utils import config as cu
from gpmcc.utils import general as gu

class ParticleDim(object):
    """Holds data, model type, and hyperparameters."""

    def __init__(self, dist, distargs=None, n_grid=30):
        """Dimension constructor.

        Arguments:
        -- X: a numpy array of data.
        -- dist: The name of the distribution.

        Optional arguments:
        -- Zr: partition of data to clusters. If not specified, no clusters are
        initialized.
        -- n_grid: number of hyperparameter grid bins. Default is 30.
        -- distargs: some data types require additional information, for example
        multinomial data requires the number of multinomial categories. See the
        documentation for each type for details
        """
        # Model type.
        self.model = cu.dist_class(dist)
        self.cctype = self.model.cctype
        self.distargs = distargs if distargs is not None else {}

        # Distribution hyperparams.
        self.n_grid = n_grid
        self.hypers_grids = dict()
        self.hypers = dict()

        # CRP hyperparams.
        self.alpha = 1
        self.alpha_grid = []

        # Initial state variables.
        self.weight = 0
        self.clusters = []
        self.Xobs = np.asarray([])
        self.Nobs = 0
        self.Zr = np.asarray([])

    def particle_initialize(self, x):
        """Clears entire state to a single observation."""
        # Initialize the SMC with first observation into cluster 0.
        self.clusters = [self.model(distargs=self.distargs, **self.hypers)]
        self.Nobs = 1
        self.Xobs = np.asarray([x])
        self.Zr = np.asarray([0])
        self.insert_element(0, self.Zr[0])
        self.weight = self.clusters[0].marginal_logp()
        # Hyperparameters.
        self.update_grids(self.Xobs)
        self.hypers = self.model.init_hypers(self.hypers_grids, self.Xobs)

    def particle_learn(self, X):
        """Iteratively applies particle learning on the observations in X."""
        if self.Nobs == 0:
            self.particle_initialize(X[0])
            X = X[1:]
        for i, x in enumerate(X):
            # Monitor progress.
            percentage = float(i+1) / len(X)
            progress = ' ' * 30
            fill = int(percentage * len(progress))
            progress = '[' + '=' * fill + progress[fill:] + ']'
            print '{} {:1.2f}%\r'.format(progress, 100 * percentage),
            sys.stdout.flush()
            # Include the new datapoint.
            self.incorporate(x)
            self.weight += self.transition_rows(target_rows=[self.Nobs-1])
            # Gibbs transition.
            self.transition_rows(target_rows=range(self.Nobs))
            self.transition_hypers()
            self.transition_alpha()
        return self.weight

    def incorporate(self, x):
        """Incorporates a new observation in particle learning."""
        # Arbitrarily assign to cluster 0, not consequential.
        self.Nobs += 1
        self.Xobs = np.append(self.Xobs, x)
        assert self.Nobs == len(self.Xobs)
        rowid = self.Nobs - 1
        self.insert_element(rowid, 0)
        self.Zr = np.append(self.Zr, 0)
        self.update_grids(self.Xobs)<commit_msg>Reintroduce plotting.<commit_after># -*- coding: utf-8 -*-

#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import sys

import numpy as np
import matplotlib.pyplot as plt
from scipy.misc import logsumexp

from gpmcc.utils import config as cu
from gpmcc.utils import general as gu

class ParticleDim(object):
    """Holds data, model type, and hyperparameters."""

    def __init__(self, dist, distargs=None, n_grid=30):
        """Dimension constructor.

        Arguments:
        -- X: a numpy array of data.
        -- dist: The name of the distribution.

        Optional arguments:
        -- Zr: partition of data to clusters. If not specified, no clusters are
        initialized.
        -- n_grid: number of hyperparameter grid bins. Default is 30.
        -- distargs: some data types require additional information, for example
        multinomial data requires the number of multinomial categories. See the
        documentation for each type for details
        """
        # Model type.
        self.model = cu.dist_class(dist)
        self.cctype = self.model.cctype
        self.distargs = distargs if distargs is not None else {}

        # Distribution hyperparams.
        self.n_grid = n_grid
        self.hypers_grids = dict()
        self.hypers = dict()

        # CRP hyperparams.
        self.alpha = 1
        self.alpha_grid = []

        # Initial state variables.
        self.weight = 0
        self.clusters = []
        self.Xobs = np.asarray([])
        self.Nobs = 0
        self.Zr = np.asarray([])

    def particle_initialize(self, x):
        """Clears entire state to a single observation."""
        # Initialize the SMC with first observation into cluster 0.
        self.clusters = [self.model(distargs=self.distargs, **self.hypers)]
        self.Nobs = 1
        self.Xobs = np.asarray([x])
        self.Zr = np.asarray([0])
        self.insert_element(0, self.Zr[0])
        self.weight = self.clusters[0].marginal_logp()
        # Hyperparameters.
        self.update_grids(self.Xobs)
        self.hypers = self.model.init_hypers(self.hypers_grids, self.Xobs)

    def particle_learn(self, X, n_gibbs=1, plot=False, progress=False):
        """Iteratively applies particle learning on the observations in X."""
        if self.Nobs == 0:
            self.particle_initialize(X[0])
            X = X[1:]
        if plot:
            plt.ion(); plt.show()
            _, ax = plt.subplots()
            self.plot_dist(ax=ax)
            plt.draw()
            plt.pause(3)
        for i, x in enumerate(X):
            if progress:
                # Monitor progress.
                percentage = float(i+1) / len(X)
                progress = ' ' * 30
                fill = int(percentage * len(progress))
                progress = '[' + '=' * fill + progress[fill:] + ']'
                print '{} {:1.2f}%\r'.format(progress, 100 * percentage),
                sys.stdout.flush()
            # Include the new datapoint.
            self.incorporate(x)
            self.weight += self.transition_rows(target_rows=[self.Nobs-1])
            for _ in xrange(n_gibbs):
                self.gibbs_transition()
                # Plot?
                if plot:
                    ax.clear()
                    self.plot_dist(ax=ax)
                    plt.draw()
                    plt.pause(3)
        return self.weight

    def gibbs_transition(self):
        # Gibbs transition.
        self.transition_rows(target_rows=range(self.Nobs))
        self.transition_hypers()
        self.transition_alpha()


    def incorporate(self, x):
        """Incorporates a new observation in particle learning."""
        # Arbitrarily assign to cluster 0, not consequential.
        self.Nobs += 1
        self.Xobs = np.append(self.Xobs, x)
        assert self.Nobs == len(self.Xobs)
        rowid = self.Nobs - 1
        self.insert_element(rowid, 0)
        self.Zr = np.append(self.Zr, 0)
        self.update_grids(self.Xobs)
{'fraction_non_alphanumeric': 0.10176568837986161, 'fraction_numerical': 0.008709138630398473, 'mean_word_length': 2.079720793534166, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '798901', 'n_tokens_mistral': 2492, 'n_tokens_neox': 2367, 'n_words': 721}