<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2015-2016 MIT Probabilistic Computing Project

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#    http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import math

import numpy as np

import gpmcc.utils.config as cu
import gpmcc.utils.general as gu
import gpmcc.utils.plots as pu


class Dim(object):
    """Dim holds sufficient statistics, DistributionGpm type, clusters, and
    shared hyperparameters and grids. Technically not GPM, but easily becomes
    one by placing creating a View with a single Dim."""

    def __init__(self, outputs, inputs=None, cctype=None, hypers=None,
            params=None, distargs=None, rng=None):
        """Dim constructor provides a convenience method for bulk incorporate
        and unincorporate by specifying the data and optional row partition.

        Parameters
        ----------
        cctype : str
            DistributionGpm name see `gpmcc.utils.config`.
        index : int
            Unique identifier for this dim.
        distargs : dict, optional.
            Distargs appropriate for the cctype. For details on
            distargs see the documentation for each DistributionGpm.
        """
        self.rng = gu.gen_rng() if rng is None else rng

        # Identifier.
        if len(outputs) != 1:
            raise ValueError('Dim requires exactly 1 output.')
        self.outputs = outputs
        self.inputs = inputs if inputs else []
        # XXX ENCAPSULATE ME!
        self.index = self.outputs[0]

        # Model type.
        self.model = cu.cctype_class(cctype)
        self.cctype = self.model.name()
        self.distargs = distargs if distargs is not None else {}

        # Hyperparams.
        self.hyper_grids = {}
        self.hypers = hypers if hypers is not None else {}

        # Clusters.
        self.clusters = []
        self.clusters_inverse = {}
        self.ignored = set([])

        # Auxiliary singleton model.
        self.aux_model = self.create_aux_model()

    # --------------------------------------------------------------------------
    # Observe

    def incorporate(self, rowid, query, evidence):
        """Record an observation.

        If k < len(self.clusters) then x will be incorporated to cluster k.
        If k == len(self.clusters) then a new cluster will be created.
        If k > len(self.clusters) then an error will be thrown.
        """
        k, evidence, valid = self.preprocess(query, evidence)
        assert k <= len(self.clusters)
        if k == len(self.clusters):
            self.clusters.append(self.aux_model)
            self.aux_model = self.create_aux_model()
        if valid:
            self.clusters[k].incorporate(rowid, query, evidence)
            self.clusters_inverse[rowid] = self.clusters[k]
        else:
            self.ignored.add(rowid)

    def unincorporate(self, rowid):
        """Remove observation rowid."""
        if rowid in self.ignored:
            self.ignored.remove(rowid)
        else:
            cluster = self.clusters_inverse[rowid]
            cluster.unincorporate(rowid)
            del self.clusters_inverse[rowid]

    # --------------------------------------------------------------------------
    # Github issue #65.

    def logpdf_score(self, k=None):
        """Return log score summed over all clusters."""
        return sum(cluster.logpdf_score() for cluster in self.clusters)

    # --------------------------------------------------------------------------
    # logpdf

    def logpdf(self, rowid, query, evidence):
        """Evaluate the log density of the query given evidence."""
        k, evidence, valid = self.preprocess(query, evidence)
        cluster = self.aux_model if k==len(self.clusters) else self.clusters[k]
        return cluster.logpdf(rowid, query, evidence) if valid else 0
<commit_msg>Raise error on multiple incorporate of same rowid.<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2015-2016 MIT Probabilistic Computing Project

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#    http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import math

import numpy as np

import gpmcc.utils.config as cu
import gpmcc.utils.general as gu
import gpmcc.utils.plots as pu


class Dim(object):
    """Dim holds sufficient statistics, DistributionGpm type, clusters, and
    shared hyperparameters and grids. Technically not GPM, but easily becomes
    one by placing creating a View with a single Dim."""

    def __init__(self, outputs, inputs=None, cctype=None, hypers=None,
            params=None, distargs=None, rng=None):
        """Dim constructor provides a convenience method for bulk incorporate
        and unincorporate by specifying the data and optional row partition.

        Parameters
        ----------
        cctype : str
            DistributionGpm name see `gpmcc.utils.config`.
        index : int
            Unique identifier for this dim.
        distargs : dict, optional.
            Distargs appropriate for the cctype. For details on
            distargs see the documentation for each DistributionGpm.
        """
        self.rng = gu.gen_rng() if rng is None else rng

        # Identifier.
        if len(outputs) != 1:
            raise ValueError('Dim requires exactly 1 output.')
        self.outputs = outputs
        self.inputs = inputs if inputs else []
        # XXX ENCAPSULATE ME!
        self.index = self.outputs[0]

        # Model type.
        self.model = cu.cctype_class(cctype)
        self.cctype = self.model.name()
        self.distargs = distargs if distargs is not None else {}

        # Hyperparams.
        self.hyper_grids = {}
        self.hypers = hypers if hypers is not None else {}

        # Clusters.
        self.clusters = []
        self.clusters_inverse = {}
        self.ignored = set([])

        # Auxiliary singleton model.
        self.aux_model = self.create_aux_model()

    # --------------------------------------------------------------------------
    # Observe

    def incorporate(self, rowid, query, evidence):
        """Record an observation.

        If k < len(self.clusters) then x will be incorporated to cluster k.
        If k == len(self.clusters) then a new cluster will be created.
        If k > len(self.clusters) then an error will be thrown.
        """
        if rowid in self.clusters_inverse or rowid in self.ignored:
            raise ValueError('rowid already incorporated: %d' % rowid)
        k, evidence, valid = self.preprocess(query, evidence)
        assert k <= len(self.clusters)
        if k == len(self.clusters):
            self.clusters.append(self.aux_model)
            self.aux_model = self.create_aux_model()
        if valid:
            self.clusters[k].incorporate(rowid, query, evidence)
            self.clusters_inverse[rowid] = self.clusters[k]
        else:
            self.ignored.add(rowid)

    def unincorporate(self, rowid):
        """Remove observation rowid."""
        if rowid in self.ignored:
            self.ignored.remove(rowid)
        elif rowid in self.clusters_inverse:
            cluster = self.clusters_inverse[rowid]
            cluster.unincorporate(rowid)
            del self.clusters_inverse[rowid]
        else:
            raise ValueError('rowid not incorporated: %d' % rowid)

    # --------------------------------------------------------------------------
    # logpdf score

    def logpdf_score(self):
        """Return log score summed over all clusters."""
        return sum(cluster.logpdf_score() for cluster in self.clusters)

    # --------------------------------------------------------------------------
    # logpdf

    def logpdf(self, rowid, query, evidence):
        """Evaluate the log density of the query given evidence."""
        k, evidence, valid = self.preprocess(query, evidence)
        cluster = self.aux_model if k==len(self.clusters) else self.clusters[k]
        return cluster.logpdf(rowid, query, evidence) if valid else 0

{'fraction_non_alphanumeric': 0.13375437013646105, 'fraction_numerical': 0.004059997744445698, 'mean_word_length': 2.033869312350325, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5589001', 'n_tokens_mistral': 2402, 'n_tokens_neox': 2288, 'n_words': 813}