<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
##
# perf_testing.py: Tests the performance of SMC estimation and likelihood
#     calls.
##
# © 2014 <NAME> (<EMAIL>) and
#        <NAME> (<EMAIL>).
#
# Licensed under the AGPL version 3.
##
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
##

## FEATURES ##################################################################

from __future__ import division

## EXPORTS ###################################################################

__all__ = [
    'timing', 'perf_test', 'perf_test_multiple'
]

## IMPORTS ###################################################################

from contextlib import contextmanager
from functools import partial
import threading
import time

import numpy as np

from qinfer.smc import SMCUpdater

## CLASSES ###################################################################

class Timer(object):
    """
    Represents the timing of a block. Call ``stop()`` to stop the
    timer, and query the ``delta_t`` property for the time since this object
    was created.
    """
    _tic = 0
    _toc = None

    def __init__(self):
        self._tic = time.time()

    def stop(self):
        self._toc = time.time()

    @property
    def delta_t(self):
        """
        Returns the time (in seconds) elapsed during the block that was 
        """
        return (self._toc if self._toc is not None else time.time()) - self._tic


class WebProgressThread(threading.Thread):
    done = False
    dirty = False
    progress = 0

    def __init__(self, task, wake_event):
        super(WebProgressThread, self).__init__()
        self._task = task
        self._wake_event = wake_event

    def run(self):
        while True:
            if self.done:
                return
            if self.dirty:
                try:
                    self._task.update(progress=self.progress)
                    self.dirty = False
                    self._wake_event.clear()
                except Exception as ex:
                    print(ex)
            self._wake_event.wait()

## CONTEXT MANAGERS ##########################################################

@contextmanager
def timing():
    """
    Times the execution of a block, returning the result as a
    :class:`qinfer.Timer()`. For example::

    >>> with timing() as t:
    ...     time.sleep(1)
    >>> print t.delta_t # Should return approximately 1.
    """
    t = Timer()
    yield t
    t.stop()

## CONSTANTS #################################################################

PERFORMANCE_DTYPE = [
    ('loss', float),
    ('resample_count', int),
    ('elapsed_time', float),
    ('outcome', int)
]

## FUNCTIONS #################################################################

def actual_dtype(model):
    if isinstance(model.expparams_dtype, str):
        # They're using simple notation for a single field.
        return PERFORMANCE_DTYPE + [('experiment', model.expparams_dtype)], True
    else:
        return PERFORMANCE_DTYPE + model.expparams_dtype, False

def perf_test(
        model, n_particles, prior, n_exp, heuristic_class,
        true_model=None, true_prior=None, true_mps = None
    ):
    """
    Runs a trial of using SMC to estimate the parameters of a model, given a
    number of particles, a prior distribution and an experiment design
    heuristic.

    :param qinfer.Model model: Model whose parameters are to
        be estimated.
    :param int n_particles: Number of SMC particles to use.
    :param qinfer.Distribution prior: Prior to use in selecting
        SMC particles.
    :param int n_exp: Number of experimental data points to draw from the
        model.
    :param qinfer.Heuristic heuristic_class: Constructor function
        for the experiment design heuristic to be used.
    :param qinfer.Model true_model: Model to be used in
        generating experimental data. If ``None``, assumed to be ``model``.
    :param qinfer.Distribution true_prior: Prior to be used in
        selecting the true model parameters. If ``None``, assumed to be
        ``prior``.
    :param np.ndarray true_mps: The true model parameters. If ``None``,
        it will be sampled from ``true_prior``. Note that the performance
        record can only handle one outcome and therefore ONLY ONE TRUE MODEL.
        An error will occur if ``true_mps.shape[0] > 1`` returns ``True``.
    :rtype np.ndarray: See :ref:`perf_testing_struct` for more details on 
        the type returned by this function.
    :return: A record array of performance metrics, indexed by the number
        of experiments performed.
    """

    if true_model is None:
        true_model = model

    if true_prior is None:
        true_prior = prior

    if true_mps is None:
        true_mps = true_prior.sample()

    dtype, is_scalar_exp = actual_dtype(model)
    performance = np.zeros((n_exp,), dtype=dtype)

    updater = SMCUpdater(model, n_particles, prior)
    heuristic = heuristic_class(updater)

    for idx_exp in xrange(n_exp):
        expparams = heuristic()
        datum = true_model.simulate_experiment(true_mps, expparams)

        with timing() as t:
            updater.update(datum, expparams)

        delta = updater.est_mean() - true_mps

        performance[idx_exp]['elapsed_time'] = t.delta_t
        performance[idx_exp]['loss'] = np.dot(delta**2, model.Q)
        performance[idx_exp]['resample_count'] = updater.resample_count
        performance[idx_exp]['outcome'] = datum
        if is_scalar_exp:
            performance[idx_exp]['experiment'] = expparams
        else:
            for param_name in [param[0] for param in model.expparams_dtype]:
                performance[idx_exp][param_name] = expparams[param_name]

    return performance
<commit_msg>added extra_updater_args to perf testing<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
##
# perf_testing.py: Tests the performance of SMC estimation and likelihood
#     calls.
##
# © 2014 <NAME> (<EMAIL>) and
#        <NAME> (<EMAIL>).
#
# Licensed under the AGPL version 3.
##
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
##

## FEATURES ##################################################################

from __future__ import division

## EXPORTS ###################################################################

__all__ = [
    'timing', 'perf_test', 'perf_test_multiple'
]

## IMPORTS ###################################################################

from contextlib import contextmanager
from functools import partial
import threading
import time

import numpy as np

from qinfer.smc import SMCUpdater

## CLASSES ###################################################################

class Timer(object):
    """
    Represents the timing of a block. Call ``stop()`` to stop the
    timer, and query the ``delta_t`` property for the time since this object
    was created.
    """
    _tic = 0
    _toc = None

    def __init__(self):
        self._tic = time.time()

    def stop(self):
        self._toc = time.time()

    @property
    def delta_t(self):
        """
        Returns the time (in seconds) elapsed during the block that was 
        """
        return (self._toc if self._toc is not None else time.time()) - self._tic


class WebProgressThread(threading.Thread):
    done = False
    dirty = False
    progress = 0

    def __init__(self, task, wake_event):
        super(WebProgressThread, self).__init__()
        self._task = task
        self._wake_event = wake_event

    def run(self):
        while True:
            if self.done:
                return
            if self.dirty:
                try:
                    self._task.update(progress=self.progress)
                    self.dirty = False
                    self._wake_event.clear()
                except Exception as ex:
                    print(ex)
            self._wake_event.wait()

## CONTEXT MANAGERS ##########################################################

@contextmanager
def timing():
    """
    Times the execution of a block, returning the result as a
    :class:`qinfer.Timer()`. For example::

    >>> with timing() as t:
    ...     time.sleep(1)
    >>> print t.delta_t # Should return approximately 1.
    """
    t = Timer()
    yield t
    t.stop()

## CONSTANTS #################################################################

PERFORMANCE_DTYPE = [
    ('loss', float),
    ('resample_count', int),
    ('elapsed_time', float),
    ('outcome', int)
]

## FUNCTIONS #################################################################

def actual_dtype(model):
    if isinstance(model.expparams_dtype, str):
        # They're using simple notation for a single field.
        return PERFORMANCE_DTYPE + [('experiment', model.expparams_dtype)], True
    else:
        return PERFORMANCE_DTYPE + model.expparams_dtype, False

def perf_test(
        model, n_particles, prior, n_exp, heuristic_class,
        true_model=None, true_prior=None, true_mps=None,
        extra_updater_args=None
    ):
    """
    Runs a trial of using SMC to estimate the parameters of a model, given a
    number of particles, a prior distribution and an experiment design
    heuristic.

    :param qinfer.Model model: Model whose parameters are to
        be estimated.
    :param int n_particles: Number of SMC particles to use.
    :param qinfer.Distribution prior: Prior to use in selecting
        SMC particles.
    :param int n_exp: Number of experimental data points to draw from the
        model.
    :param qinfer.Heuristic heuristic_class: Constructor function
        for the experiment design heuristic to be used.
    :param qinfer.Model true_model: Model to be used in
        generating experimental data. If ``None``, assumed to be ``model``.
    :param qinfer.Distribution true_prior: Prior to be used in
        selecting the true model parameters. If ``None``, assumed to be
        ``prior``.
    :param np.ndarray true_mps: The true model parameters. If ``None``,
        it will be sampled from ``true_prior``. Note that the performance
        record can only handle one outcome and therefore ONLY ONE TRUE MODEL.
        An error will occur if ``true_mps.shape[0] > 1`` returns ``True``.
    :param dict extra_updater_args: Extra keyword arguments for the updater,
        such as resampling and zero-weight policies.
    :rtype np.ndarray: See :ref:`perf_testing_struct` for more details on 
        the type returned by this function.
    :return: A record array of performance metrics, indexed by the number
        of experiments performed.
    """

    if true_model is None:
        true_model = model

    if true_prior is None:
        true_prior = prior

    if true_mps is None:
        true_mps = true_prior.sample()

    if extra_updater_args is None:
        extra_updater_args = {}

    dtype, is_scalar_exp = actual_dtype(model)
    performance = np.zeros((n_exp,), dtype=dtype)

    updater = SMCUpdater(model, n_particles, prior, **extra_updater_args)
    heuristic = heuristic_class(updater)

    for idx_exp in xrange(n_exp):
        expparams = heuristic()
        datum = true_model.simulate_experiment(true_mps, expparams)

        with timing() as t:
            updater.update(datum, expparams)

        delta = updater.est_mean() - true_mps

        performance[idx_exp]['elapsed_time'] = t.delta_t
        performance[idx_exp]['loss'] = np.dot(delta**2, model.Q)
        performance[idx_exp]['resample_count'] = updater.resample_count
        performance[idx_exp]['outcome'] = datum
        if is_scalar_exp:
            performance[idx_exp]['experiment'] = expparams
        else:
            for param_name in [param[0] for param in model.expparams_dtype]:
                performance[idx_exp][param_name] = expparams[param_name]

    return performance

{'fraction_non_alphanumeric': 0.1538580009240721, 'fraction_numerical': 0.0023101801940551363, 'mean_word_length': 2.9498175182481754, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3485981', 'n_tokens_mistral': 3717, 'n_tokens_neox': 3553, 'n_words': 1225}