<commit_before>import numpy as np
import pprint

# optional visdom
try:
    import visdom
except ImportError:
    visdom = None


class Plotter(object):

    def __init__(self, xp, visdom_opts):
        super(Plotter, self).__init__()

        if visdom_opts is None:
            visdom_opts = {}

        assert visdom is not None, "visdom could not be imported"

        # visdom env is given by Experiment name unless specified
        if 'env' not in list(visdom_opts.keys()):
            visdom_opts['env'] = xp.name

        self.viz = visdom.Visdom(**visdom_opts)
        self.windows = {}
        self.append = {}

    def _plot_xy(self, name, tag, x, y, time_idx=True):
        xlabel = 'Time (s)' if time_idx else 'Index'
        if name not in list(self.windows.keys()):
            self.windows[name] = \
                self.viz.line(Y=y, X=x,
                              opts={'legend': [tag],
                                    'title': name,
                                    'xlabel': xlabel})
        else:
            self.viz.updateTrace(Y=y, X=x,
                                 name=tag,
                                 win=self.windows[name],
                                 append=True)

    def plot_xp(self, xp):

        if 'git_diff' in xp.config.keys():
            config = xp.config.copy()
            config.pop('git_diff')
        self.plot_config(config)

        for tag in sorted(xp.logged.keys()):
            for name in sorted(xp.logged[tag].keys()):
                self.plot_logged(xp.logged, tag, name)

    def plot_logged(self, logged, tag, name):
        xy = logged[tag][name]
        x = np.array(xy.keys()).astype(np.float)
        y = np.array(xy.values())
        time_idx = not np.isclose(x, x.astype(np.int)).all()
        self._plot_xy(name, tag, x, y, time_idx)

    def plot_metric(self, metric):
        name, tag = metric.name, metric.tag
        x = np.array([metric.index.get()])
        y = np.array([metric.get()])
        self._plot_xy(name, tag, x, y, metric.time_idx)

    def plot_config(self, config):
        # format dictionary with pretty print
        pp = pprint.PrettyPrinter(indent=4)
        msg = pp.pformat(config)
        # display dict on visdom
        self.viz.text(msg)
<commit_msg>cache for data transimission failures<commit_after>import numpy as np
import pprint

from collections import defaultdict

# optional visdom
try:
    import visdom
except ImportError:
    visdom = None


class Cache(object):
    def __init__(self):
        self.clear()

    def clear(self):
        self._x = []
        self._y = []

    def update(self, metric):
        self._x.append(metric.index.get())
        self._y.append(metric.get())

    @property
    def x(self):
        return np.array(self._x)

    @property
    def y(self):
        return np.array(self._y)


class Plotter(object):

    def __init__(self, xp, visdom_opts):
        super(Plotter, self).__init__()

        if visdom_opts is None:
            visdom_opts = {}

        assert visdom is not None, "visdom could not be imported"

        # visdom env is given by Experiment name unless specified
        if 'env' not in list(visdom_opts.keys()):
            visdom_opts['env'] = xp.name

        self.viz = visdom.Visdom(**visdom_opts)
        self.windows = {}
        self.append = {}
        self.cache = defaultdict(Cache)

    def _plot_xy(self, name, tag, x, y, time_idx=True):
        """
        Creates a window if it does not exist yet.
        Returns True / non-empty string if data has been sent successfully,
        False otherwise.
        """
        xlabel = 'Time (s)' if time_idx else 'Index'
        if name not in list(self.windows.keys()):
            opts = {'legend': [tag], 'title': name, 'xlabel': xlabel}
            self.windows[name] = self.viz.line(Y=y, X=x, opts=opts)
            return True
        else:
            return self.viz.updateTrace(Y=y, X=x, name=tag,
                                        win=self.windows[name],
                                        append=True)

    def plot_xp(self, xp):

        if 'git_diff' in xp.config.keys():
            config = xp.config.copy()
            config.pop('git_diff')
        self.plot_config(config)

        for tag in sorted(xp.logged.keys()):
            for name in sorted(xp.logged[tag].keys()):
                self.plot_logged(xp.logged, tag, name)

    def plot_logged(self, logged, tag, name):
        xy = logged[tag][name]
        x = np.array(xy.keys()).astype(np.float)
        y = np.array(xy.values())
        time_idx = not np.isclose(x, x.astype(np.int)).all()
        self._plot_xy(name, tag, x, y, time_idx)

    def plot_metric(self, metric):
        name, tag = metric.name, metric.tag
        cache = self.cache[metric.name_id()]
        cache.update(metric)
        sent = self._plot_xy(name, tag, cache.x, cache.y,
                             metric.time_idx)
        # clear cache if data has been sent successfully
        if sent:
            cache.clear()

    def plot_config(self, config):
        # format dictionary with pretty print
        pp = pprint.PrettyPrinter(indent=4)
        msg = pp.pformat(config)
        # display dict on visdom
        self.viz.text(msg)

{'fraction_non_alphanumeric': 0.11828571428571429, 'fraction_numerical': 0.00038095238095238096, 'mean_word_length': 1.8399134667387778, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1827906', 'n_tokens_mistral': 1649, 'n_tokens_neox': 1603, 'n_words': 405}