<commit_before>import os
import re
from calendar import timegm
from collections import defaultdict
from itertools import cycle
from multiprocessing import Pool, cpu_count

import matplotlib
matplotlib.use("Agg")
matplotlib.rcParams.update({"font.size": 5})
matplotlib.rcParams.update({"lines.linewidth": 0.5})
matplotlib.rcParams.update({"lines.marker": "."})
matplotlib.rcParams.update({"lines.markersize": 3})
matplotlib.rcParams.update({"lines.linestyle": 'None'})
matplotlib.rcParams.update({"axes.linewidth": 0.5})
matplotlib.rcParams.update({"axes.grid": True})
matplotlib.rcParams.update({"axes.formatter.limits": (-6, 6)})
matplotlib.rcParams.update({"legend.numpoints": 1})
matplotlib.rcParams.update({"legend.fancybox": True})
matplotlib.rcParams.update({"legend.markerscale": 1.5})
matplotlib.rcParams.update({"legend.loc": 0})
matplotlib.rcParams.update({"legend.frameon": True})
import matplotlib.pyplot as plt

import numpy as np
import pandas as pd
from cbagent.stores import SerieslyStore
from django.conf import settings
from eventlet import GreenPool
from seriesly import Seriesly

from cbmonitor import models
from cbmonitor.constants import (LABELS, PALETTE,
                                 HISTOGRAMS, ZOOM_HISTOGRAMS, NON_ZERO_VALUES)


class Colors(object):

    def __init__(self):
        self.cycle = cycle(PALETTE)

    def next(self):
        return self.cycle.next()


# Defined externally in order to be pickled
def save_png(filename, series, ylabel, labels, histogram, rebalances):
    fig = plt.figure(figsize=(4.66, 2.625))

    colors = Colors()

    ax = fig.add_subplot(1, 1, 1)
    ax.ticklabel_format(useOffset=False)
    ax.set_ylabel(ylabel)
    if histogram:
        ax.set_xlabel("Percentile")
        width = cycle((0.6, 0.4))
        align = cycle(("edge", "center"))
        if histogram == "_lt90":
            percentiles = range(1, 90)
            x = percentiles
            plt.xlim(0, 90)
        elif histogram == "_gt90":
            percentiles = (90, 95, 97.5, 99, 99.9, 99.99, 99.999)
            x = range(len(percentiles))
            plt.xticks(x, percentiles)
        else:
            percentiles = range(1, 100)
            x = percentiles
            plt.xlim(0, 100)
        for i, s in enumerate(series):
            y = np.percentile(s.values, percentiles)
            ax.bar(x, y, linewidth=0.0, label=labels[i],
                   width=width.next(), align=align.next(), color=colors.next())
    else:
        ax.set_xlabel("Time elapsed, sec")
        for i, s in enumerate(series):
            ax.plot(s.index, s.values, label=labels[i], color=colors.next())
        ymin, ymax = ax.get_ylim()
        plt.ylim(ymin=0, ymax=max(1, ymax * 1.05))

        colors = Colors()
        for rebalance_start, rebalance_end in rebalances:
            plt.axvspan(rebalance_start, rebalance_end,
                        facecolor=colors.next(), alpha=0.1, linewidth=0.5)
    legend = ax.legend()
    legend.get_frame().set_linewidth(0.5)

    fig.tight_layout()
    fig.savefig(filename, dpi=200)
    plt.close()


class Plotter(object):

    def __init__(self):
        self.db = Seriesly()
        self.all_dbs = self.db.list_dbs()

        self.urls = list()
        self.images = list()

        self.eventlet_pool = GreenPool()
        self.mp_pool = Pool(cpu_count())

    def __del__(self):
        self.mp_pool.close()

    def query_data(self, snapshot, cluster, server, bucket, metric, collector):
        query_params = {"ptr": "/{0}".format(metric), "reducer": "avg",
                        "group": 5000}
        if snapshot.name != "all_data":
            ts_from = timegm(snapshot.ts_from.timetuple()) * 1000
            ts_to = timegm(snapshot.ts_to.timetuple()) * 1000
            group = max((ts_from - ts_to) / 500, 5000)  # min 5 sec; max 500 points
            query_params.update({"group": group, "from": ts_from, "to": ts_to})
        db_name = SerieslyStore.build_dbname(cluster, server, bucket, collector)
        if db_name in self.all_dbs:
            return self.db[db_name].query(query_params)
        else:
            return None

    def generate_png_meta(self, snapshot, cluster, server, bucket, metric):
        metric = metric.replace("/", "_")
        title = "{0}] {1}".format(bucket, metric)  # [server bucket] metric
        if server:
            title = "[{0} {1}".format(server, title)
        else:
            title = "[" + title

        filename = "".join((snapshot.name, cluster, title))
        filename = re.sub(r"[\[\]/\\:\*\?\"<>\|& ]", "", filename)
        filename += "{suffix}.png"

        media_url = settings.MEDIA_URL + filename
        media_path = os.path.join(settings.MEDIA_ROOT, filename)
        return title, media_url, media_path

    def get_series(self, metric, data):
        data = dict((k, v[0]) for k, v in data.iteritems())
        series = pd.Series(data)
        series.index = series.index.astype("uint64")
        series.rename(lambda x: x - series.index.values.min(), inplace=True)
        series.rename(lambda x: x / 1000, inplace=True)  # ms -> s

        if metric in NON_ZERO_VALUES and (series == 0).all():
            return None
        else:<commit_msg>handle connection errors while querying seriesly data<commit_after>import os
import re
from calendar import timegm
from collections import defaultdict
from itertools import cycle
from multiprocessing import Pool, cpu_count

import matplotlib
matplotlib.use("Agg")
matplotlib.rcParams.update({"font.size": 5})
matplotlib.rcParams.update({"lines.linewidth": 0.5})
matplotlib.rcParams.update({"lines.marker": "."})
matplotlib.rcParams.update({"lines.markersize": 3})
matplotlib.rcParams.update({"lines.linestyle": 'None'})
matplotlib.rcParams.update({"axes.linewidth": 0.5})
matplotlib.rcParams.update({"axes.grid": True})
matplotlib.rcParams.update({"axes.formatter.limits": (-6, 6)})
matplotlib.rcParams.update({"legend.numpoints": 1})
matplotlib.rcParams.update({"legend.fancybox": True})
matplotlib.rcParams.update({"legend.markerscale": 1.5})
matplotlib.rcParams.update({"legend.loc": 0})
matplotlib.rcParams.update({"legend.frameon": True})
import matplotlib.pyplot as plt

import numpy as np
import pandas as pd
import seriesly
from cbagent.stores import SerieslyStore
from django.conf import settings
from eventlet import GreenPool

from cbmonitor import models
from cbmonitor.constants import (LABELS, PALETTE,
                                 HISTOGRAMS, ZOOM_HISTOGRAMS, NON_ZERO_VALUES)


class Colors(object):

    def __init__(self):
        self.cycle = cycle(PALETTE)

    def next(self):
        return self.cycle.next()


# Defined externally in order to be pickled
def save_png(filename, series, ylabel, labels, histogram, rebalances):
    fig = plt.figure(figsize=(4.66, 2.625))

    colors = Colors()

    ax = fig.add_subplot(1, 1, 1)
    ax.ticklabel_format(useOffset=False)
    ax.set_ylabel(ylabel)
    if histogram:
        ax.set_xlabel("Percentile")
        width = cycle((0.6, 0.4))
        align = cycle(("edge", "center"))
        if histogram == "_lt90":
            percentiles = range(1, 90)
            x = percentiles
            plt.xlim(0, 90)
        elif histogram == "_gt90":
            percentiles = (90, 95, 97.5, 99, 99.9, 99.99, 99.999)
            x = range(len(percentiles))
            plt.xticks(x, percentiles)
        else:
            percentiles = range(1, 100)
            x = percentiles
            plt.xlim(0, 100)
        for i, s in enumerate(series):
            y = np.percentile(s.values, percentiles)
            ax.bar(x, y, linewidth=0.0, label=labels[i],
                   width=width.next(), align=align.next(), color=colors.next())
    else:
        ax.set_xlabel("Time elapsed, sec")
        for i, s in enumerate(series):
            ax.plot(s.index, s.values, label=labels[i], color=colors.next())
        ymin, ymax = ax.get_ylim()
        plt.ylim(ymin=0, ymax=max(1, ymax * 1.05))

        colors = Colors()
        for rebalance_start, rebalance_end in rebalances:
            plt.axvspan(rebalance_start, rebalance_end,
                        facecolor=colors.next(), alpha=0.1, linewidth=0.5)
    legend = ax.legend()
    legend.get_frame().set_linewidth(0.5)

    fig.tight_layout()
    fig.savefig(filename, dpi=200)
    plt.close()


class Plotter(object):

    def __init__(self):
        self.db = seriesly.Seriesly()
        self.all_dbs = self.db.list_dbs()

        self.urls = list()
        self.images = list()

        self.eventlet_pool = GreenPool()
        self.mp_pool = Pool(cpu_count())

    def __del__(self):
        self.mp_pool.close()

    def query_data(self, snapshot, cluster, server, bucket, metric, collector):
        query_params = {"ptr": "/{0}".format(metric), "reducer": "avg",
                        "group": 5000}
        if snapshot.name != "all_data":
            ts_from = timegm(snapshot.ts_from.timetuple()) * 1000
            ts_to = timegm(snapshot.ts_to.timetuple()) * 1000
            group = max((ts_from - ts_to) / 500, 5000)  # min 5s; max 500 points
            query_params.update({"group": group, "from": ts_from, "to": ts_to})
        db_name = SerieslyStore.build_dbname(cluster, server, bucket, collector)
        if db_name in self.all_dbs:
            try:
                return self.db[db_name].query(query_params)
            except seriesly.exceptions.ConnectionError:
                return
        else:
            return

    def generate_png_meta(self, snapshot, cluster, server, bucket, metric):
        metric = metric.replace("/", "_")
        title = "{0}] {1}".format(bucket, metric)  # [server bucket] metric
        if server:
            title = "[{0} {1}".format(server, title)
        else:
            title = "[" + title

        filename = "".join((snapshot.name, cluster, title))
        filename = re.sub(r"[\[\]/\\:\*\?\"<>\|& ]", "", filename)
        filename += "{suffix}.png"

        media_url = settings.MEDIA_URL + filename
        media_path = os.path.join(settings.MEDIA_ROOT, filename)
        return title, media_url, media_path

    def get_series(self, metric, data):
        data = dict((k, v[0]) for k, v in data.iteritems())
        series = pd.Series(data)
        series.index = series.index.astype("uint64")
        series.rename(lambda x: x - series.index.values.min(), inplace=True)
        series.rename(lambda x: x / 1000, inplace=True)  # ms -> s

        if metric in NON_ZERO_VALUES and (series == 0).all():
            return None
        else:
{'fraction_non_alphanumeric': 0.13721393979679042, 'fraction_numerical': 0.02222011205013769, 'mean_word_length': 2.6594857539958303, 'pattern_counts': {'":': 44, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '896799', 'n_tokens_mistral': 3607, 'n_tokens_neox': 3372, 'n_words': 759}