<commit_before>#!/usr/bin/env python
# vim: fileencoding=utf-8

from argparse import ArgumentParser
from os.path import expanduser
from collections import defaultdict
import glob
import math
import os, sys
import sqlite3

import matplotlib
import matplotlib.pyplot as plt
import numpy as np

matplotlib.rc('axes', labelsize='large')
matplotlib.rc('figure', figsize=(8, 1.5))
matplotlib.rc('figure.subplot', left=0.09, right=0.92, bottom=0.275)
matplotlib.rc('font', size=7)
matplotlib.rc('font', **{'sans-serif' : 'DejaVu LGC Sans', 'family' : 'sans-serif'})

class SmartList(list):
    """Stupid extended list."""
    def __init__(self, *args, **kwargs):
        list.__init__(self, *args, **kwargs)
    def __add__(self, other):
        return super(SmartList, self).__add__([other])
    def __iadd__(self, other):
        return super(SmartList, self).__iadd__([other])

def html_tag(tag, *args, **kwargs):
    attr = " ".join(['{0}="{1}"'.format(a, b.replace('"', r'\"')) for a, b in kwargs.items()])
    return '<{0}>\n{1}\n</{2}>\n'.format(" ".join([tag, attr]), "\n".join(args), tag)

def get_stats(dir, label):
    s = defaultdict(list)
    for subdir in glob.glob(os.path.join(dir, label, 'successful', '*')):
        (start, ready, done) = parse.get_wrapper_times(subdir)
        (finit, fopen, first, cs) = parse.get_cmssw_times(subdir)

        s['start'].append(start)
        s['ready'].append(ready)
        s['done'].append(done)
        s['finit'].append(finit)
        s['fopen'].append(fopen)
        s['first'].append(first)
        s['changed servers'].append(cs)

    return s

def make_histo(a, num_bins, xlabel, ylabel, filename, dir, **kwargs):
    # fig, (ax0, ax1) = plt.subplots(nrows=2, sharex=True)
    if 'log' in kwargs:
        if kwargs['log'] == True or kwargs['log'] == 'y':
            plt.yscale('log')
        elif kwargs['log'] == 'x':
            plt.xscale('log')
        del kwargs['log']

    if 'stats' in kwargs:
        stats = kwargs['stats']
        del kwargs['stats']
    else:
        stats = False

    plt.hist(a, bins=num_bins, histtype='barstacked', **kwargs)

    if stats:
        all = np.concatenate(a)
        avg = np.average(all)
        var = np.var(all)
        plt.figtext(0.75, 0.775, u"μ = {0:.3g}, σ = {1:.3g}".format(avg, var), ha="center")

    # ax0.set_title(t)

    # ax0.set_ylabel(ylabel)
    # ax1.set_xlabel(xlabel)
    # ax1.set_ylabel(ylabel)

    # plt.set_title(t)
    # plt.set_xlabel(xlabel)
    # plt.set_ylabel(ylabel)

    # plt.title(t)
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)

    if 'label' in kwargs:
        plt.legend(bbox_to_anchor=(0.5, 0.9), loc='lower center', ncol=len(kwargs['label']), prop={'size': 7})

    return save_and_close(dir, filename)

def make_frequency_pie(a, name, dir):
    vals = np.unique(a)
    counts = [len(a[a == v]) for v in vals]
    plt.pie(counts, labels=vals)
    fig = plt.gcf()
    fig.set_size_inches(6, 6)<commit_msg>Display the std. dev., not the variance.<commit_after>#!/usr/bin/env python
# vim: fileencoding=utf-8

from argparse import ArgumentParser
from os.path import expanduser
from collections import defaultdict
import glob
import math
import os, sys
import sqlite3

import matplotlib
import matplotlib.pyplot as plt
import numpy as np

matplotlib.rc('axes', labelsize='large')
matplotlib.rc('figure', figsize=(8, 1.5))
matplotlib.rc('figure.subplot', left=0.09, right=0.92, bottom=0.275)
matplotlib.rc('font', size=7)
matplotlib.rc('font', **{'sans-serif' : 'DejaVu LGC Sans', 'family' : 'sans-serif'})

class SmartList(list):
    """Stupid extended list."""
    def __init__(self, *args, **kwargs):
        list.__init__(self, *args, **kwargs)
    def __add__(self, other):
        return super(SmartList, self).__add__([other])
    def __iadd__(self, other):
        return super(SmartList, self).__iadd__([other])

def html_tag(tag, *args, **kwargs):
    attr = " ".join(['{0}="{1}"'.format(a, b.replace('"', r'\"')) for a, b in kwargs.items()])
    return '<{0}>\n{1}\n</{2}>\n'.format(" ".join([tag, attr]), "\n".join(args), tag)

def get_stats(dir, label):
    s = defaultdict(list)
    for subdir in glob.glob(os.path.join(dir, label, 'successful', '*')):
        (start, ready, done) = parse.get_wrapper_times(subdir)
        (finit, fopen, first, cs) = parse.get_cmssw_times(subdir)

        s['start'].append(start)
        s['ready'].append(ready)
        s['done'].append(done)
        s['finit'].append(finit)
        s['fopen'].append(fopen)
        s['first'].append(first)
        s['changed servers'].append(cs)

    return s

def make_histo(a, num_bins, xlabel, ylabel, filename, dir, **kwargs):
    # fig, (ax0, ax1) = plt.subplots(nrows=2, sharex=True)
    if 'log' in kwargs:
        if kwargs['log'] == True or kwargs['log'] == 'y':
            plt.yscale('log')
        elif kwargs['log'] == 'x':
            plt.xscale('log')
        del kwargs['log']

    if 'stats' in kwargs:
        stats = kwargs['stats']
        del kwargs['stats']
    else:
        stats = False

    plt.hist(a, bins=num_bins, histtype='barstacked', **kwargs)

    if stats:
        all = np.concatenate(a)
        avg = np.average(all)
        var = np.std(all)
        plt.figtext(0.75, 0.775, u"μ = {0:.3g}, σ = {1:.3g}".format(avg, var), ha="center")

    # ax0.set_title(t)

    # ax0.set_ylabel(ylabel)
    # ax1.set_xlabel(xlabel)
    # ax1.set_ylabel(ylabel)

    # plt.set_title(t)
    # plt.set_xlabel(xlabel)
    # plt.set_ylabel(ylabel)

    # plt.title(t)
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)

    if 'label' in kwargs:
        plt.legend(bbox_to_anchor=(0.5, 0.9), loc='lower center', ncol=len(kwargs['label']), prop={'size': 7})

    return save_and_close(dir, filename)

def make_frequency_pie(a, name, dir):
    vals = np.unique(a)
    counts = [len(a[a == v]) for v in vals]
    plt.pie(counts, labels=vals)
    fig = plt.gcf()
    fig.set_size_inches(6, 6)
{'fraction_non_alphanumeric': 0.1800708143651998, 'fraction_numerical': 0.015511718091384252, 'mean_word_length': 3.2432045779685263, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1641014', 'n_tokens_mistral': 2261, 'n_tokens_neox': 2211, 'n_words': 465}