<commit_before>#!/usr/bin/env python
"""Compute basic stats about CBench data."""

import csv
import numpy
import pprint
import matplotlib.pyplot as pyplot
import argparse
import sys


class Stats(object):

    """Compute stats and/or graph data."""

    results_file = "results.csv"
    log_file = "cbench.log"
    precision = 3
    run_index = 0
    flow_index = 1
    start_time_index = 2
    end_time_index = 3
    start_steal_time_index = 10
    end_steal_time_index = 11
    used_ram_index = 13
    one_load_index = 16
    five_load_index = 17
    fifteen_load_index = 18
    start_iowait_index = 20
    end_iowait_index = 21

    def __init__(self):
        """Setup some flags and data structures, kick off build_cols call."""
        self.build_cols()
        self.results = {}
        self.results["sample_size"] = len(self.run_col)

    def build_cols(self):
        """Parse results file into lists of values, one per column."""
        self.run_col= []
        self.flows_col = []
        self.runtime_col = []
        self.used_ram_col = []
        self.iowait_col = []
        self.steal_time_col = []
        self.one_load_col = []
        self.five_load_col = []
        self.fifteen_load_col = []

        with open(self.results_file, "rb") as results_fd:
            results_reader = csv.reader(results_fd)
            for row in results_reader:
                try:
                    self.run_col.append(float(row[self.run_index]))
                    self.flows_col.append(float(row[self.flow_index]))
                    self.runtime_col.append(float(row[self.end_time_index]) - \
                        float(row[self.start_time_index]))
                    self.used_ram_col.append(float(row[self.used_ram_index]))
                    self.iowait_col.append(float(row[self.end_iowait_index]) - \
                        float(row[self.start_iowait_index]))
                    self.steal_time_col.append(float(row[self.end_steal_time_index]) - \
                        float(row[self.start_steal_time_index]))
                    self.one_load_col.append(float(row[self.one_load_index]))
                    self.five_load_col.append(float(row[self.five_load_index]))
                    self.fifteen_load_col.append(float(row[self.fifteen_load_index]))
                except ValueError:
                    # Skips header
                    continue

    def compute_flow_stats(self):
        """Compute CBench flows/second stats."""
        flow_stats = {}
        flow_stats["min"] = int(round(numpy.amin(self.flows_col)))
        flow_stats["max"] = int(round(numpy.amax(self.flows_col)))
        flow_stats["mean"] = round(numpy.mean(self.flows_col),
                                                       self.precision)
        flow_stats["stddev"] = round(numpy.std(self.flows_col), self.precision)
        flow_stats["relstddev"] = round((numpy.std(self.flows_col) / \
            numpy.mean(self.flows_col)) * \
            100, self.precision)
        self.results["flows"] = flow_stats

    def build_flow_graph(self, total_graph_count, graph_num):
        """Plot flows/sec data.

        :param total_graph_count: Total number of graphs to render.
        :type total_graph_count: int
        :param graph_num: Number for this graph, <= total_graph_count.
        :type graph_num: int

        """
        pyplot.subplot(total_graph_count, 1, graph_num)
        # "go" means green O's
        pyplot.plot(self.run_col, self.flows_col, "go")
        pyplot.xlabel("Run Number")
        pyplot.ylabel("Flows per Second")

    def compute_ram_stats(self):
        """Compute used RAM stats."""
        ram_stats = {}
        ram_stats["min"] = int(numpy.amin(self.used_ram_col))
        ram_stats["max"] = int(numpy.amax(self.used_ram_col))
        ram_stats["mean"] = round(numpy.mean(self.used_ram_col), self.precision)
        ram_stats["stddev"] = round(numpy.std(self.used_ram_col), self.precision)
        self.results["ram"] = ram_stats

    def build_ram_graph(self, total_graph_count, graph_num):
        """Plot used RAM data.

        :param total_graph_count: Total number of graphs to render.
        :type total_graph_count: int
        :param graph_num: Number for this graph, <= total_graph_count.
        :type graph_num: int

        """
        # Params are numrows, numcols, fignum
        pyplot.subplot(total_graph_count, 1, graph_num)
        # "go" means green O's
        pyplot.plot(self.run_col, self.used_ram_col, "go")
        pyplot.xlabel("Run Number")
        pyplot.ylabel("Used RAM")

    def compute_runtime_stats(self):
        """Compute CBench runtime length stats."""
        runtime_stats = {}
        runtime_stats["min"] = int(numpy.amin(self.runtime_col))
        runtime_stats["max"] = int(numpy.amax(self.runtime_col))
        runtime_stats["mean"] = round(numpy.mean(self.runtime_col), self.precision)
        runtime_stats["stddev"] = round(numpy.std(self.runtime_col), self.precision)
        self.results["runtime"] = runtime_stats

    def build_runtime_graph(self, total_graph_count, graph_num):
        """Plot CBench runtime length data.

        :paruntime total_graph_count: Total number of graphs to render.
        :type total_graph_count: int
        :paruntime graph_num: Number for this graph, <= total_graph_count.
        :type graph_num: int

        """
        # Paruntimes are numrows, numcols, fignum
        pyplot.subplot(total_graph_count, 1, graph_num)
        # "go" means green O's
        pyplot.plot(self.run_col, self.runtime_col, "go")
        pyplot.xlabel("Run Number")
        pyplot.ylabel("CBench runtime (sec)")

# Build stats object
stats = Stats()<commit_msg>Converted stats/graph fns to use generic helper<commit_after>#!/usr/bin/env python
"""Compute basic stats about CBench data."""

import csv
import numpy
import pprint
import matplotlib.pyplot as pyplot
import argparse
import sys


class Stats(object):

    """Compute stats and/or graph data."""

    results_file = "results.csv"
    log_file = "cbench.log"
    precision = 3
    run_index = 0
    flow_index = 1
    start_time_index = 2
    end_time_index = 3
    start_steal_time_index = 10
    end_steal_time_index = 11
    used_ram_index = 13
    one_load_index = 16
    five_load_index = 17
    fifteen_load_index = 18
    start_iowait_index = 20
    end_iowait_index = 21

    def __init__(self):
        """Setup some flags and data structures, kick off build_cols call."""
        self.build_cols()
        self.results = {}
        self.results["sample_size"] = len(self.run_col)

    def build_cols(self):
        """Parse results file into lists of values, one per column."""
        self.run_col= []
        self.flows_col = []
        self.runtime_col = []
        self.used_ram_col = []
        self.iowait_col = []
        self.steal_time_col = []
        self.one_load_col = []
        self.five_load_col = []
        self.fifteen_load_col = []

        with open(self.results_file, "rb") as results_fd:
            results_reader = csv.reader(results_fd)
            for row in results_reader:
                try:
                    self.run_col.append(float(row[self.run_index]))
                    self.flows_col.append(float(row[self.flow_index]))
                    self.runtime_col.append(float(row[self.end_time_index]) - \
                        float(row[self.start_time_index]))
                    self.used_ram_col.append(float(row[self.used_ram_index]))
                    self.iowait_col.append(float(row[self.end_iowait_index]) - \
                        float(row[self.start_iowait_index]))
                    self.steal_time_col.append(float(row[self.end_steal_time_index]) - \
                        float(row[self.start_steal_time_index]))
                    self.one_load_col.append(float(row[self.one_load_index]))
                    self.five_load_col.append(float(row[self.five_load_index]))
                    self.fifteen_load_col.append(float(row[self.fifteen_load_index]))
                except ValueError:
                    # Skips header
                    continue

    def compute_flow_stats(self):
        """Compute CBench flows/second stats."""
        self.compute_generic_stats("flows", self.flows_col)

    def build_flow_graph(self, total_gcount, graph_num):
        """Plot flows/sec data.

        :param total_gcount: Total number of graphs to render.
        :type total_gcount: int
        :param graph_num: Number for this graph, <= total_gcount.
        :type graph_num: int

        """
        self.build_generic_graph(total_gcount, graph_num,
            "Flows per Second", self.flows_col)

    def compute_ram_stats(self):
        """Compute used RAM stats."""
        self.compute_generic_stats("used_ram", self.used_ram_col)

    def build_ram_graph(self, total_gcount, graph_num):
        """Plot used RAM data.

        :param total_gcount: Total number of graphs to render.
        :type total_gcount: int
        :param graph_num: Number for this graph, <= total_gcount.
        :type graph_num: int

        """
        self.build_generic_graph(total_gcount, graph_num,
            "Used RAM (MB)", self.used_ram_col)

    def compute_runtime_stats(self):
        """Compute CBench runtime length stats."""
        self.compute_generic_stats("runtime", self.runtime_col)

    def build_runtime_graph(self, total_gcount, graph_num):
        """Plot CBench runtime length data.

        :paruntime total_gcount: Total number of graphs to render.
        :type total_gcount: int
        :paruntime graph_num: Number for this graph, <= total_gcount.
        :type graph_num: int

        """
        self.build_generic_graph(total_gcount, graph_num,
            "CBench Runtime (sec)", self.runtime_col)

    def compute_generic_stats(self, stats_name, stats_col):
        """Helper for computing generic stats."""
        generic_stats = {}
        generic_stats["min"] = int(numpy.amin(stats_col))
        generic_stats["max"] = int(numpy.amax(stats_col))
        generic_stats["mean"] = round(numpy.mean(stats_col), self.precision)
        generic_stats["stddev"] = round(numpy.std(stats_col), self.precision)
        generic_stats["relstddev"] = round(generic_stats["stddev"] / \
            generic_stats["mean"] * 100, self.precision)
        self.results[stats_name] = generic_stats

    def build_generic_graph(self, total_gcount, graph_num, y_label, data_col):
        """Helper for plotting generic data.

        :pageneric total_gcount: Total number of graphs to render.
        :type total_gcount: int
        :pageneric graph_num: Number for this graph, <= total_gcount.
        :type graph_num: int
        :param y_label: Lable of Y axis.
        :type y_label: string
        :param data_col: Data to graph.
        :type data_col: list

        """
        # Pagenerics are numrows, numcols, fignum
        pyplot.subplot(total_gcount, 1, graph_num)
        # "go" means green O's
        pyplot.plot(self.run_col, data_col, "go")
        pyplot.xlabel("Run Number")
        pyplot.ylabel(y_label)


# Build stats object
stats = Stats()
{'fraction_non_alphanumeric': 0.10706580366774542, 'fraction_numerical': 0.004674577490111471, 'mean_word_length': 2.4125766871165646, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3030369', 'n_tokens_mistral': 3553, 'n_tokens_neox': 3397, 'n_words': 693}