<commit_before>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str

import pickle
import numpy as np
import matplotlib as mpl
mpl.use('Agg')
from matplotlib import pyplot as plt
from datetime import datetime

from indra.util.get_version import get_git_info
from indra.tools.reading.util.reporter import Reporter


class DbAwsStatReporter(Reporter):
    """A class to handle generating the reports made at the end of a job.

    Parameters
    ----------
    job_name : str
        The name of the job about which this report is being produced.
    s3_log_prefix: str
        The desired prefix location in which the information about the job is
        stored on s3.
    s3 : boto3.client('s3') instance
        An s3 client from boto3.
    bucket_name : str
        The name of the root s3 bucket.
    """
    def __init__(self, job_name, s3_log_prefix, s3, bucket_name):
        super(DbAwsStatReporter, self).__init__('%s_summary' % job_name)
        self.s3_prefix = s3_log_prefix + 'statistics/'
        self.bucket_name = bucket_name
        self.s3 = s3
        self.summary_dict = {}
        self.hist_dict = {}
        self.set_title("Report of Database Reading Batch Job")
        self.sections = {'Summary Statistics': [], 'Plots': [], 'Git Info': [],
                         'Job Info': []}
        self._make_job_line('Job Name', job_name)
        self._make_job_line('Job s3 prefix', s3_log_prefix)
        self._get_git_info()
        self.set_section_order(['Job Info', 'Git Info', 'Summary Statistics',
                                'Plots'])
        return

    def _get_git_info(self):
        """Stash the git info in a text file."""
        git_info_dict = get_git_info()
        text_file_content = ''
        for key, val in git_info_dict.items():
            label = key.replace('_', ' ').capitalize()
            text_file_content += '%s: %s\n' % (label, val)
            self.add_text('%s: %s' % (label, val), section='Git Info')
        self.s3.put_object(Key=self.s3_prefix + 'git_info.txt',
                           Body=text_file_content, Bucket=self.bucket_name)
        return

    def _plot_hist(self, agged, agg_over, data):
        """Make and stash a figure for histogram-like data."""
        fig = plt.figure()
        plt.hist(data, bins=np.arange(len(data)))
        plt.xlabel('Number of %s for %s' % (agged, agg_over))
        plt.ylabel('Number of %s with a given number of %s' % (agg_over, agged))
        fname = '%s_per_%s.png' % (agged, agg_over)
        fig.set_size_inches(6, 4)
        fig.savefig(fname)
        with open(fname, 'rb') as f:
            s3_key = self.s3_prefix + fname
            self.s3.put_object(Key=s3_key, Body=f.read(),
                               Bucket=self.bucket_name)
        self.add_image(fname, width=6, height=4, section='Plots')
        return

    def _make_timing_report(self, starts, ends):
        """Stash a text file with the timings: start, end, and duration."""
        # Report on the timing
        timing_str = ''
        for step in ['reading', 'statement production', 'stats']:
            time_taken = ends[step] - starts[step]
            timing_str += ('%22s: start: %s, end: %s, duration: %s\n'
                           % (step, starts[step], ends[step], time_taken))
<commit_msg>Improve histgrams.<commit_after>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str

import pickle
import numpy as np
import matplotlib as mpl
mpl.use('Agg')
from matplotlib import pyplot as plt
from datetime import datetime

from indra.util.get_version import get_git_info
from indra.tools.reading.util.reporter import Reporter


class DbAwsStatReporter(Reporter):
    """A class to handle generating the reports made at the end of a job.

    Parameters
    ----------
    job_name : str
        The name of the job about which this report is being produced.
    s3_log_prefix: str
        The desired prefix location in which the information about the job is
        stored on s3.
    s3 : boto3.client('s3') instance
        An s3 client from boto3.
    bucket_name : str
        The name of the root s3 bucket.
    """
    def __init__(self, job_name, s3_log_prefix, s3, bucket_name):
        super(DbAwsStatReporter, self).__init__('%s_summary' % job_name)
        self.s3_prefix = s3_log_prefix + 'statistics/'
        self.bucket_name = bucket_name
        self.s3 = s3
        self.summary_dict = {}
        self.hist_dict = {}
        self.set_title("Report of Database Reading Batch Job")
        self.sections = {'Summary Statistics': [], 'Plots': [], 'Git Info': [],
                         'Job Info': []}
        self._make_job_line('Job Name', job_name)
        self._make_job_line('Job s3 prefix', s3_log_prefix)
        self._get_git_info()
        self.set_section_order(['Job Info', 'Git Info', 'Summary Statistics',
                                'Plots'])
        return

    def _get_git_info(self):
        """Stash the git info in a text file."""
        git_info_dict = get_git_info()
        text_file_content = ''
        for key, val in git_info_dict.items():
            label = key.replace('_', ' ').capitalize()
            text_file_content += '%s: %s\n' % (label, val)
            self.add_text('%s: %s' % (label, val), section='Git Info')
        self.s3.put_object(Key=self.s3_prefix + 'git_info.txt',
                           Body=text_file_content, Bucket=self.bucket_name)
        return

    def _plot_hist(self, agged, agg_over, data):
        """Make and stash a figure for histogram-like data."""
        fig = plt.figure()

        # Check if it's worth actually making a histogram...
        if not data:
            self.add_text('No data for %s per %s.' % (agged, agg_over),
                          section='Plots')
            return
        if len(set(data)) == 1:
            self.add_text('%s per %s: %d' % (agged, agg_over, data[0]),
                          section='Plots')
            return

        # Make the histogram.
        plt.hist(data, bins=np.arange(len(data)), log=True)
        plt.xlabel('Number of %s for %s' % (agged, agg_over))
        plt.ylabel('Number of %s with a given number of %s' % (agg_over, agged))
        fname = '%s_per_%s.png' % (agged, agg_over)
        fig.set_size_inches(6, 4)
        fig.savefig(fname)
        with open(fname, 'rb') as f:
            s3_key = self.s3_prefix + fname
            self.s3.put_object(Key=s3_key, Body=f.read(),
                               Bucket=self.bucket_name)
        self.add_image(fname, width=6, height=4, section='Plots')
        return

    def _make_timing_report(self, starts, ends):
        """Stash a text file with the timings: start, end, and duration."""
        # Report on the timing
        timing_str = ''
        for step in ['reading', 'statement production', 'stats']:
            time_taken = ends[step] - starts[step]
            timing_str += ('%22s: start: %s, end: %s, duration: %s\n'
                           % (step, starts[step], ends[step], time_taken))

{'fraction_non_alphanumeric': 0.11637385804638088, 'fraction_numerical': 0.008151791988756148, 'mean_word_length': 2.27926267281106, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6605977', 'n_tokens_mistral': 2266, 'n_tokens_neox': 2203, 'n_words': 618}