<commit_before>import sys
import re
from time import sleep
import traceback
import datetime
from collections import defaultdict, namedtuple

from elasticsearch_dsl import Search

from . import Reporter, runerror, get_configfile, get_template

# Various config values and their default values
config_vals = {'num_clusters': 100, 'jobs_per_cluster': 1e6,
               'num_hosts_per_site': 1000, 'errors_per_host': 1000,
               'num_failed_sites': 1000}
logfile = 'jobsuccessratereport.log'
default_templatefile = 'template_jobrate.html'
output_time_format = "%Y-%m-%d %H:%M:%S"


@Reporter.init_reporter_parser
def parse_opts(parser):
    """
    Specific argument parser for this report.  The decorator initializes the
    argparse.ArgumentParser object, calls this function on that object to
    modify it, and then returns the Namespace from that object.

    :param parser: argparse.ArgumentParser object that we intend to add to
    :return: None
    """
    # Report-specific args
    parser.add_argument("-E", "--experiment", dest="vo",
                        help="experiment name", type=unicode, required=True)


def sum_errors(dic):
    """Helper function to sum up number of failed jobs per host.
    Assumes that dic is in the form

    :param dict dic: {"error_code1":count1, "error_code2":count2, etc.}
    :return int: Sum of all values in dic
    """
    return sum(value for key, value in dic.iteritems())


class Jobs:
    """Class to assign jobs to sites"""
    def __init__(self):
        self.jobs = defaultdict(list)

    def add_job(self, job):

        """
        Adds job to self.jobs dict

        :param site: OSG site where job ran
        :param job: Job object that contains info about a job
        :return: None
        """
        self.jobs[job.site].append(job)


class Job:
    """Class that holds job information
    :param str end_time: End time of job
    :param str start_time: Start time of job
    :param str jobid: JobID of job
    :param str site: Site where job ran
    :param str host: Host where job ran
    :param str exit__code: Exit code of job
    """
    def __init__(self, end_time, start_time, jobid, site, host, exit__code):
        self.end_time = end_time
        self.start_time = start_time
        self.jobid = jobid
        self.site = site
        self.host = host
        self.exit_code = exit__code


class JobSuccessRateReporter(Reporter):
    """
    Class to hold information about and run Job Success Rate report.  To
    execute, instantiate the class and then use the run_report() method.

    :param config: Report Configuration file
    :param str start: Start time of report range
    :param str end: End time of report range
    :param str vo: Experiment to run report on
    :param str template: Filename of HTML template to generate report
    :param bool is_test: Whether or not this is a test run.
    :param bool verbose: Verbose flag
    :param bool no_email: If true, don't actually send the email
    """
    def __init__(self, config, start, end, vo, template, is_test,
                 verbose, no_email, ov_logfile=None):
        report = 'JobSuccessRate'
        self.vo = vo

        logfile_fname = ov_logfile if ov_logfile is not None else logfile
        logfile_override = True if ov_logfile is not None else False

        self.title = "{0} Production Jobs Success Rate on the OSG Sites " \
                     "({1} - {2})".format(self.vo, start, end)

        super(JobSuccessRateReporter, self).__init__(report, config, start,
                                                     end, verbose,
                                                     althost="landscapeitb",
                                                     index_key="landscape_index_pattern",
                                                     is_test=is_test,
                                                     no_email=no_email,
                                                     logfile=logfile_fname,
                                                     logfile_override=logfile_override,
                                                     check_vo=True)
        self.template = template
        self.run = Jobs()
        self.clusters = defaultdict(lambda: {'userid': None, 'jobs': []})
        self.connectStr = None

        # Patch while we fix gratia probes to include CommonName Field
        self.dnusermatch_CILogon = re.compile('.+CN=UID:(\w+)')
        self.dnusermatch_FNAL = None
        # End patch

        self.usermatch_CILogon = re.compile('.+CN=UID:(\w+)')
        self.usermatch_FNAL = re.compile('.+/(\w+\.fnal\.gov)')
        self.globaljobparts = re.compile('\w+\.(.+\.fnal\.gov)#(\d+\.\d+)#.+')
        self.realhost_pattern = re.compile('\s\(primary\)')
        self.jobpattern = re.compile('(\d+).\d+@(.+\.fnal\.gov)')
        self.text = ''
        self.limit_sites = self._limit_site_check()

    def run_report(self):
        """Method that runs all of the applicable actions in this class."""
        self.generate()
        self.generate_report_file()
        smsg = "Report sent for {0} to {1}".format(
            self.vo,", ".join(self.email_info['to']['email']))
        self.send_report(successmessage=smsg)

    def query(self):
        """
        Method to query Elasticsearch cluster for EfficiencyReport information

        :return elasticsearch_dsl.Search: Search object containing ES query
        """
        # Set up our search parameters
        rep_config = self.config[self.report_type.lower()][self.vo.lower()]
        fqan_string = rep_config['fqan']


        # TODO:  We want starttimeq and endtimeq to be created depending on the field mapping
        # of the relevant time field (CompletionDate here).  We should dynamically search the 
        # mapping and choose whether to use get_epoch_time_range_utc or isoformat()
        # It's going to have something like:
        # self.client.indices.get_field_mapping(fields=['CompletionDate'], index=[self.indexpattern]), 
        # And then parse through the dictionary to get the mapping type
        starttimeq, endtimeq = self.get_epoch_time_range_utc(self.start_time, self.end_time)
        starttimeq, endtimeq = (x / 1000 for x in (starttimeq, endtimeq))
        # starttimeq, endtimeq = (t.isoformat() for t in (self.start_time, self.end_time))

        self.logger.info(self.indexpattern)
        if self.verbose:
            sleep(3)

        # Elasticsearch query
        s = Search(using=self.client, index=self.indexpattern) \
            .filter("range", CompletionDate={"gte": starttimeq, "lt": endtimeq}) \
            .filter("term", JobStatus=4) \
            .exclude("term", JobUniverse=7)     # Exclude DAGS<commit_msg>Made exclusion backward-compatible<commit_after>import sys
import re
from time import sleep
import traceback
import datetime
from collections import defaultdict, namedtuple

from elasticsearch_dsl import Search, Q

from . import Reporter, runerror, get_configfile, get_template

# Various config values and their default values
config_vals = {'num_clusters': 100, 'jobs_per_cluster': 1e6,
               'num_hosts_per_site': 1000, 'errors_per_host': 1000,
               'num_failed_sites': 1000}
logfile = 'jobsuccessratereport.log'
default_templatefile = 'template_jobrate.html'
output_time_format = "%Y-%m-%d %H:%M:%S"


@Reporter.init_reporter_parser
def parse_opts(parser):
    """
    Specific argument parser for this report.  The decorator initializes the
    argparse.ArgumentParser object, calls this function on that object to
    modify it, and then returns the Namespace from that object.

    :param parser: argparse.ArgumentParser object that we intend to add to
    :return: None
    """
    # Report-specific args
    parser.add_argument("-E", "--experiment", dest="vo",
                        help="experiment name", type=unicode, required=True)


def sum_errors(dic):
    """Helper function to sum up number of failed jobs per host.
    Assumes that dic is in the form

    :param dict dic: {"error_code1":count1, "error_code2":count2, etc.}
    :return int: Sum of all values in dic
    """
    return sum(value for key, value in dic.iteritems())


class Jobs:
    """Class to assign jobs to sites"""
    def __init__(self):
        self.jobs = defaultdict(list)

    def add_job(self, job):

        """
        Adds job to self.jobs dict

        :param site: OSG site where job ran
        :param job: Job object that contains info about a job
        :return: None
        """
        self.jobs[job.site].append(job)


class Job:
    """Class that holds job information
    :param str end_time: End time of job
    :param str start_time: Start time of job
    :param str jobid: JobID of job
    :param str site: Site where job ran
    :param str host: Host where job ran
    :param str exit__code: Exit code of job
    """
    def __init__(self, end_time, start_time, jobid, site, host, exit__code):
        self.end_time = end_time
        self.start_time = start_time
        self.jobid = jobid
        self.site = site
        self.host = host
        self.exit_code = exit__code


class JobSuccessRateReporter(Reporter):
    """
    Class to hold information about and run Job Success Rate report.  To
    execute, instantiate the class and then use the run_report() method.

    :param config: Report Configuration file
    :param str start: Start time of report range
    :param str end: End time of report range
    :param str vo: Experiment to run report on
    :param str template: Filename of HTML template to generate report
    :param bool is_test: Whether or not this is a test run.
    :param bool verbose: Verbose flag
    :param bool no_email: If true, don't actually send the email
    """
    def __init__(self, config, start, end, vo, template, is_test,
                 verbose, no_email, ov_logfile=None):
        report = 'JobSuccessRate'
        self.vo = vo

        logfile_fname = ov_logfile if ov_logfile is not None else logfile
        logfile_override = True if ov_logfile is not None else False

        self.title = "{0} Production Jobs Success Rate on the OSG Sites " \
                     "({1} - {2})".format(self.vo, start, end)

        super(JobSuccessRateReporter, self).__init__(report, config, start,
                                                     end, verbose,
                                                     althost="landscapeitb",
                                                     index_key="landscape_index_pattern",
                                                     is_test=is_test,
                                                     no_email=no_email,
                                                     logfile=logfile_fname,
                                                     logfile_override=logfile_override,
                                                     check_vo=True)
        self.template = template
        self.run = Jobs()
        self.clusters = defaultdict(lambda: {'userid': None, 'jobs': []})
        self.connectStr = None

        # Patch while we fix gratia probes to include CommonName Field
        self.dnusermatch_CILogon = re.compile('.+CN=UID:(\w+)')
        self.dnusermatch_FNAL = None
        # End patch

        self.usermatch_CILogon = re.compile('.+CN=UID:(\w+)')
        self.usermatch_FNAL = re.compile('.+/(\w+\.fnal\.gov)')
        self.globaljobparts = re.compile('\w+\.(.+\.fnal\.gov)#(\d+\.\d+)#.+')
        self.realhost_pattern = re.compile('\s\(primary\)')
        self.jobpattern = re.compile('(\d+).\d+@(.+\.fnal\.gov)')
        self.text = ''
        self.limit_sites = self._limit_site_check()

    def run_report(self):
        """Method that runs all of the applicable actions in this class."""
        self.generate()
        self.generate_report_file()
        smsg = "Report sent for {0} to {1}".format(
            self.vo,", ".join(self.email_info['to']['email']))
        self.send_report(successmessage=smsg)

    def query(self):
        """
        Method to query Elasticsearch cluster for EfficiencyReport information

        :return elasticsearch_dsl.Search: Search object containing ES query
        """
        # Set up our search parameters
        rep_config = self.config[self.report_type.lower()][self.vo.lower()]
        fqan_string = rep_config['fqan']


        # TODO:  We want starttimeq and endtimeq to be created depending on the field mapping
        # of the relevant time field (CompletionDate here).  We should dynamically search the 
        # mapping and choose whether to use get_epoch_time_range_utc or isoformat()
        # It's going to have something like:
        # self.client.indices.get_field_mapping(fields=['CompletionDate'], index=[self.indexpattern]), 
        # And then parse through the dictionary to get the mapping type
        starttimeq, endtimeq = self.get_epoch_time_range_utc(self.start_time, self.end_time)
        starttimeq, endtimeq = (x / 1000 for x in (starttimeq, endtimeq))
        # starttimeq, endtimeq = (t.isoformat() for t in (self.start_time, self.end_time))

        self.logger.info(self.indexpattern)
        if self.verbose:
            sleep(3)

        # Elasticsearch query
        s = Search(using=self.client, index=self.indexpattern) \
            .filter("range", CompletionDate={"gte": starttimeq, "lt": endtimeq}) \
            .filter("term", JobStatus=4) \
            .query("bool", filter=[~Q("term", JobUniverse=7)])  # Exclude DAGS
            # .exclude("term", JobUniverse=7)     # Once djw repo elasticsearch-dsl is upgraded to 6.0.0

{'fraction_non_alphanumeric': 0.097412145272754, 'fraction_numerical': 0.0051463020144096455, 'mean_word_length': 2.2809937288953206, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3745202', 'n_tokens_mistral': 3910, 'n_tokens_neox': 3745, 'n_words': 1191}