<commit_before>import sys
import traceback
import re

from elasticsearch_dsl import Search

from . import Reporter, runerror, get_configfile, get_template, coroutine
from . import TextUtils, NiceNum

default_templatefile = 'template_topwastedhoursvo.html'
logfile = 'topwastedhoursvo.log'
perc_cutoff = 0.5
hours_cutoff = 1000


@Reporter.init_reporter_parser
def parse_opts(parser):
    """
    Specific argument parser for this report.  The decorator initializes the
    argparse.ArgumentParser object, calls this function on that object to
    modify it, and then returns the Namespace from that object.

    :param parser: argparse.ArgumentParser object that we intend to add to
    :return: None
    """
    parser.add_argument("-E", "--experiment", dest="vo",
                        help="experiment name", default=None, required=True)
    parser.add_argument("-F", "--facility", dest="facility",
                        help="facility name/host description", default=None, required=True)
    parser.add_argument("-N", "--numrank", dest="numrank",
                        help="Number of Users to rank",
                        default=100, type=int)


class User:
    """
    Holds all user-specific information for this report

    :param str user_name: username of user (or DN if username couldn't
    be determined)
    """

    def __init__(self, user_name):
        self.user = user_name

        # When filled, these both will be of the form
        # {'Njobs': <value>, 'CoreHours': <value>}
        self.success = {}
        self.failure = {}

        self.total_Njobs = 0
        self.total_CoreHours = 0

    @staticmethod
    def _check_datadict(data):
        """
        Data validation for incoming data dictionary
        :param dict data: Dict of the form
        {'Status': [Success|Failure], ['Njobs'|'CoreHours']: <value>}

        :return bool: True if data passes test
        """
        return 'Status' in data and ('Njobs' in data or 'CoreHours' in data)

    def add_data(self, datadict):
        """
        Adds to the success or failure dict for User instance

        :param dict datadict:  Dictionary of data that satsifies
            self._check_datadict(datadict) == True
        :return None:
        """
        if self._check_datadict(datadict):
            rawattr = datadict['Status'].lower()
            selfrawattr = getattr(self, rawattr)
            del datadict['Status']
            for key in datadict:
                selfrawattr[key] = datadict[key]

                # Update the appropriate total
                totalkey = 'total_{0}'.format(key)
                setattr(self, totalkey, getattr(self, totalkey) + datadict[key])
        else:
            raise ValueError("Improper format for data passing.  "
                             "Must be a dict with keys Status, and (Njobs or CoreHours)")

    def get_job_failure_percent(self):
        """
        Calculates the failure rate of a user's jobs
        (Njobs failed / Njobs total)

        :return float: Failure rate as a percentage (not decimal)
        """
        failure_rate = 0
        if self.total_Njobs > 0:
            failure_rate = (self.failure['Njobs'] / self.total_Njobs) * 100.
        return failure_rate

    def get_wasted_hours_percent(self):
        """
        Gets a user's wasted hours percentage
        (Failed CoreHours/ Total CoreHours)

        :return float: Wasted Hours as a percentage of total hours (not decimal)
        """
        waste_per = 0
        if self.total_CoreHours > 0:
            waste_per = (self.failure['CoreHours'] / self.total_CoreHours) * 100.
        return waste_per


class TopWastedHoursReport(Reporter):
    """
    Class to hold information about and run Top Wasted Hours report.
    :param str config: Report Configuration file
    :param str start: Start time of report range
    :param str end: End time of report range
    :param str template: Filename of HTML template to generate report
    :param str vo: VO we want to filter this report on
    :param int numrank: How many entries that match other filters we want to show
    :param str facility: Host description in GRACC (e.g. GPGrid)
    :param bool is_test: Whether or not this is a test run.
    :param bool verbose: Verbose flag
    :param bool no_email: If true, don't actually send the email
    :param str ov_logfile: Path to override logfile
    """
    def __init__(self, config, start, end, template, vo,
                 numrank=100, facility=None, is_test=True,
                 verbose=False, no_email=False, ov_logfile=None):
        report = 'TopWastedHoursVO'
        self.vo = vo

        if ov_logfile:
            rlogfile = ov_logfile
            logfile_override = True
        else:
            rlogfile = logfile
            logfile_override = False

        Reporter.__init__(self, report, config, start, end=end,
                          verbose=verbose, is_test=is_test,
                          no_email=no_email, logfile=rlogfile,
                          logfile_override=logfile_override, check_vo=True)

        self.template = template
        self.facility = facility
        self.numrank = numrank
        self._get_configfile_limits()
        self.users = {}
        self.text = ''
        self.title = "Top {0} Users in {1} Ranked by Percent Wasted Hours on {2} " \
                     "({3:%Y-%m-%d %H:%M} - {4:%Y-%m-%d %H:%M})".format(
                        self.numrank,
                        self.vo,
                        self.facility,
                        self.start_time,
                        self.end_time)
        self.dnusermatch_CILogon = re.compile('.+CN=UID:(\w+)$')
        self.dnusermatch_FNAL = re.compile('.+\/(.+\.fnal\.gov)$')

    def run_report(self):
        """Higher-level method to run all the other methods in report
        generation"""
        self.generate()
        self.generate_report_file()
        self.send_report()
        return

    def _get_configfile_limits(self):
        """Get limits from config file"""
        for attr in ('hours_cutoff', 'perc_cutoff'):
            try:
                value = self.config[self.vo.lower()][self.report_type.lower()][attr]
            except KeyError:
                value = globals()[attr]
                self.logger.warning('Could not find value for attribute {0}'
                                    ' in config file.  Will use module '
                                    'default of {1}'.format(attr, value))
            finally:
                setattr(self, attr, value)

    def query(self):
        """
        Method to query Elasticsearch cluster for this report's information

        :return elasticsearch_dsl.Search: Search object containing ES query
        """
        wildcardProbeNameq = 'condor:fifebatch?.fnal.gov'

        starttimeq = self.start_time.isoformat()
        endtimeq = self.end_time.isoformat()

        s = Search(using=self.client, index=self.indexpattern) \
            .filter("wildcard", ProbeName=wildcardProbeNameq) \
            .filter("range", EndTime={"gte": starttimeq, "lt": endtimeq}) \
            .filter("term", Host_description=self.facility) \
            .filter("term", VOName=self.vo) \
            .filter("term", ResourceType="Payload") \
            [0:0]   # Only print aggregations

        # Aggregations

        Buckets = s.aggs.bucket('DN', 'terms', field='DN', size=2**31-1) \
            .bucket('Status', 'filters', filters={
                'Success': {'bool': {'must': {'term': {'Resource_ExitCode': 0}}}},
                'Failure': {'bool': {'must_not': {'term': {'Resource_ExitCode': 0}}}}})
<commit_msg>Make sure VO is lowercase in ES query<commit_after>import sys
import traceback
import re

from elasticsearch_dsl import Search

from . import Reporter, runerror, get_configfile, get_template, coroutine
from . import TextUtils, NiceNum

default_templatefile = 'template_topwastedhoursvo.html'
logfile = 'topwastedhoursvo.log'
perc_cutoff = 0.5
hours_cutoff = 1000


@Reporter.init_reporter_parser
def parse_opts(parser):
    """
    Specific argument parser for this report.  The decorator initializes the
    argparse.ArgumentParser object, calls this function on that object to
    modify it, and then returns the Namespace from that object.

    :param parser: argparse.ArgumentParser object that we intend to add to
    :return: None
    """
    parser.add_argument("-E", "--experiment", dest="vo",
                        help="experiment name", default=None, required=True)
    parser.add_argument("-F", "--facility", dest="facility",
                        help="facility name/host description", default=None,
                        required=True)
    parser.add_argument("-N", "--numrank", dest="numrank",
                        help="Number of Users to rank",
                        default=100, type=int)


class User:
    """
    Holds all user-specific information for this report

    :param str user_name: username of user (or DN if username couldn't
    be determined)
    """

    def __init__(self, user_name):
        self.user = user_name

        # When filled, these both will be of the form
        # {'Njobs': <value>, 'CoreHours': <value>}
        self.success = {}
        self.failure = {}

        self.total_Njobs = 0
        self.total_CoreHours = 0

    @staticmethod
    def _check_datadict(data):
        """
        Data validation for incoming data dictionary
        :param dict data: Dict of the form
        {'Status': [Success|Failure], ['Njobs'|'CoreHours']: <value>}

        :return bool: True if data passes test
        """
        return 'Status' in data and ('Njobs' in data or 'CoreHours' in data)

    def add_data(self, datadict):
        """
        Adds to the success or failure dict for User instance

        :param dict datadict:  Dictionary of data that satsifies
            self._check_datadict(datadict) == True
        :return None:
        """
        if self._check_datadict(datadict):
            rawattr = datadict['Status'].lower()
            selfrawattr = getattr(self, rawattr)
            del datadict['Status']
            for key in datadict:
                selfrawattr[key] = datadict[key]

                # Update the appropriate total
                totalkey = 'total_{0}'.format(key)
                setattr(self, totalkey, getattr(self, totalkey) + datadict[key])
        else:
            raise ValueError("Improper format for data passing.  "
                             "Must be a dict with keys Status, and (Njobs or CoreHours)")

    def get_job_failure_percent(self):
        """
        Calculates the failure rate of a user's jobs
        (Njobs failed / Njobs total)

        :return float: Failure rate as a percentage (not decimal)
        """
        failure_rate = 0
        if self.total_Njobs > 0:
            failure_rate = (self.failure['Njobs'] / self.total_Njobs) * 100.
        return failure_rate

    def get_wasted_hours_percent(self):
        """
        Gets a user's wasted hours percentage
        (Failed CoreHours/ Total CoreHours)

        :return float: Wasted Hours as a percentage of total hours (not decimal)
        """
        waste_per = 0
        if self.total_CoreHours > 0:
            waste_per = (self.failure['CoreHours'] / self.total_CoreHours) * 100.
        return waste_per


class TopWastedHoursReport(Reporter):
    """
    Class to hold information about and run Top Wasted Hours report.
    :param str config: Report Configuration file
    :param str start: Start time of report range
    :param str end: End time of report range
    :param str template: Filename of HTML template to generate report
    :param str vo: VO we want to filter this report on
    :param int numrank: How many entries that match other filters we want to show
    :param str facility: Host description in GRACC (e.g. GPGrid)
    :param bool is_test: Whether or not this is a test run.
    :param bool verbose: Verbose flag
    :param bool no_email: If true, don't actually send the email
    :param str ov_logfile: Path to override logfile
    """
    def __init__(self, config, start, end, template, vo,
                 numrank=100, facility=None, is_test=True,
                 verbose=False, no_email=False, ov_logfile=None):
        report = 'TopWastedHoursVO'
        self.vo = vo

        if ov_logfile:
            rlogfile = ov_logfile
            logfile_override = True
        else:
            rlogfile = logfile
            logfile_override = False

        Reporter.__init__(self, report, config, start, end=end,
                          verbose=verbose, is_test=is_test,
                          no_email=no_email, logfile=rlogfile,
                          logfile_override=logfile_override, check_vo=True)

        self.template = template
        self.facility = facility
        self.numrank = numrank
        self._get_configfile_limits()
        self.users = {}
        self.text = ''
        self.title = "Top {0} Users in {1} Ranked by Percent Wasted Hours on {2} " \
                     "({3:%Y-%m-%d %H:%M} - {4:%Y-%m-%d %H:%M})".format(
                        self.numrank,
                        self.vo,
                        self.facility,
                        self.start_time,
                        self.end_time)
        self.dnusermatch_CILogon = re.compile('.+CN=UID:(\w+)$')
        self.dnusermatch_FNAL = re.compile('.+\/(.+\.fnal\.gov)$')

    def run_report(self):
        """Higher-level method to run all the other methods in report
        generation"""
        self.generate()
        self.generate_report_file()
        self.send_report()
        return

    def _get_configfile_limits(self):
        """Get limits from config file"""
        for attr in ('hours_cutoff', 'perc_cutoff'):
            try:
                value = self.config[self.vo.lower()][self.report_type.lower()][attr]
            except KeyError:
                value = globals()[attr]
                self.logger.warning('Could not find value for attribute {0}'
                                    ' in config file.  Will use module '
                                    'default of {1}'.format(attr, value))
            finally:
                setattr(self, attr, value)

    def query(self):
        """
        Method to query Elasticsearch cluster for this report's information

        :return elasticsearch_dsl.Search: Search object containing ES query
        """
        wildcardProbeNameq = 'condor:fifebatch?.fnal.gov'

        starttimeq = self.start_time.isoformat()
        endtimeq = self.end_time.isoformat()

        s = Search(using=self.client, index=self.indexpattern) \
            .filter("wildcard", ProbeName=wildcardProbeNameq) \
            .filter("range", EndTime={"gte": starttimeq, "lt": endtimeq}) \
            .filter("term", Host_description=self.facility) \
            .filter("term", VOName=self.vo.lower()) \
            .filter("term", ResourceType="Payload") \
            [0:0]   # Only print aggregations

        # Aggregations

        Buckets = s.aggs.bucket('DN', 'terms', field='DN', size=2**31-1) \
            .bucket('Status', 'filters', filters={
                'Success': {'bool': {'must': {'term': {'Resource_ExitCode': 0}}}},
                'Failure': {'bool': {'must_not': {'term': {'Resource_ExitCode': 0}}}}})

{'fraction_non_alphanumeric': 0.10342358760739391, 'fraction_numerical': 0.005206977349648529, 'mean_word_length': 2.098406936882436, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6607468', 'n_tokens_mistral': 4471, 'n_tokens_neox': 4266, 'n_words': 1265}