<commit_before>import os
import time

try:
    from collections import defaultdict
except ImportError:
    from compat.defaultdict import defaultdict

from datetime import datetime
from glob import glob

try:
    from xml.etree.ElementTree import ElementTree
except ImportError:
    from elementtree import ElementTree

from util import get_hostname
from checks import AgentCheck

class Jenkins(AgentCheck):
    datetime_format = '%Y-%m-%d_%H-%M-%S'

    def __init__(self, name, init_config, agentConfig):
        AgentCheck.__init__(self, name, init_config, agentConfig)
        self.high_watermarks = {}

    def _extract_timestamp(self, dir_name):
        try:
            # Parse the timestamp from the directory name
            date_str = os.path.basename(dir_name)
            time_tuple = time.strptime(date_str, self.datetime_format)
            return time.mktime(time_tuple)
        except ValueError:
            raise Exception("Error with build directory name, not a parsable date: %s" % (dir_name))

    def _get_build_metadata(self, dir_name):
        # Read the build.xml metadata file that Jenkins generates
        build_metadata = os.path.join(dir_name, 'build.xml')

        if not os.access(build_metadata, os.R_OK):
            self.log.error("Can't read build file at %s" % (build_metadata))
            raise Exception("Can't read build file at %s" % (build_metadata))
        else:
            tree = ElementTree()
            tree.parse(build_metadata)

            keys = ['result', 'number', 'duration']

            kv_pairs = ((k, tree.find(k)) for k in keys)
            d = dict([(k, v.text)
                        for k, v in kv_pairs
                        if v is not None])
            return d

    def _get_build_results(self, instance_key, job_dir):
        job_name = os.path.basename(job_dir)

        try:
            dirs = glob(os.path.join(job_dir, 'builds', '*_*'))
            if len(dirs) > 0:
                dirs = sorted(dirs, reverse=True)
                # We try to get the last valid build
                for index in xrange(0, len(dirs) - 1):
                    dir_name = dirs[index]
                    timestamp = self._extract_timestamp(dir_name)
                    # Check if it's a new build
                    if timestamp > self.high_watermarks[instance_key][job_name]:
                        # If we can't get build metadata, we try the previous one
                        try:
                            build_metadata = self._get_build_metadata(dir_name)
                        except Exception:
                            continue

                        output = {
                                'job_name':     job_name,
                                'timestamp':    timestamp,
                                'event_type':   'build result'
                            }
                        output.update(build_metadata)
                        self.high_watermarks[instance_key][job_name] = timestamp
                        yield output
                    # If it not a new build, stop here
                    else:
                        break
        except Exception, e:
            self.log.error("Error while working on job %s, exception: %s" % (job_name, e))

    def check(self, instance, create_event=True):
        if self.high_watermarks.get(instance.get('name'), None) is None:
            # On the first run of check(), prime the high_watermarks dict
            # so that we only send events that occured after the agent
            # started.
            # (Setting high_watermarks in the next statement prevents
            #  any kind of infinite loop (assuming nothing ever sets
            #  high_watermarks to None again!))
            self.high_watermarks[instance.get('name')] = defaultdict(lambda: 0)
            self.check(instance, create_event=False)

        jenkins_home = instance.get('jenkins_home', None)

        if not jenkins_home:
            raise Exception("No jenkins_home directory set in the config file")

        job_dirs = glob(os.path.join(jenkins_home, 'jobs', '*'))

        build_events = []

        for job_dir in job_dirs:
            for output in self._get_build_results(instance.get('name'), job_dir):
                output['api_key'] = self.agentConfig['api_key']
                output['host'] = get_hostname(self.agentConfig)
                if create_event:
                    log.debug("Creating event for job: %s" % output['job_name'])
                    self.event(output)

    @staticmethod
    def parse_agent_config(agentConfig):
        if not agentConfig.get('hudson_home'):
            return False

        return {
            'instances': [{
                'name': 'default',
                'jenkins_home': agentConfig.get('hudson_home'),
            }]
        }

<commit_msg>Fix the logging for jenkins check<commit_after>import os
import time

try:
    from collections import defaultdict
except ImportError:
    from compat.defaultdict import defaultdict

from datetime import datetime
from glob import glob

try:
    from xml.etree.ElementTree import ElementTree
except ImportError:
    from elementtree import ElementTree

from util import get_hostname
from checks import AgentCheck

class Jenkins(AgentCheck):
    datetime_format = '%Y-%m-%d_%H-%M-%S'

    def __init__(self, name, init_config, agentConfig):
        AgentCheck.__init__(self, name, init_config, agentConfig)
        self.high_watermarks = {}

    def _extract_timestamp(self, dir_name):
        try:
            # Parse the timestamp from the directory name
            date_str = os.path.basename(dir_name)
            time_tuple = time.strptime(date_str, self.datetime_format)
            return time.mktime(time_tuple)
        except ValueError:
            raise Exception("Error with build directory name, not a parsable date: %s" % (dir_name))

    def _get_build_metadata(self, dir_name):
        # Read the build.xml metadata file that Jenkins generates
        build_metadata = os.path.join(dir_name, 'build.xml')

        if not os.access(build_metadata, os.R_OK):
            self.log.debug("Can't read build file at %s" % (build_metadata))
            raise Exception("Can't read build file at %s" % (build_metadata))
        else:
            tree = ElementTree()
            tree.parse(build_metadata)

            keys = ['result', 'number', 'duration']

            kv_pairs = ((k, tree.find(k)) for k in keys)
            d = dict([(k, v.text)
                        for k, v in kv_pairs
                        if v is not None])
            return d

    def _get_build_results(self, instance_key, job_dir):
        job_name = os.path.basename(job_dir)

        try:
            dirs = glob(os.path.join(job_dir, 'builds', '*_*'))
            if len(dirs) > 0:
                dirs = sorted(dirs, reverse=True)
                # We try to get the last valid build
                for index in xrange(0, len(dirs) - 1):
                    dir_name = dirs[index]
                    timestamp = self._extract_timestamp(dir_name)
                    # Check if it's a new build
                    if timestamp > self.high_watermarks[instance_key][job_name]:
                        # If we can't get build metadata, we try the previous one
                        try:
                            build_metadata = self._get_build_metadata(dir_name)
                        except Exception:
                            continue

                        output = {
                                'job_name':     job_name,
                                'timestamp':    timestamp,
                                'event_type':   'build result'
                            }
                        output.update(build_metadata)
                        self.high_watermarks[instance_key][job_name] = timestamp
                        yield output
                    # If it not a new build, stop here
                    else:
                        break
        except Exception, e:
            self.log.error("Error while working on job %s, exception: %s" % (job_name, e))

    def check(self, instance, create_event=True):
        if self.high_watermarks.get(instance.get('name'), None) is None:
            # On the first run of check(), prime the high_watermarks dict
            # so that we only send events that occured after the agent
            # started.
            # (Setting high_watermarks in the next statement prevents
            #  any kind of infinite loop (assuming nothing ever sets
            #  high_watermarks to None again!))
            self.high_watermarks[instance.get('name')] = defaultdict(lambda: 0)
            self.check(instance, create_event=False)

        jenkins_home = instance.get('jenkins_home', None)

        if not jenkins_home:
            raise Exception("No jenkins_home directory set in the config file")

        job_dirs = glob(os.path.join(jenkins_home, 'jobs', '*'))

        build_events = []

        for job_dir in job_dirs:
            for output in self._get_build_results(instance.get('name'), job_dir):
                output['api_key'] = self.agentConfig['api_key']
                output['host'] = get_hostname(self.agentConfig)
                if create_event:
                    self.log.debug("Creating event for job: %s" % output['job_name'])
                    self.event(output)

    @staticmethod
    def parse_agent_config(agentConfig):
        if not agentConfig.get('hudson_home'):
            return False

        return {
            'instances': [{
                'name': 'default',
                'jenkins_home': agentConfig.get('hudson_home'),
            }]
        }


{'fraction_non_alphanumeric': 0.08653153616627225, 'fraction_numerical': 0.0008231299516411153, 'mean_word_length': 1.7496463932107496, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '765218', 'n_tokens_mistral': 2640, 'n_tokens_neox': 2532, 'n_words': 721}