<commit_before>#
# Description:
#   This module implements the functions needed
#   to monitor the glidein factory
#
# Author:
#   <NAME> (Dec 11th 2006)
#

import os,os.path
import re,time,copy,string,math,random,fcntl
import xmlFormat,timeConversion
import rrdSupport

############################################################
#
# Configuration
#
############################################################

class MonitoringConfig:
    def __init__(self):
        # set default values
        # user should modify if needed
        self.rrd_step=300       #default to 5 minutes
        self.rrd_heartbeat=1800 #default to 30 minutes, should be at least twice the loop time
        self.rrd_ds_name="val"
        self.rrd_archives=[('AVERAGE',0.8,1,740),      # max precision, keep 2.5 days
                           ('AVERAGE',0.92,12,740),       # 1 h precision, keep for a month (30 days)
                           ('AVERAGE',0.98,144,740)        # 12 hour precision, keep for a year
                           ]

        # The name of the attribute that identifies the glidein
        self.monitor_dir="monitor/"
        self.log_dir="log/"

        self.rrd_obj=rrdSupport.rrdSupport()

        self.my_name="Unknown"

    def logCompleted(self,entered_dict):
        now=time.time()

        job_ids=entered_dict.keys()
        if len(job_ids)==0:
            return # nothing to do
        job_ids.sort()
        
        relative_fname="completed_jobs_%s.log"%time.strftime("%Y%m%d",time.localtime(now))
        fname=os.path.join(self.log_dir,relative_fname)
        fd=open(fname,"a")
        try:
            for job_id in job_ids:
                el=entered_dict[job_id]
                jobs_duration=el['jobs_duration']
                waste_mill=el['wastemill']
                fd.write(("<job %37s %17s %17s %22s>"%(('terminated="%s"'%timeConversion.getISO8601_Local(now)),
                                                       ('id="%s"'%job_id),
                                                       ('duration="%i"'%el['duration']),
                                                       ('condor_started="%s"'%(el['condor_started']==True))))+
                         ("<user %14s %17s %16s %19s/>"%(('jobsnr="%i"'%el['jobsnr']),
                                                         ('duration="%i"'%jobs_duration['total']),
                                                         ('goodput="%i"'%jobs_duration['goodput']),
                                                         ('terminated="%i"'%jobs_duration['terminated'])))+
                         ("<wastemill %17s %11s %16s %13s/></job>\n"%(('validation="%i"'%waste_mill['validation']),
                                                                      ('idle="%i"'%waste_mill['idle']),
                                                                      ('nosuccess="%i"'%waste_mill['nosuccess']),
                                                                      ('badput="%i"'%waste_mill['badput']))))
        finally:
            fd.close()

    def write_file(self,relative_fname,str):
        fname=os.path.join(self.monitor_dir,relative_fname)
        #print "Writing "+fname
        fd=open(fname+".tmp","w")
        try:<commit_msg>Add condor_duration to the completed log file<commit_after>#
# Description:
#   This module implements the functions needed
#   to monitor the glidein factory
#
# Author:
#   <NAME> (Dec 11th 2006)
#

import os,os.path
import re,time,copy,string,math,random,fcntl
import xmlFormat,timeConversion
import rrdSupport

############################################################
#
# Configuration
#
############################################################

class MonitoringConfig:
    def __init__(self):
        # set default values
        # user should modify if needed
        self.rrd_step=300       #default to 5 minutes
        self.rrd_heartbeat=1800 #default to 30 minutes, should be at least twice the loop time
        self.rrd_ds_name="val"
        self.rrd_archives=[('AVERAGE',0.8,1,740),      # max precision, keep 2.5 days
                           ('AVERAGE',0.92,12,740),       # 1 h precision, keep for a month (30 days)
                           ('AVERAGE',0.98,144,740)        # 12 hour precision, keep for a year
                           ]

        # The name of the attribute that identifies the glidein
        self.monitor_dir="monitor/"
        self.log_dir="log/"

        self.rrd_obj=rrdSupport.rrdSupport()

        self.my_name="Unknown"

    def logCompleted(self,entered_dict):
        now=time.time()

        job_ids=entered_dict.keys()
        if len(job_ids)==0:
            return # nothing to do
        job_ids.sort()
        
        relative_fname="completed_jobs_%s.log"%time.strftime("%Y%m%d",time.localtime(now))
        fname=os.path.join(self.log_dir,relative_fname)
        fd=open(fname,"a")
        try:
            for job_id in job_ids:
                el=entered_dict[job_id]
                jobs_duration=el['jobs_duration']
                waste_mill=el['wastemill']
                fd.write(("<job %37s %17s %17s %22s %24s>"%(('terminated="%s"'%timeConversion.getISO8601_Local(now)),
                                                            ('id="%s"'%job_id),
                                                            ('duration="%i"'%el['duration']),
                                                            ('condor_started="%s"'%(el['condor_started']==True)),
                                                            ('condor_duration="%i"'%el['condor_duration'])))+
                         ("<user %14s %17s %16s %19s/>"%(('jobsnr="%i"'%el['jobsnr']),
                                                         ('duration="%i"'%jobs_duration['total']),
                                                         ('goodput="%i"'%jobs_duration['goodput']),
                                                         ('terminated="%i"'%jobs_duration['terminated'])))+
                         ("<wastemill %17s %11s %16s %13s/></job>\n"%(('validation="%i"'%waste_mill['validation']),
                                                                      ('idle="%i"'%waste_mill['idle']),
                                                                      ('nosuccess="%i"'%waste_mill['nosuccess']),
                                                                      ('badput="%i"'%waste_mill['badput']))))
        finally:
            fd.close()

    def write_file(self,relative_fname,str):
        fname=os.path.join(self.monitor_dir,relative_fname)
        #print "Writing "+fname
        fd=open(fname+".tmp","w")
        try:
{'fraction_non_alphanumeric': 0.1637593984962406, 'fraction_numerical': 0.022857142857142857, 'mean_word_length': 1.6041503523884104, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 44, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2342919', 'n_tokens_mistral': 1950, 'n_tokens_neox': 1776, 'n_words': 309}