<commit_before>#
# Description:
#   This module implements classes and functions to parse
#   the condor log files.
#
# Author:
#   <NAME> (Feb 1st 2007)
#

import os,os.path,stat
import re,mmap
import cPickle

class cachedLogClass:
    # virtual, do not use
    # the Constructor needs to define logname and cachename
    # also loadFromLog need to be implemented

    # compare to cache, and tell if the log file has changed since last checked
    def has_changed(self):
        if os.path.isfile(self.logname):
            fstat=os.lstat(self.logname)
            logtime=fstat[stat.ST_MTIME]
        else:
            return False # does not exist, so it could not change
        
        if os.path.isfile(self.cachename):
            fstat=os.lstat(self.cachename)
            cachetime=fstat[stat.ST_MTIME]
        else:
            return True # no cache, so it has changed for sure
        
        # both exist -> check if log file is newer
        return (logtime>cachetime)

    # load from the most recent one, and update the cache if needed
    def load(self):
        if not self.has_changed():
            # cache is newer, just load the cache
            return self.loadCache()

        while 1: #could need more than one loop if the log file is changing
            fstat=os.lstat(self.logname)
            start_logtime=fstat[stat.ST_MTIME]
            del fstat
            
            self.loadFromLog()
            try:
                self.saveCache()
            except IOError:
                return # silently ignore, this was a load in the end
            # the log may have changed -> check
            fstat=os.lstat(self.logname)
            logtime=fstat[stat.ST_MTIME]
            del fstat
            if logtime<=start_logtime:
                return # OK, not changed, can exit
        
        return # should never reach this point

        
    def loadCache(self):
        fd=open(self.cachename,"r")
        self.data=cPickle.load(fd)
        fd.close()
        return

    ####### PRIVATE ###########
    def saveCache(self):
        # two steps; first create a tmp file, then rename
        tmpname=self.cachename+(".tmp_%i"%os.getpid())
        fd=open(tmpname,"w")
        cPickle.dump(self.data,fd)
        fd.close()

        try:
            os.remove(self.cachename)
        except:
            pass # may not exist
        os.rename(tmpname,self.cachename)
        
        return

        
# this class will keep track of:
#  - counts of statuses (Wait, Idle, Running, Held, Completed, Removed)
#  - list of completed jobs
# These data is available in self.data dictionary
class logSummary(cachedLogClass):
    def __init__(self,logname):
        self.logname=logname
        self.cachename=logname+".clspk"

    def loadFromLog(self):
        tmpdata={}
        jobs = parseSubmitLogFastRaw(self.logname)
        status  = listAndInterpretRawStatuses(jobs)
        counts={}
        for s in status.keys():
            counts[s]=len(status[s])
        tmpdata['counts']=counts
        if status.has_key("Completed"):
            tmpdata['completed_jobs']=status['Completed']
        else:
            tmpdata['completed_jobs']=0
        self.data=tmpdata
        return

# this class will keep track of
#  counts of statuses (Wait, Idle, Running, Held, Completed, Removed)
# These data is available in self.data dictionary
class logCounts(cachedLogClass):
    def __init__(self,logname):
        self.logname=logname
        self.cachename=logname+".clcpk"

    def loadFromLog(self):
        tmpdata={}
        jobs = parseSubmitLogFastRaw(self.logname)
        self.data  = countAndInterpretRawStatuses(jobs)
        return


##############################################################################
#
# Low level functions
#
##############################################################################

################################
#  Condor log parsing functions
################################

# read a condor submit log
# return a dictionary of jobStrings each having the last statusString
# for example {'1583.004': '000', '3616.008': '009'}<commit_msg>Changed logSummary and added logCompleted<commit_after>#
# Description:
#   This module implements classes and functions to parse
#   the condor log files.
#
# Author:
#   <NAME> (Feb 1st 2007)
#

import os,os.path,stat
import re,mmap
import cPickle

class cachedLogClass:
    # virtual, do not use
    # the Constructor needs to define logname and cachename
    # also loadFromLog need to be implemented

    # compare to cache, and tell if the log file has changed since last checked
    def has_changed(self):
        if os.path.isfile(self.logname):
            fstat=os.lstat(self.logname)
            logtime=fstat[stat.ST_MTIME]
        else:
            return False # does not exist, so it could not change
        
        if os.path.isfile(self.cachename):
            fstat=os.lstat(self.cachename)
            cachetime=fstat[stat.ST_MTIME]
        else:
            return True # no cache, so it has changed for sure
        
        # both exist -> check if log file is newer
        return (logtime>cachetime)

    # load from the most recent one, and update the cache if needed
    def load(self):
        if not self.has_changed():
            # cache is newer, just load the cache
            return self.loadCache()

        while 1: #could need more than one loop if the log file is changing
            fstat=os.lstat(self.logname)
            start_logtime=fstat[stat.ST_MTIME]
            del fstat
            
            self.loadFromLog()
            try:
                self.saveCache()
            except IOError:
                return # silently ignore, this was a load in the end
            # the log may have changed -> check
            fstat=os.lstat(self.logname)
            logtime=fstat[stat.ST_MTIME]
            del fstat
            if logtime<=start_logtime:
                return # OK, not changed, can exit
        
        return # should never reach this point

        
    def loadCache(self):
        fd=open(self.cachename,"r")
        self.data=cPickle.load(fd)
        fd.close()
        return

    ####### PRIVATE ###########
    def saveCache(self):
        # two steps; first create a tmp file, then rename
        tmpname=self.cachename+(".tmp_%i"%os.getpid())
        fd=open(tmpname,"w")
        cPickle.dump(self.data,fd)
        fd.close()

        try:
            os.remove(self.cachename)
        except:
            pass # may not exist
        os.rename(tmpname,self.cachename)
        
        return

        
# this class will keep track of:
#  jobs in various of statuses (Wait, Idle, Running, Held, Completed, Removed)
# These data is available in self.data dictionary
class logSummary(cachedLogClass):
    def __init__(self,logname):
        self.logname=logname
        self.cachename=logname+".cstpk"

    def loadFromLog(self):
        jobs = parseSubmitLogFastRaw(self.logname)
        self.data = listAndInterpretRawStatuses(jobs)
        return

# this class will keep track of:
#  - counts of statuses (Wait, Idle, Running, Held, Completed, Removed)
#  - list of completed jobs
# These data is available in self.data dictionary
class logCompleted(cachedLogClass):
    def __init__(self,logname):
        self.logname=logname
        self.cachename=logname+".clspk"

    def loadFromLog(self):
        tmpdata={}
        jobs = parseSubmitLogFastRaw(self.logname)
        status  = listAndInterpretRawStatuses(jobs)
        counts={}
        for s in status.keys():
            counts[s]=len(status[s])
        tmpdata['counts']=counts
        if status.has_key("Completed"):
            tmpdata['completed_jobs']=status['Completed']
        else:
            tmpdata['completed_jobs']=[]
        self.data=tmpdata
        return

# this class will keep track of
#  counts of statuses (Wait, Idle, Running, Held, Completed, Removed)
# These data is available in self.data dictionary
class logCounts(cachedLogClass):
    def __init__(self,logname):
        self.logname=logname
        self.cachename=logname+".clcpk"

    def loadFromLog(self):
        tmpdata={}
        jobs = parseSubmitLogFastRaw(self.logname)
        self.data  = countAndInterpretRawStatuses(jobs)
        return


##############################################################################
#
# Low level functions
#
##############################################################################

################################
#  Condor log parsing functions
################################

# read a condor submit log
# return a dictionary of jobStrings each having the last statusString
# for example {'1583.004': '000', '3616.008': '009'}
{'fraction_non_alphanumeric': 0.13652502577024395, 'fraction_numerical': 0.006070324132401787, 'mean_word_length': 2.3636363636363638, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6777442', 'n_tokens_mistral': 2548, 'n_tokens_neox': 2474, 'n_words': 704}