<commit_before>#!/usr/bin/python
# Functions used to fetch all the number and content of err messages generated by condor 
# <NAME> 2013 MIT
# Assumes linux machine just for clear screen! Why do you want to run it on something else anyway?

#Inputs:
# idir : Initial Directory
import os, sys, time, re 
from Script_Tools import *

def fetchErrs(idir,detailed):
        if not os.path.exists(idir+'/main.py'):
            #Not a task directory
            for folder in os.listdir(idir):
                if os.path.isdir(idir+'/'+folder) and folder[0] != '.':
                    fetchErrs(idir+'/'+folder,detailed)
        else:                
            errids = set()
            jobs = glob.glob(idir+'/CondorOutput/err/*.err')
            for job in jobs:
                if os.path.getsize(idir+'/CondorOutput/err/*.err'):
                    _,_,jobname = job.rpartition('/')
                    errid,_,_ = jobname.rpartition('.')
                    errids.add(eval(errid))

            total       = len(glob.glob(idir+'/CondorOutput/log/*.log'))
            errs        = len(errids);

            if errs:
                print"%s: %s%d%s/%s%d%s"  % (idir.replace('./',''),RED,errs,NOCOLOR,TOTAL_COLOR,total,NOCOLOR)
            if detailed:
                logs = ''
                for errid in errids:
                    command = "tail -n 30 " + idir+'/CondorOutput/err/%d.err' % errid
                    sysCommandHandle = os.popen(command)
                    empty = True
                    for line in sysCommandHandle:
                        print RED+"#%02d: %s"  % (errid, line.rstrip('\n\r')) + NOCOLOR
                        empty = False
                    if empty:
                        print RED+"#%02d: Empty Err File!\n"  % (errid)+ NOCOLOR
if __name__ == '__main__':
    os.system('clear');
    print('*********************************************************');    
    print('************** Reporting For Duty! **********************');    
    print('*********************************************************');    
    detailed = len(sys.argv) > 1
    fetchErrs('.',detailed)   
    
<commit_msg>errs should exclude 0 size<commit_after>#!/usr/bin/python
# Functions used to fetch all the number and content of err messages generated by condor 
# <NAME> 2013 MIT
# Assumes linux machine just for clear screen! Why do you want to run it on something else anyway?

#Inputs:
# idir : Initial Directory
import os, sys, time, re 
from Script_Tools import *

def fetchErrs(idir,detailed):
        if not os.path.exists(idir+'/main.py'):
            #Not a task directory
            for folder in os.listdir(idir):
                if os.path.isdir(idir+'/'+folder) and folder[0] != '.':
                    fetchErrs(idir+'/'+folder,detailed)
        else:                
            errids = set()
            jobs = glob.glob(idir+'/CondorOutput/err/*.err')
            for job in jobs:
                if os.path.getsize(job):
                    _,_,jobname = job.rpartition('/')
                    errid,_,_ = jobname.rpartition('.')
                    errids.add(eval(errid))

            total       = len(glob.glob(idir+'/CondorOutput/log/*.log'))
            errs        = len(errids);

            if errs:
                print"%s: %s%d%s/%s%d%s"  % (idir.replace('./',''),RED,errs,NOCOLOR,TOTAL_COLOR,total,NOCOLOR)
            if detailed:
                logs = ''
                for errid in errids:
                    command = "tail -n 30 " + idir+'/CondorOutput/err/%d.err' % errid
                    sysCommandHandle = os.popen(command)
                    empty = True
                    for line in sysCommandHandle:
                        print RED+"#%02d: %s"  % (errid, line.rstrip('\n\r')) + NOCOLOR
                        empty = False
                    if empty:
                        print RED+"#%02d: Empty Err File!\n"  % (errid)+ NOCOLOR
if __name__ == '__main__':
    os.system('clear');
    print('*********************************************************');    
    print('************** Reporting For Duty! **********************');    
    print('*********************************************************');    
    detailed = len(sys.argv) > 1
    fetchErrs('.',detailed)   
    

{'fraction_non_alphanumeric': 0.19259606373008434, 'fraction_numerical': 0.005857544517338332, 'mean_word_length': 1.7920209287115763, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4637576', 'n_tokens_mistral': 1266, 'n_tokens_neox': 1202, 'n_words': 292}