<commit_before>#
# Description:
#   This module implements the functions needed
#   to aggregate the monitoring fo the frontend
#
# Author:
#   <NAME> (Mar 19th 2009)
#

import time,string,os.path
import timeConversion
import xmlParse,xmlFormat
import glideinFrontendMonitoring


############################################################
#
# Configuration
#
############################################################

class MonitorAggregatorConfig:
    def __init__(self):
        # The name of the attribute that identifies the glidein
        self.monitor_dir="monitor/"

        # list of entries
        self.entries=[]

        # name of the status files
        self.status_relname="frontend_status.xml"

    def config_frontend(self,monitor_dir,groups):
        self.monitor_dir=monitor_dir
        self.groups=groups
        glideinFrontendMonitoring.monitoringConfig.monitor_dir=monitor_dir
    

# global configuration of the module
monitorAggregatorConfig=MonitorAggregatorConfig()

###########################################################
#
# Functions
#
###########################################################

status_attributes={'Jobs':("Idle","OldIdle","Running","Total"),
                   'Slots':("Idle","Running","Total")}

##############################################################################
# create an aggregate of status files, write it in an aggregate status file
# end return the values
def aggregateStatus():
    global monitorAggregatorConfig

    global_total={'Jobs':None,'Slots':None}
    status={'groups':{},'total':global_total}
    nr_groups=0
    for group in monitorAggregatorConfig.groups:
        # load group status file
        status_fname=os.path.join(os.path.join(monitorAggregatorConfig.monitor_dir,'group_'+group),
                                  monitorAggregatorConfig.status_relname)
        try:
            group_data=xmlParse.xmlfile2dict(status_fname)
        except IOError:
            continue # file not found, ignore

        nr_groups+=1
        status['groups'][group]=group_data

        for w in global_total.keys():
            tel=global_total[w]
            if not group_data.has_key(w):
                continue 
            el=group_data[w]
            if tel==None:
                # new one, just copy over
                global_total[w]={}
                tel=global_total[w]
                for a in el.keys():
                    tel[a]=int(el[a]) #coming from XML, everything is a string
            else:                
                # successive, sum 
                for a in el.keys():
                    if tel.has_key(a):
                        tel[a]+=int(el[a])
                            
                # if any attribute from prev. frontends are not in the current one, remove from total
                for a in tel.keys():
                    if not el.has_key(a):
                        del tel[a]
        
    for w in global_total.keys():
        if global_total[w]==None:
            del global_total[w] # remove group if not defined

    # Write xml files
    updated=time.time()
    xml_str=('<?xml version="1.0" encoding="ISO-8859-1"?>\n\n'+
             '<VOFrontendStats>\n'+
             get_xml_updated(updated,indent_tab=xmlFormat.DEFAULT_TAB,leading_tab=xmlFormat.DEFAULT_TAB)+"\n"+
             xmlFormat.dict2string(status["groups"],dict_name="groups",el_name="group",
                                   leading_tab=xmlFormat.DEFAULT_TAB)+"\n"+
             xmlFormat.class2string(status["total"],inst_name="total",leading_tab=xmlFormat.DEFAULT_TAB)+"\n"+
             "</VOFrontendStats>\n")
    glideFactoryMonitoring.monitoringConfig.write_file(monitorAggregatorConfig.status_relname,xml_str)

    # Write rrds
    glideFactoryMonitoring.monitoringConfig.establish_dir("total")

    type_strings={'Jobs':'Jobs','Slots':'Slots'}<commit_msg>Fix typo<commit_after>#
# Description:
#   This module implements the functions needed
#   to aggregate the monitoring fo the frontend
#
# Author:
#   <NAME> (Mar 19th 2009)
#

import time,string,os.path
import timeConversion
import xmlParse,xmlFormat
import glideinFrontendMonitoring


############################################################
#
# Configuration
#
############################################################

class MonitorAggregatorConfig:
    def __init__(self):
        # The name of the attribute that identifies the glidein
        self.monitor_dir="monitor/"

        # list of entries
        self.entries=[]

        # name of the status files
        self.status_relname="frontend_status.xml"

    def config_frontend(self,monitor_dir,groups):
        self.monitor_dir=monitor_dir
        self.groups=groups
        glideinFrontendMonitoring.monitoringConfig.monitor_dir=monitor_dir
    

# global configuration of the module
monitorAggregatorConfig=MonitorAggregatorConfig()

###########################################################
#
# Functions
#
###########################################################

status_attributes={'Jobs':("Idle","OldIdle","Running","Total"),
                   'Slots':("Idle","Running","Total")}

##############################################################################
# create an aggregate of status files, write it in an aggregate status file
# end return the values
def aggregateStatus():
    global monitorAggregatorConfig

    global_total={'Jobs':None,'Slots':None}
    status={'groups':{},'total':global_total}
    nr_groups=0
    for group in monitorAggregatorConfig.groups:
        # load group status file
        status_fname=os.path.join(os.path.join(monitorAggregatorConfig.monitor_dir,'group_'+group),
                                  monitorAggregatorConfig.status_relname)
        try:
            group_data=xmlParse.xmlfile2dict(status_fname)
        except IOError:
            continue # file not found, ignore

        nr_groups+=1
        status['groups'][group]=group_data

        for w in global_total.keys():
            tel=global_total[w]
            if not group_data.has_key(w):
                continue 
            el=group_data[w]
            if tel==None:
                # new one, just copy over
                global_total[w]={}
                tel=global_total[w]
                for a in el.keys():
                    tel[a]=int(el[a]) #coming from XML, everything is a string
            else:                
                # successive, sum 
                for a in el.keys():
                    if tel.has_key(a):
                        tel[a]+=int(el[a])
                            
                # if any attribute from prev. frontends are not in the current one, remove from total
                for a in tel.keys():
                    if not el.has_key(a):
                        del tel[a]
        
    for w in global_total.keys():
        if global_total[w]==None:
            del global_total[w] # remove group if not defined

    # Write xml files
    updated=time.time()
    xml_str=('<?xml version="1.0" encoding="ISO-8859-1"?>\n\n'+
             '<VOFrontendStats>\n'+
             get_xml_updated(updated,indent_tab=xmlFormat.DEFAULT_TAB,leading_tab=xmlFormat.DEFAULT_TAB)+"\n"+
             xmlFormat.dict2string(status["groups"],dict_name="groups",el_name="group",
                                   subtypes_params={"class":{}},
                                   leading_tab=xmlFormat.DEFAULT_TAB)+"\n"+
             xmlFormat.class2string(status["total"],inst_name="total",leading_tab=xmlFormat.DEFAULT_TAB)+"\n"+
             "</VOFrontendStats>\n")
    glideFactoryMonitoring.monitoringConfig.write_file(monitorAggregatorConfig.status_relname,xml_str)

    # Write rrds
    glideFactoryMonitoring.monitoringConfig.establish_dir("total")

    type_strings={'Jobs':'Jobs','Slots':'Slots'}
{'fraction_non_alphanumeric': 0.17580025608194622, 'fraction_numerical': 0.0046094750320102434, 'mean_word_length': 2.586317722681359, 'pattern_counts': {'":': 1, '<': 11, '<?xml version=': 2, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 32}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3303570', 'n_tokens_mistral': 2131, 'n_tokens_neox': 2046, 'n_words': 417}