<commit_before>#!/bin/env python
#
# Description:
#   This is the main of the glideinFrontend
#
# Arguments:
#   $1 = work_dir
#
# Author:
#   <NAME>
#

import os
import os.path
import sys

STARTUP_DIR=sys.path[0]

import fcntl
import popen2
import traceback
import signal
import time
import string
import copy
import threading
sys.path.append(os.path.join(STARTUP_DIR,"../lib"))

import glideinFrontendPidLib
import glideinFrontendConfig
import glideinFrontendLib
import glideinFrontendMonitorAggregator
import logSupport

# this thread will be used for lazy updates of rrd history conversions
rrd_thread=None

def create_history_thread():
    glideinFrontendMonitorAggregator.create_status_history()
    glideinFrontendMonitorAggregator.create_log_history()
    return

############################################################
def aggregate_stats():
    global rrd_thread
    
    status=glideinFrontendMonitorAggregator.aggregateStatus()
    status=glideinFrontendMonitorAggregator.aggregateLogSummary()

    # keep just one thread per monitoring type running at any given time
    # if the old one is still running, do nothing (lazy)
    # create_support_history can take a-while
    if rrd_thread==None:
        thread_alive=0
    else:
        thread_alive=rrd_thread.isAlive()
        if not thread_alive:
            rrd_thread.join()

    if not thread_alive:
        glideinFrontendLib.log_files.logActivity("Writing lazy stats")
        rrd_thread=threading.Thread(target=create_history_thread)
        rrd_thread.start()

    return

############################################################
def spawn(sleep_time,advertize_rate,work_dir,
          glideinDescript,groups):

    global STARTUP_DIR
    childs={}
    glideinFrontendLib.log_files.logActivity("Starting groups %s"%groups)
    try:
        for group_name in groups:
            childs[group_name]=popen2.Popen3("%s %s %s %s %s"%(sys.executable,os.path.join(STARTUP_DIR,"glideinFrontendElement.py"),os.getpid(),work_dir,group_name),True)

        for group_name in childs.keys():
            childs[group_name].tochild.close()
            # set it in non blocking mode
            # since we will run for a long time, we do not want to block
            for fd  in (childs[group_name].fromchild.fileno(),childs[group_name].childerr.fileno()):
                fl = fcntl.fcntl(fd, fcntl.F_GETFL)
                fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)



        while 1:
            glideinFrontendLib.log_files.logActivity("Checking groups %s"%groups)
            for group_name in childs.keys():
                child=childs[group_name]

                # empty stdout and stderr
                try:
                    tempOut = child.fromchild.read()
                    if len(tempOut)!=0:
                        print child, tempOut
                except IOError:
                    pass # ignore
                try:
                    tempErr = child.childerr.read()
                    if len(tempErr)!=0:
                        print child, tempErr
                except IOError:
                    pass # ignore
                
                # look for exit childs
                if child.poll()!=-1:
                    # the child exited
                    tempOut = child.fromchild.readlines()
                    tempErr = child.childerr.readlines()
                    del childs[group_name]
                    raise RuntimeError,"Group '%s' exited, quit the whole frontend:\n%s\n%s"%(group_name,tempOut,tempErr)

            glideinFrontendLib.log_files.logActivity("Aggregate monitoring data")
            aggregate_stats()

            glideinFrontendLib.log_files.logActivity("Sleep")
            time.sleep(sleep_time)
    finally:        
        # cleanup at exit
        for group_name in childs.keys():
            try:
                os.kill(childs[group_name].pid,signal.SIGTERM)
            except OSError:
                pass # ignore failed kills of non-existent processes
        
        
############################################################
def main(work_dir):
    startup_time=time.time()

    # disable locking... else I can get in deadlock with groups
    glideinFrontendMonitorAggregator.glideinFrontendMonitoring.monitoringConfig.lock_dir=None

    # create log files in the glidein log directory
    glideinFrontendLib.log_files=glideinFrontendLib.LogFiles(os.path.join(work_dir,"log"))
    
    try:
        glideinFrontendConfig.frontendConfig.frontend_descript_file=os.path.join(work_dir,glideinFrontendConfig.frontendConfig.frontend_descript_file)
        frontendDescript=glideinFrontendConfig.FrontendDescript(work_dir)
        frontendDescript.load_pub_key(recreate=True)

        sleep_time=int(frontendDescript.data['LoopDelay'])
        advertize_rate=int(frontendDescript.data['AdvertiseDelay'])
        glideinFrontendMonitorAggregator.glideinFrontendMonitoring.monitoringConfig.wanted_graphs=string.split(frontendDescript.data['FrontendWantedMonitorGraphs'],',')
        
        groups=string.split(frontendDescript.data['Groups'],',')
        groups.sort()

        glideinFrontendMonitorAggregator.monitorAggregatorConfig.config_frontend(os.path.join(work_dir,"monitor"),groups)
    except:
        tb = traceback.format_exception(sys.exc_info()[0],sys.exc_info()[1],
                                        sys.exc_info()[2])
        glideinFrontendLib.log_files.logWarning("Exception at %s: %s" % (time.ctime(),tb))
        print tb
        raise

    # create lock file<commit_msg>Disable monitoring until we implement it<commit_after>#!/bin/env python
#
# Description:
#   This is the main of the glideinFrontend
#
# Arguments:
#   $1 = work_dir
#
# Author:
#   <NAME>
#

import os
import os.path
import sys

STARTUP_DIR=sys.path[0]

import fcntl
import popen2
import traceback
import signal
import time
import string
import copy
import threading
sys.path.append(os.path.join(STARTUP_DIR,"../lib"))

import glideinFrontendPidLib
import glideinFrontendConfig
import glideinFrontendLib
#import glideinFrontendMonitorAggregator
import logSupport

# this thread will be used for lazy updates of rrd history conversions
rrd_thread=None

def create_history_thread():
    #glideinFrontendMonitorAggregator.create_status_history()
    #glideinFrontendMonitorAggregator.create_log_history()
    return

############################################################
def aggregate_stats():
    global rrd_thread
    
    #status=glideinFrontendMonitorAggregator.aggregateStatus()
    #status=glideinFrontendMonitorAggregator.aggregateLogSummary()

    # keep just one thread per monitoring type running at any given time
    # if the old one is still running, do nothing (lazy)
    # create_support_history can take a-while
    if rrd_thread==None:
        thread_alive=0
    else:
        thread_alive=rrd_thread.isAlive()
        if not thread_alive:
            rrd_thread.join()

    if not thread_alive:
        glideinFrontendLib.log_files.logActivity("Writing lazy stats")
        rrd_thread=threading.Thread(target=create_history_thread)
        rrd_thread.start()

    return

############################################################
def spawn(sleep_time,advertize_rate,work_dir,
          glideinDescript,groups):

    global STARTUP_DIR
    childs={}
    glideinFrontendLib.log_files.logActivity("Starting groups %s"%groups)
    try:
        for group_name in groups:
            childs[group_name]=popen2.Popen3("%s %s %s %s %s"%(sys.executable,os.path.join(STARTUP_DIR,"glideinFrontendElement.py"),os.getpid(),work_dir,group_name),True)

        for group_name in childs.keys():
            childs[group_name].tochild.close()
            # set it in non blocking mode
            # since we will run for a long time, we do not want to block
            for fd  in (childs[group_name].fromchild.fileno(),childs[group_name].childerr.fileno()):
                fl = fcntl.fcntl(fd, fcntl.F_GETFL)
                fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)



        while 1:
            glideinFrontendLib.log_files.logActivity("Checking groups %s"%groups)
            for group_name in childs.keys():
                child=childs[group_name]

                # empty stdout and stderr
                try:
                    tempOut = child.fromchild.read()
                    if len(tempOut)!=0:
                        print child, tempOut
                except IOError:
                    pass # ignore
                try:
                    tempErr = child.childerr.read()
                    if len(tempErr)!=0:
                        print child, tempErr
                except IOError:
                    pass # ignore
                
                # look for exit childs
                if child.poll()!=-1:
                    # the child exited
                    tempOut = child.fromchild.readlines()
                    tempErr = child.childerr.readlines()
                    del childs[group_name]
                    raise RuntimeError,"Group '%s' exited, quit the whole frontend:\n%s\n%s"%(group_name,tempOut,tempErr)

            glideinFrontendLib.log_files.logActivity("Aggregate monitoring data")
            aggregate_stats()

            glideinFrontendLib.log_files.logActivity("Sleep")
            time.sleep(sleep_time)
    finally:        
        # cleanup at exit
        for group_name in childs.keys():
            try:
                os.kill(childs[group_name].pid,signal.SIGTERM)
            except OSError:
                pass # ignore failed kills of non-existent processes
        
        
############################################################
def main(work_dir):
    startup_time=time.time()

    # disable locking... else I can get in deadlock with groups
    #glideinFrontendMonitorAggregator.glideinFrontendMonitoring.monitoringConfig.lock_dir=None

    # create log files in the glidein log directory
    glideinFrontendLib.log_files=glideinFrontendLib.LogFiles(os.path.join(work_dir,"log"))
    
    try:
        glideinFrontendConfig.frontendConfig.frontend_descript_file=os.path.join(work_dir,glideinFrontendConfig.frontendConfig.frontend_descript_file)
        frontendDescript=glideinFrontendConfig.FrontendDescript(work_dir)
        frontendDescript.load_pub_key(recreate=True)

        sleep_time=int(frontendDescript.data['LoopDelay'])
        advertize_rate=int(frontendDescript.data['AdvertiseDelay'])
        #glideinFrontendMonitorAggregator.glideinFrontendMonitoring.monitoringConfig.wanted_graphs=string.split(frontendDescript.data['FrontendWantedMonitorGraphs'],',')
        
        groups=string.split(frontendDescript.data['Groups'],',')
        groups.sort()

        #glideinFrontendMonitorAggregator.monitorAggregatorConfig.config_frontend(os.path.join(work_dir,"monitor"),groups)
    except:
        tb = traceback.format_exception(sys.exc_info()[0],sys.exc_info()[1],
                                        sys.exc_info()[2])
        glideinFrontendLib.log_files.logWarning("Exception at %s: %s" % (time.ctime(),tb))
        print tb
        raise

    # create lock file
{'fraction_non_alphanumeric': 0.11558826168895271, 'fraction_numerical': 0.0023333034191869335, 'mean_word_length': 2.6657894736842107, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3799776', 'n_tokens_mistral': 3300, 'n_tokens_neox': 3217, 'n_words': 616}