<commit_before>#!/bin/env python
#
# Description:
#   This is the main of the glideinFactoryEntry
#
# Arguments:
#   $1 = poll period (in seconds)
#   $2 = advertize rate (every $2 loops)
#   $3 = glidein submit_dir
#   $4 = entry name
#
# Author:
#   <NAME> (Sep 15th 2006 - as glideFactory.py)
#   <NAME> (Apr 9th 2007 - as glideFactoryEntry.py)
#

import signal
import os,os.path,sys,fcntl
import traceback
import time,string,math
import copy,random
import threading
sys.path.append(os.path.join(sys.path[0],"../lib"))

import glideFactoryPidLib
import glideFactoryConfig
import glideFactoryLib
import glideFactoryMonitoring
import glideFactoryInterface
import glideFactoryLogParser
import glideFactoryDowntimeLib
import logSupport

############################################################
def check_parent(parent_pid):
    if os.path.exists('/proc/%s'%parent_pid):
        return # parent still exists, we are fine
    
    glideFactoryLib.log_files.logActivity("Parent died, exit.")    
    raise KeyboardInterrupt,"Parent died"


############################################################
def perform_work(entry_name,
                 schedd_name,
                 client_name,client_int_name,client_int_req,
                 idle_glideins,max_running,max_held,
                 jobDescript,
                 x509_proxy_fnames,x509_proxy_usernames,
                 client_web,
                 params):
    glideFactoryLib.factoryConfig.client_internals[client_int_name]={"CompleteName":client_name,"ReqName":client_int_req}

    if params.has_key("GLIDEIN_Collector"):
        condor_pool=params["GLIDEIN_Collector"]
    else:
        condor_pool=None
    
    #glideFactoryLib.log_files.logActivity("QueryQ (%s,%s,%s,%s,%s)"%(glideFactoryLib.factoryConfig.factory_name,glideFactoryLib.factoryConfig.glidein_name,entry_name,client_name,schedd_name))
    try:
        condorQ=glideFactoryLib.getCondorQData(entry_name,client_int_name,schedd_name)
    except glideFactoryLib.condorExe.ExeError,e:
        glideFactoryLib.log_files.logActivity("Client '%s', schedd not responding, skipping"%client_int_name)
        glideFactoryLib.log_files.logWarning("getCondorQData failed: %s"%e)
        # protect and skip
        return 0
    except:
        glideFactoryLib.log_files.logActivity("Client '%s', schedd not responding, skipping"%client_int_name)
        tb = traceback.format_exception(sys.exc_info()[0],sys.exc_info()[1],
                                        sys.exc_info()[2])
        glideFactoryLib.log_files.logWarning("getCondorQData failed, traceback: %s"%string.join(tb,''))
        # protect and skip
        return 0

    #glideFactoryLib.log_files.logActivity("QueryS (%s,%s,%s,%s,%s)"%(glideFactoryLib.factoryConfig.factory_name,glideFactoryLib.factoryConfig.glidein_name,entry_name,client_name,schedd_name))

    # Temporary disable queries to the collector
    # Not really used by anybody, so let reduce the load
    #try:
    #    condorStatus=glideFactoryLib.getCondorStatusData(entry_name,client_name,condor_pool)
    #except:
    if 1:
        condorStatus=None # this is not fundamental information, can live without
    #glideFactoryLib.log_files.logActivity("Work")

    x509_proxy_keys=x509_proxy_fnames.keys()
    random.shuffle(x509_proxy_keys) # randomize so I don't favour any proxy over another

    # find out the users it is using
    usernames={}
    for x509_proxy_id in x509_proxy_keys:
        username=x509_proxy_usernames[x509_proxy_id]
        usernames[username]=True

    log_stats={}
    for username in usernames.keys():
        log_stats[username]=glideFactoryLogParser.dirSummaryTimingsOut(glideFactoryLib.factoryConfig.get_client_log_dir(entry_name,x509_proxy_usernames[x509_proxy_id]),
                                                                       glideFactoryLib.log_files.log_dir,
                                                                       client_name,username)
        # should not need privsep for reading logs
        log_stats[username].load()

    glideFactoryLib.logStats(condorQ,condorStatus,client_int_name)
    glideFactoryLib.factoryConfig.log_stats.logSummary(client_int_name,log_stats)
        

    submit_attrs=[]

    # use the extended params for submission
    proxy_fraction=1.0/len(x509_proxy_keys)

    # I will shuffle proxies around, so I may as well round up all of them
    idle_glideins_pproxy=math.ceil(idle_glideins*proxy_fraction)
    max_running_pproxy=math.ceil(max_running*proxy_fraction)

    # not reducing the held, as that is effectively per proxy, not per request
    nr_submitted=0
    for x509_proxy_id in x509_proxy_keys:
        nr_submitted+=glideFactoryLib.keepIdleGlideins(condorQ,client_int_name,
                                                       idle_glideins_pproxy,max_running_pproxy,max_held,submit_attrs,
                                                       x509_proxy_id,x509_proxy_fnames[x509_proxy_id],x509_proxy_usernames[x509_proxy_id],
                                                       client_web,params)
    if nr_submitted>0:
        #glideFactoryLib.log_files.logActivity("Submitted")
        return 1 # we submitted something, return immediately

    if condorStatus!=None: # temporary glitch, no sanitization this round<commit_msg>Fix bug - We were using a variable from another loop; it worked because all deployements always had a 1-to-1 mapping between the client and the UID. Use the proper value now.<commit_after>#!/bin/env python
#
# Description:
#   This is the main of the glideinFactoryEntry
#
# Arguments:
#   $1 = poll period (in seconds)
#   $2 = advertize rate (every $2 loops)
#   $3 = glidein submit_dir
#   $4 = entry name
#
# Author:
#   <NAME> (Sep 15th 2006 - as glideFactory.py)
#   <NAME> (Apr 9th 2007 - as glideFactoryEntry.py)
#

import signal
import os,os.path,sys,fcntl
import traceback
import time,string,math
import copy,random
import threading
sys.path.append(os.path.join(sys.path[0],"../lib"))

import glideFactoryPidLib
import glideFactoryConfig
import glideFactoryLib
import glideFactoryMonitoring
import glideFactoryInterface
import glideFactoryLogParser
import glideFactoryDowntimeLib
import logSupport

############################################################
def check_parent(parent_pid):
    if os.path.exists('/proc/%s'%parent_pid):
        return # parent still exists, we are fine
    
    glideFactoryLib.log_files.logActivity("Parent died, exit.")    
    raise KeyboardInterrupt,"Parent died"


############################################################
def perform_work(entry_name,
                 schedd_name,
                 client_name,client_int_name,client_int_req,
                 idle_glideins,max_running,max_held,
                 jobDescript,
                 x509_proxy_fnames,x509_proxy_usernames,
                 client_web,
                 params):
    glideFactoryLib.factoryConfig.client_internals[client_int_name]={"CompleteName":client_name,"ReqName":client_int_req}

    if params.has_key("GLIDEIN_Collector"):
        condor_pool=params["GLIDEIN_Collector"]
    else:
        condor_pool=None
    
    #glideFactoryLib.log_files.logActivity("QueryQ (%s,%s,%s,%s,%s)"%(glideFactoryLib.factoryConfig.factory_name,glideFactoryLib.factoryConfig.glidein_name,entry_name,client_name,schedd_name))
    try:
        condorQ=glideFactoryLib.getCondorQData(entry_name,client_int_name,schedd_name)
    except glideFactoryLib.condorExe.ExeError,e:
        glideFactoryLib.log_files.logActivity("Client '%s', schedd not responding, skipping"%client_int_name)
        glideFactoryLib.log_files.logWarning("getCondorQData failed: %s"%e)
        # protect and skip
        return 0
    except:
        glideFactoryLib.log_files.logActivity("Client '%s', schedd not responding, skipping"%client_int_name)
        tb = traceback.format_exception(sys.exc_info()[0],sys.exc_info()[1],
                                        sys.exc_info()[2])
        glideFactoryLib.log_files.logWarning("getCondorQData failed, traceback: %s"%string.join(tb,''))
        # protect and skip
        return 0

    #glideFactoryLib.log_files.logActivity("QueryS (%s,%s,%s,%s,%s)"%(glideFactoryLib.factoryConfig.factory_name,glideFactoryLib.factoryConfig.glidein_name,entry_name,client_name,schedd_name))

    # Temporary disable queries to the collector
    # Not really used by anybody, so let reduce the load
    #try:
    #    condorStatus=glideFactoryLib.getCondorStatusData(entry_name,client_name,condor_pool)
    #except:
    if 1:
        condorStatus=None # this is not fundamental information, can live without
    #glideFactoryLib.log_files.logActivity("Work")

    x509_proxy_keys=x509_proxy_fnames.keys()
    random.shuffle(x509_proxy_keys) # randomize so I don't favour any proxy over another

    # find out the users it is using
    usernames={}
    for x509_proxy_id in x509_proxy_keys:
        username=x509_proxy_usernames[x509_proxy_id]
        usernames[username]=True

    log_stats={}
    for username in usernames.keys():
        log_stats[username]=glideFactoryLogParser.dirSummaryTimingsOut(glideFactoryLib.factoryConfig.get_client_log_dir(entry_name,username),
                                                                       glideFactoryLib.log_files.log_dir,
                                                                       client_name,username)
        # should not need privsep for reading logs
        log_stats[username].load()

    glideFactoryLib.logStats(condorQ,condorStatus,client_int_name)
    glideFactoryLib.factoryConfig.log_stats.logSummary(client_int_name,log_stats)
        

    submit_attrs=[]

    # use the extended params for submission
    proxy_fraction=1.0/len(x509_proxy_keys)

    # I will shuffle proxies around, so I may as well round up all of them
    idle_glideins_pproxy=math.ceil(idle_glideins*proxy_fraction)
    max_running_pproxy=math.ceil(max_running*proxy_fraction)

    # not reducing the held, as that is effectively per proxy, not per request
    nr_submitted=0
    for x509_proxy_id in x509_proxy_keys:
        nr_submitted+=glideFactoryLib.keepIdleGlideins(condorQ,client_int_name,
                                                       idle_glideins_pproxy,max_running_pproxy,max_held,submit_attrs,
                                                       x509_proxy_id,x509_proxy_fnames[x509_proxy_id],x509_proxy_usernames[x509_proxy_id],
                                                       client_web,params)
    if nr_submitted>0:
        #glideFactoryLib.log_files.logActivity("Submitted")
        return 1 # we submitted something, return immediately

    if condorStatus!=None: # temporary glitch, no sanitization this round
{'fraction_non_alphanumeric': 0.10343539955190441, 'fraction_numerical': 0.01549663928304705, 'mean_word_length': 3.2360616844602608, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1754710', 'n_tokens_mistral': 3327, 'n_tokens_neox': 3165, 'n_words': 593}