<commit_before>import os
import re
import sys
import time
import random
import datetime
import commands
from taskbuffer.OraDBProxy import DBProxy
from taskbuffer.TaskBuffer import taskBuffer
from pandalogger.PandaLogger import PandaLogger
from dataservice.Adder import Adder
from brokerage.SiteMapper import SiteMapper

# password
from config import panda_config
passwd = <PASSWORD>

# logger
_logger = PandaLogger().getLogger('add')

_logger.debug("===================== start =====================")

# kill old process
try:
    # time limit
    timeLimit = datetime.datetime.utcnow() - datetime.timedelta(hours=2)
    # get process list
    scriptName = sys.argv[0]
    out = commands.getoutput('env TZ=UTC ps axo user,pid,lstart,args | grep %s' % scriptName)
    for line in out.split('\n'):
        items = line.split()
        # owned process
        if not items[0] in ['sm','atlpan','root']: # ['os.getlogin()']: doesn't work in cron
            continue
        # look for python
        if re.search('python',line) == None:
            continue
        # PID
        pid = items[1]
        # start time
        timeM = re.search('(\S+\s+\d+ \d+:\d+:\d+ \d+)',line)
        startTime = datetime.datetime(*time.strptime(timeM.group(1),'%b %d %H:%M:%S %Y')[:6])
        # kill old process
        if startTime < timeLimit:
            _logger.debug("old process : %s %s" % (pid,startTime))
            _logger.debug(line)            
            commands.getoutput('kill -9 %s' % pid)
except:
    type, value, traceBack = sys.exc_info()
    _logger.error("kill process : %s %s" % (type,value))
    
# instantiate DB proxies
proxyS = DBProxy()
proxyS.connect(panda_config.dbhost,panda_config.dbpasswd,panda_config.dbuser,panda_config.dbname)

# delete
_logger.debug("Del session")
status,retSel = proxyS.querySQLS("SELECT MAX(PandaID) FROM ATLAS_PANDA.jobsDefined4",{})
if retSel != None:
    try:
        maxID = retSel[0][0]
        _logger.debug("maxID : %s" % maxID)
        if maxID != None:
            varMap = {}
            varMap[':maxID'] = maxID
            varMap[':jobStatus1'] = 'activated'
            varMap[':jobStatus2'] = 'waiting'
            varMap[':jobStatus3'] = 'failed'
            status,retDel = proxyS.querySQLS("DELETE FROM ATLAS_PANDA.jobsDefined4 WHERE PandaID<:maxID AND (jobStatus=:jobStatus1 OR jobStatus=:jobStatus2 OR jobStatus=:jobStatus3)",varMap)
    except:
        pass
    
# instantiate TB
taskBuffer.init(panda_config.dbhost,panda_config.dbpasswd,nDBConnection=1)

# instantiate sitemapper
aSiteMapper = SiteMapper(taskBuffer)

# get buildJobs in the holding state
holdingAna = []
varMap = {}
varMap[':prodSourceLabel'] = 'panda'
varMap[':jobStatus'] = 'holding'
status,res = proxyS.querySQLS("SELECT PandaID from ATLAS_PANDA.jobsActive4 WHERE prodSourceLabel=:prodSourceLabel AND jobStatus=:jobStatus",varMap)
if res != None:
    for id, in res:
        holdingAna.append(id)
_logger.debug("holding Ana %s " % holdingAna)
    
# add files
_logger.debug("Adder session")
timeNow = datetime.datetime.utcnow()
timeInt = datetime.datetime.utcnow()
dirName = panda_config.logdir
fileList = os.listdir(dirName)
fileList.sort() 
# remove duplicated files
tmpList = []
uMap = {}
for file in fileList:
    match = re.search('^(\d+)_([^_]+)_.{36}$',file)
    if match != None:
        fileName = '%s/%s' % (dirName,file)
        id = match.group(1)
        if uMap.has_key(id):
            try:
                os.remove(fileName)
            except:
                pass
        else:
            uMap[id] = fileName
            if long(id) in holdingAna:
                # give a priority to buildJobs
                tmpList.insert(0,file)
            else:
                tmpList.append(file)
fileList = tmpList            

# add
while len(fileList) != 0:
    # time limit to aviod too many copyArchve running at the sametime
    if (datetime.datetime.utcnow() - timeNow) > datetime.timedelta(minutes=40):
        _logger.debug("time over in Adder session")
        break
    # get fileList
    if (datetime.datetime.utcnow() - timeInt) > datetime.timedelta(minutes=15):
        timeInt = datetime.datetime.utcnow()
        # get file
        fileList = os.listdir(dirName)
        fileList.sort() 
        # remove duplicated files
        tmpList = []
        uMap = {}
        for file in fileList:
            match = re.search('^(\d+)_([^_]+)_.{36}$',file)
            if match != None:
                fileName = '%s/%s' % (dirName,file)
                id = match.group(1)
                if uMap.has_key(id):
                    try:
                        os.remove(fileName)
                    except:
                        pass
                else:
                    uMap[id] = fileName
                    if long(id) in holdingAna:
                        # give a priority to buildJob<commit_msg>increase limit to 60min<commit_after>import os
import re
import sys
import time
import random
import datetime
import commands
from taskbuffer.OraDBProxy import DBProxy
from taskbuffer.TaskBuffer import taskBuffer
from pandalogger.PandaLogger import PandaLogger
from dataservice.Adder import Adder
from brokerage.SiteMapper import SiteMapper

# password
from config import panda_config
passwd = <PASSWORD>

# logger
_logger = PandaLogger().getLogger('add')

_logger.debug("===================== start =====================")

# kill old process
try:
    # time limit
    timeLimit = datetime.datetime.utcnow() - datetime.timedelta(hours=2)
    # get process list
    scriptName = sys.argv[0]
    out = commands.getoutput('env TZ=UTC ps axo user,pid,lstart,args | grep %s' % scriptName)
    for line in out.split('\n'):
        items = line.split()
        # owned process
        if not items[0] in ['sm','atlpan','root']: # ['os.getlogin()']: doesn't work in cron
            continue
        # look for python
        if re.search('python',line) == None:
            continue
        # PID
        pid = items[1]
        # start time
        timeM = re.search('(\S+\s+\d+ \d+:\d+:\d+ \d+)',line)
        startTime = datetime.datetime(*time.strptime(timeM.group(1),'%b %d %H:%M:%S %Y')[:6])
        # kill old process
        if startTime < timeLimit:
            _logger.debug("old process : %s %s" % (pid,startTime))
            _logger.debug(line)            
            commands.getoutput('kill -9 %s' % pid)
except:
    type, value, traceBack = sys.exc_info()
    _logger.error("kill process : %s %s" % (type,value))
    
# instantiate DB proxies
proxyS = DBProxy()
proxyS.connect(panda_config.dbhost,panda_config.dbpasswd,panda_config.dbuser,panda_config.dbname)

# delete
_logger.debug("Del session")
status,retSel = proxyS.querySQLS("SELECT MAX(PandaID) FROM ATLAS_PANDA.jobsDefined4",{})
if retSel != None:
    try:
        maxID = retSel[0][0]
        _logger.debug("maxID : %s" % maxID)
        if maxID != None:
            varMap = {}
            varMap[':maxID'] = maxID
            varMap[':jobStatus1'] = 'activated'
            varMap[':jobStatus2'] = 'waiting'
            varMap[':jobStatus3'] = 'failed'
            status,retDel = proxyS.querySQLS("DELETE FROM ATLAS_PANDA.jobsDefined4 WHERE PandaID<:maxID AND (jobStatus=:jobStatus1 OR jobStatus=:jobStatus2 OR jobStatus=:jobStatus3)",varMap)
    except:
        pass
    
# instantiate TB
taskBuffer.init(panda_config.dbhost,panda_config.dbpasswd,nDBConnection=1)

# instantiate sitemapper
aSiteMapper = SiteMapper(taskBuffer)

# get buildJobs in the holding state
holdingAna = []
varMap = {}
varMap[':prodSourceLabel'] = 'panda'
varMap[':jobStatus'] = 'holding'
status,res = proxyS.querySQLS("SELECT PandaID from ATLAS_PANDA.jobsActive4 WHERE prodSourceLabel=:prodSourceLabel AND jobStatus=:jobStatus",varMap)
if res != None:
    for id, in res:
        holdingAna.append(id)
_logger.debug("holding Ana %s " % holdingAna)
    
# add files
_logger.debug("Adder session")
timeNow = datetime.datetime.utcnow()
timeInt = datetime.datetime.utcnow()
dirName = panda_config.logdir
fileList = os.listdir(dirName)
fileList.sort() 
# remove duplicated files
tmpList = []
uMap = {}
for file in fileList:
    match = re.search('^(\d+)_([^_]+)_.{36}$',file)
    if match != None:
        fileName = '%s/%s' % (dirName,file)
        id = match.group(1)
        if uMap.has_key(id):
            try:
                os.remove(fileName)
            except:
                pass
        else:
            uMap[id] = fileName
            if long(id) in holdingAna:
                # give a priority to buildJobs
                tmpList.insert(0,file)
            else:
                tmpList.append(file)
fileList = tmpList            

# add
while len(fileList) != 0:
    # time limit to aviod too many copyArchve running at the sametime
    if (datetime.datetime.utcnow() - timeNow) > datetime.timedelta(minutes=60):
        _logger.debug("time over in Adder session")
        break
    # get fileList
    if (datetime.datetime.utcnow() - timeInt) > datetime.timedelta(minutes=15):
        timeInt = datetime.datetime.utcnow()
        # get file
        fileList = os.listdir(dirName)
        fileList.sort() 
        # remove duplicated files
        tmpList = []
        uMap = {}
        for file in fileList:
            match = re.search('^(\d+)_([^_]+)_.{36}$',file)
            if match != None:
                fileName = '%s/%s' % (dirName,file)
                id = match.group(1)
                if uMap.has_key(id):
                    try:
                        os.remove(fileName)
                    except:
                        pass
                else:
                    uMap[id] = fileName
                    if long(id) in holdingAna:
                        # give a priority to buildJob
{'fraction_non_alphanumeric': 0.12892629850133444, 'fraction_numerical': 0.006569492917265448, 'mean_word_length': 2.41859649122807, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5489146', 'n_tokens_mistral': 3135, 'n_tokens_neox': 3039, 'n_words': 710}