<commit_before>from TaskManager import TaskManager
from db import MongoDataAdapter
from time import sleep
from daemon.pidfile import PIDLockFile
import psutil

def create_task_manager(db, max_tasks, scan_interval):
    # TaskManager instance.
    tm = TaskManager(mongo, max_tasks=max_tasks, scan_interval=scan_interval)
    return tm

lock = PIDLockFile('pet.pid')

if lock.is_locked() and not lock.i_am_locking():
    if not psutil.pid_exists(lock.read_pid()):
        print 'The pidlock is hold by a void process. Breaking it.'
        lock.break_lock()
    else:
        print 'Another process is holding pidlock. Exit.'
        exit()

print 'Acquiring pidlock.'
lock.acquire(timeout=5)

# TODO Load config.
mongo = MongoDataAdapter.create_adapter('GatesPet')
max_tasks = 50
scan_interval = 10
process_interval = 120

tm = create_task_manager(mongo, max_tasks, scan_interval)

# Fix tasks that previous instance did not proceed.
mongo.fix_tasks()

# Main loop
tm.start()

print 'Main program started.'
while True:
    try:
        if not tm.isAlive():
            tmd = tm
            tm = create_task_manager(mongo, max_tasks, scan_interval)
            tm.queue.queue.append(tmd.queue.queue)
            tm.start()
            
        tasks = mongo.get_scheduled_tasks()
        for task in tasks:
            tm.queue_task(task['module'], task['_id'], task['configure'], True)
            # print "[pet] Task %s added." % task['_id']
    except StandardError as err:
        mongo.push_error('pet', {'message':err.args, 'position':'pet.py >> main loop'})
        
    sleep(process_interval)
        <commit_msg>enhance error handle.<commit_after>from TaskManager import TaskManager
from db import MongoDataAdapter
from time import sleep
from daemon.pidfile import PIDLockFile
import psutil

def create_task_manager(db, max_tasks, scan_interval):
    # TaskManager instance.
    tm = TaskManager(mongo, max_tasks=max_tasks, scan_interval=scan_interval)
    return tm

lock = PIDLockFile('pet.pid')

if lock.is_locked() and not lock.i_am_locking():
    if not psutil.pid_exists(lock.read_pid()):
        print 'The pidlock is hold by a void process. Breaking it.'
        lock.break_lock()
    else:
        print 'Another process is holding pidlock. Exit.'
        exit()

print 'Acquiring pidlock.'
lock.acquire(timeout=5)

# TODO Load config.
mongo = MongoDataAdapter.create_adapter('GatesPet')
max_tasks = 50
scan_interval = 10
process_interval = 120

tm = create_task_manager(mongo, max_tasks, scan_interval)

# Fix tasks that previous instance did not proceed.
mongo.fix_tasks()

# Main loop
tm.start()

print 'Main program started.'
while True:
    try:
        if not tm.isAlive():
            tm = create_task_manager(mongo, max_tasks, scan_interval)
            tm.start()
            
        tasks = mongo.get_scheduled_tasks()
        for task in tasks:
            tm.queue_task(task['module'], task['_id'], task['configure'], True)
            # print "[pet] Task %s added." % task['_id']
    except StandardError as err:
        mongo.push_error('pet', {'message':err.args, 'position':'pet.py >> main loop'})
        
    sleep(process_interval)
        
{'fraction_non_alphanumeric': 0.10352422907488987, 'fraction_numerical': 0.005034612964128383, 'mean_word_length': 3.1994715984147954, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4947061', 'n_tokens_mistral': 999, 'n_tokens_neox': 975, 'n_words': 271}