<commit_before>import os

os.environ['DJANGO_SETTINGS_MODULE'] = 'astrometry.server.settings'

import sys
import socket
import time
import tempfile
import thread
from datetime import datetime

from urllib import urlencode
from urllib2 import urlopen

import pyfits

import astrometry.server.settings as settings

settings.LOGFILE = 'worker.log'

from astrometry.server.log import log
from astrometry.server.models import *

from astrometry.net.util.run_command import run_command

def keep_alive(workerid):
    while True:
        me = Worker.objects.all().get(id=workerid)
        me.keepalive = datetime.utcnow()
        me.save()
        time.sleep(10)

def get_header(header, key, default):
    #if key in header:
    #    return header[key]
    #return default
    try:
        return header[key]
    except KeyError:
        return default

def callback(jobid, fn):
    print 'callback.'
    js=QueuedJob.objects.all().filter(jobid=jobid)
    if js.count() == 0:
        return
    j=js[0]
    if j.stopwork:
        print 'Touching file', fn
        f = open(fn, 'wb')
        f.write('')
        f.close()

def main(indexdirs):
    qname = 'test'
    #backendconfig = 'backend.cfg'
    q = JobQueue.objects.get(name=qname)

    hostname = socket.gethostname()
    ip = socket.gethostbyname(hostname)
    me = Worker(hostname=hostname,
                ip=ip,
                processid=os.getpid())
    me.save()

    thread.start_new_thread(keep_alive, (me.id,))

    #f = open(backendconfig, 'rb')
    #be = f.read()
    #f.close()
    #lines = be.split('\n')

    indexes = []
    for d in indexdirs:
        if not os.path.exists(d):<commit_msg>keep-alive<commit_after>import os

os.environ['DJANGO_SETTINGS_MODULE'] = 'astrometry.server.settings'

import sys
import socket
import time
import tempfile
import thread
from datetime import datetime

from urllib import urlencode
from urllib2 import urlopen

import pyfits

import astrometry.server.settings as settings

settings.LOGFILE = 'worker.log'

from astrometry.server.log import log
from astrometry.server.models import *

from astrometry.net.util.run_command import run_command

def keep_alive(workerid):
    while True:
        print 'Stamping keep-alive.'
        me = Worker.objects.all().get(id=workerid)
        me.keepalive = datetime.utcnow()
        me.save()
        time.sleep(10)

def get_header(header, key, default):
    #if key in header:
    #    return header[key]
    #return default
    try:
        return header[key]
    except KeyError:
        return default

def callback(jobid, fn):
    print 'callback.'
    js=QueuedJob.objects.all().filter(jobid=jobid)
    if js.count() == 0:
        return
    j=js[0]
    if j.stopwork:
        print 'Touching file', fn
        f = open(fn, 'wb')
        f.write('')
        f.close()

def main(indexdirs):
    qname = 'test'
    #backendconfig = 'backend.cfg'
    q = JobQueue.objects.get(name=qname)

    hostname = socket.gethostname()
    ip = socket.gethostbyname(hostname)
    me = Worker(hostname=hostname,
                ip=ip,
                processid=os.getpid(),
                keepalive = datetime.utcnow()
                )
    me.save()

    thread.start_new_thread(keep_alive, (me.id,))

    #f = open(backendconfig, 'rb')
    #be = f.read()
    #f.close()
    #lines = be.split('\n')

    indexes = []
    for d in indexdirs:
        if not os.path.exists(d):
{'fraction_non_alphanumeric': 0.11502347417840375, 'fraction_numerical': 0.0029342723004694834, 'mean_word_length': 2.657725321888412, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6153197', 'n_tokens_mistral': 1149, 'n_tokens_neox': 1151, 'n_words': 266}