<commit_before>#!/usr/bin/env python

import pathfix
import sys
import logging
import logging.handlers
import json
from time import sleep
import os
import random
from subprocess import Popen, PIPE
from hpc import Job, LocalHPCJob, HPCJobStatus
import urllib2
from datetime import datetime
import traceback
import subprocess
import commands

# Setup the logger
log = logging.getLogger()
log.setLevel(logging.DEBUG)
log.addHandler(logging.StreamHandler())

log.debug("Starting modeld.py")

currentJob = None
currentCycle = 0

minSleepTime = 10
maxSleepTime = 60

# The Main Loop
while True:

    # Sleep a time, then go round again
    # Inject randomness into the sleep time to reduce likelihood of multiple
    # scripts running against the same species.
    randSleepTime = random.randint(minSleepTime, maxSleepTime)
    log.debug("Sleeping %i seconds.", randSleepTime)
    sleep(randSleepTime)

    currentCycle += 1
    log.debug("Loop %i (%s). Current Job: %s", currentCycle, datetime.today(), currentJob)

    # Check if we are handling a current job
    if currentJob == None:
        try:
            log.debug("No current job...")

            # If we don't have a current job,<commit_msg>Update modelling/bin/local_modeld.py<commit_after>#!/usr/bin/env python

import pathfix
import sys
import logging
import logging.handlers
import json
from time import sleep
import os
import random
from subprocess import Popen, PIPE
from hpc import Job, LocalHPCJob, HPCJobStatus
import urllib2
from datetime import datetime
import traceback
import subprocess
import commands

# Setup the logger
log = logging.getLogger()
log.setLevel(logging.DEBUG)
log.addHandler(logging.StreamHandler())

log.debug("Starting modeld.py")

currentJob = None
currentCycle = 0

minSleepTime = 10
maxSleepTime = 60

# The Main Loop
while True:

    # Sleep a time, then go round again
    # Inject randomness into the sleep time to reduce likelihood of multiple
    # scripts running against the same species (no longer an issue, the web request 
    # for next species will no longer return the same species multiple times, unless it needs remodelling).
    randSleepTime = random.randint(minSleepTime, maxSleepTime)
    log.debug("Sleeping %i seconds.", randSleepTime)
    sleep(randSleepTime)

    currentCycle += 1
    log.debug("Loop %i (%s). Current Job: %s", currentCycle, datetime.today(), currentJob)

    # Check if we are handling a current job
    if currentJob == None:
        try:
            log.debug("No current job...")

            # If we don't have a current job,
{'fraction_non_alphanumeric': 0.07728337236533958, 'fraction_numerical': 0.00546448087431694, 'mean_word_length': 3.8358490566037737, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5499319', 'n_tokens_mistral': 768, 'n_tokens_neox': 718, 'n_words': 283}