<commit_before>import os
import sys
import atexit
import time

PROJECT_PATH = os.path.abspath(os.path.dirname(__file__))
sys.path.append(os.path.join(PROJECT_PATH, '..', '..'))

from pygithub3 import Github

import src.todoMelvin
from src.todoMelvin import settings
from src.todoLogging import WarningLevels, log
from src.db.todoRepos import RepoQueues, Repo

def exitHandler():
    log(WarningLevels.Info(), "Cloning Worker shutting down...")


if __name__ == "__main__":
    atexit.register(exitHandler)

    log(WarningLevels.Info(), "Starting Cloning Worker.")

    redis = src.db.todoRedis.connect()
    gh = Github(login = settings.ghLogin, password = settings.ghPassword)

    while True:
        try:
            cloneCount = redis.llen(RepoQueues.Cloning)
            parseCount = redis.llen(RepoQueues.Parsing)
        except:
            log(WarningLevels.Fatal(), "Cloning Worker unable to reach Redis")
            sys.exit()

        if cloneCount > 0 and parseCount < int(settings.maxParseQueueCount):
            repoKey = redis.lpop(RepoQueues.Cloning)
            
            repo = Repo()
            repo.loadFromKey(repoKey)

            #sanity check our loaded key
            assert repo.key() == repoKey, "Bad repo saved in coning Queue! Key %s not found!"%(repoKey)

            #clone the repo and add it to the parse queue
            src.todoMelvin.checkoutRepo(repo)
            redis.rpush(repoKey)

        else:
            sleepTime = float(settings.clonerSleepTime)
            log(WarningLevels.Debug(), "No Repos to clone.  Cloning Worker going to sleep...")
            time.sleep(sleepTime)




            <commit_msg>Typos<commit_after>import os
import sys
import atexit
import time

PROJECT_PATH = os.path.abspath(os.path.dirname(__file__))
sys.path.append(os.path.join(PROJECT_PATH, '..', '..'))

from pygithub3 import Github

import src.todoMelvin
from src.todoMelvin import settings
from src.todoLogging import WarningLevels, log
from src.db.todoRepos import RepoQueues, Repo

def exitHandler():
    log(WarningLevels.Info(), "Cloning Worker shutting down...")


if __name__ == "__main__":
    atexit.register(exitHandler)

    log(WarningLevels.Info(), "Starting Cloning Worker.")

    redis = src.db.todoRedis.connect()
    gh = Github(login = settings.ghLogin, password = settings.ghPassword)

    while True:
        try:
            cloneCount = redis.llen(RepoQueues.Cloning)
            parseCount = redis.llen(RepoQueues.Parsing)
        except:
            log(WarningLevels.Fatal(), "Cloning Worker unable to reach Redis")
            sys.exit()

        if cloneCount > 0 and parseCount < int(settings.maxParseQueueCount):
            repoKey = redis.lpop(RepoQueues.Cloning)
            
            repo = Repo()
            repo.loadFromKey(repoKey)

            #sanity check our loaded key
            assert repo.key() == repoKey, "Bad repo saved in cloning Queue! Key %s not found!"%(repoKey)

            #clone the repo and add it to the parse queue
            src.todoMelvin.checkoutRepo(repo)
            redis.rpush(RepoQueues.Parsing, repoKey)

        else:
            sleepTime = float(settings.clonerSleepTime)
            log(WarningLevels.Debug(), "No Repos to clone.  Cloning Worker going to sleep...")
            time.sleep(sleepTime)




            
{'fraction_non_alphanumeric': 0.10027018913239268, 'fraction_numerical': 0.0012008405884118883, 'mean_word_length': 2.540913921360255, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4700841', 'n_tokens_mistral': 1065, 'n_tokens_neox': 1012, 'n_words': 249}