<commit_before>import logging
import requests
from multiprocessing import Process, Queue
import time
import sqlalchemy as s
import pandas as pd
import os
import json
logging.basicConfig(filename='housekeeper.log')

class Housekeeper:

    def __init__(self, jobs, broker, broker_host, broker_port, user, password, host, port, dbname):

        self.broker_host = broker_host
        self.broker_port = broker_port
        self.broker = broker
        DB_STR = 'postgresql://{}:{}@{}:{}/{}'.format(
            user, password, host, port, dbname
        )

        dbschema='augur_data'
        self.db = s.create_engine(DB_STR, poolclass=s.pool.NullPool,
            connect_args={'options': '-csearch_path={}'.format(dbschema)})

        helper_schema = 'augur_operations'
        self.helper_db = s.create_engine(DB_STR, poolclass = s.pool.NullPool,
            connect_args={'options': '-csearch_path={}'.format(helper_schema)})

        repoUrlSQL = s.sql.text("""
            SELECT repo_git FROM repo
        """)

        rs = pd.read_sql(repoUrlSQL, self.db, params={})

        all_repos = rs['repo_git'].values.tolist()

        # List of tasks that need periodic updates
        self.__updatable = self.prep_jobs(jobs)

        self.__processes = []
        self.__updater()

    @staticmethod
    def updater_process(broker_host, broker_port, broker, model, given, delay, repos, repo_group_id=None):
        """
        Controls a given plugin's update process
        :param name: name of object to be updated 
        :param delay: time needed to update
        :param shared: shared object that is to also be updated
        """
        if repo_group_id:
            logging.info('Housekeeper spawned {} model updater process for subsection {} with PID {}'.format(model, repo_group_id, os.getpid()))
        else:
            logging.info('Housekeeper spawned {} model updater process for repo {} with PID {}'.format(model, repos[0]['repo_id'], os.getpid()))

        try:
            compatible_worker_found = False
            # Waiting for compatible worker
            while True:
                for worker in list(broker._getvalue().keys()):
                    # logging.info("{} {} {} {} {}".format(worker, model, broker[worker]['models'], given, broker[worker]['given']))
                    if model in broker[worker]['models'] and given in broker[worker]['given']:
                        compatible_worker_found = True
                if compatible_worker_found:
                    logging.info("Housekeeper recognized that the broker has a worker that " + 
                        "can handle the {} model... beginning to distribute maintained tasks".format(model))
                    time.sleep(4)
                    while True:
                        logging.info('Housekeeper updating {} model with given {}...'.format(
                            model, given[0]))
                        
                        if given[0] == 'git_url' or given[0] == 'github_url':
                            for repo in repos:
                                if given[0] == 'github_url' and 'github' not in repo['repo_git']:
                                    continue
                                given_key = 'git_url' if given[0] == 'git_url' else 'github_url'
                                task = {
                                    "job_type": "MAINTAIN", 
                                    "models": [model], 
                                    "display_name": "{} model for url: {}".format(model, repo['repo_git']),
                                    "given": {}
                                }
                                task['given'][given_key] = repo['repo_git']
                                if "focused_task" in repo:
                                    task["focused_task"] = repo['focused_task']
                                try:
                                    requests.post('http://{}:{}/api/unstable/task'.format(
                                        broker_host,broker_port), json=task, timeout=10)
                                except Exception as e:
                                    logging.info("Error encountered: {}".format(e))

                                time.sleep(0.5)
                        elif given[0] == 'repo_group':
                            task = {
                                    "job_type": "MAINTAIN", 
                                    "models": [model], 
                                    "display_name": "{} model for repo group id: {}".format(model, repo_group_id),
                                    "given": {
                                        "repo_group": repos<commit_msg>extend the condition to be a github url to 'github.com'<commit_after>import logging
import requests
from multiprocessing import Process, Queue
import time
import sqlalchemy as s
import pandas as pd
import os
import json
logging.basicConfig(filename='housekeeper.log')

class Housekeeper:

    def __init__(self, jobs, broker, broker_host, broker_port, user, password, host, port, dbname):

        self.broker_host = broker_host
        self.broker_port = broker_port
        self.broker = broker
        DB_STR = 'postgresql://{}:{}@{}:{}/{}'.format(
            user, password, host, port, dbname
        )

        dbschema='augur_data'
        self.db = s.create_engine(DB_STR, poolclass=s.pool.NullPool,
            connect_args={'options': '-csearch_path={}'.format(dbschema)})

        helper_schema = 'augur_operations'
        self.helper_db = s.create_engine(DB_STR, poolclass = s.pool.NullPool,
            connect_args={'options': '-csearch_path={}'.format(helper_schema)})

        repoUrlSQL = s.sql.text("""
            SELECT repo_git FROM repo
        """)

        rs = pd.read_sql(repoUrlSQL, self.db, params={})

        all_repos = rs['repo_git'].values.tolist()

        # List of tasks that need periodic updates
        self.__updatable = self.prep_jobs(jobs)

        self.__processes = []
        self.__updater()

    @staticmethod
    def updater_process(broker_host, broker_port, broker, model, given, delay, repos, repo_group_id=None):
        """
        Controls a given plugin's update process
        :param name: name of object to be updated 
        :param delay: time needed to update
        :param shared: shared object that is to also be updated
        """
        if repo_group_id:
            logging.info('Housekeeper spawned {} model updater process for subsection {} with PID {}'.format(model, repo_group_id, os.getpid()))
        else:
            logging.info('Housekeeper spawned {} model updater process for repo {} with PID {}'.format(model, repos[0]['repo_id'], os.getpid()))

        try:
            compatible_worker_found = False
            # Waiting for compatible worker
            while True:
                for worker in list(broker._getvalue().keys()):
                    # logging.info("{} {} {} {} {}".format(worker, model, broker[worker]['models'], given, broker[worker]['given']))
                    if model in broker[worker]['models'] and given in broker[worker]['given']:
                        compatible_worker_found = True
                if compatible_worker_found:
                    logging.info("Housekeeper recognized that the broker has a worker that " + 
                        "can handle the {} model... beginning to distribute maintained tasks".format(model))
                    time.sleep(4)
                    while True:
                        logging.info('Housekeeper updating {} model with given {}...'.format(
                            model, given[0]))
                        
                        if given[0] == 'git_url' or given[0] == 'github_url':
                            for repo in repos:
                                if given[0] == 'github_url' and 'github.com' not in repo['repo_git']:
                                    continue
                                given_key = 'git_url' if given[0] == 'git_url' else 'github_url'
                                task = {
                                    "job_type": "MAINTAIN", 
                                    "models": [model], 
                                    "display_name": "{} model for url: {}".format(model, repo['repo_git']),
                                    "given": {}
                                }
                                task['given'][given_key] = repo['repo_git']
                                if "focused_task" in repo:
                                    task["focused_task"] = repo['focused_task']
                                try:
                                    requests.post('http://{}:{}/api/unstable/task'.format(
                                        broker_host,broker_port), json=task, timeout=10)
                                except Exception as e:
                                    logging.info("Error encountered: {}".format(e))

                                time.sleep(0.5)
                        elif given[0] == 'repo_group':
                            task = {
                                    "job_type": "MAINTAIN", 
                                    "models": [model], 
                                    "display_name": "{} model for repo group id: {}".format(model, repo_group_id),
                                    "given": {
                                        "repo_group": repos
{'fraction_non_alphanumeric': 0.10880609652836579, 'fraction_numerical': 0.002540220152413209, 'mean_word_length': 1.4593961478396669, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4716807', 'n_tokens_mistral': 2441, 'n_tokens_neox': 2273, 'n_words': 592}